From 677ef6e304a146e7d3c0652573121b0d12b3f4ea Mon Sep 17 00:00:00 2001 From: Nicholas Reich Date: Wed, 2 Sep 2026 16:28:02 -0400 Subject: [PATCH 1/2] Add the hubverse model output interface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The reason for the rewrite: `alloscore_model_out()` takes a hubverse `model_out_tbl` and `oracle_output` and scores them, replacing roughly thirty lines of per-analysis boilerplate (nest `ps`/`qs`, `deframe`, `add_pdqr_funs(dist = "distfromq")`, join truth) reproduced in every downstream script — see `utility-eval-papers/R/run-alloscore.R`. The one genuinely new design decision is grouping. An allocation problem is a *set* of targets sharing one budget, which hubverse has no single column for, so `target_cols` names the task ID columns whose combinations enumerate those targets and the allocation unit is derived as `model_id` plus everything else. That mirrors `compound_taskid_set`: the columns you name vary within a group, the rest are held constant. `as_alloscore_df()` is exported as well, since it is where a mis-specified join or target set shows up. Validation follows `hubEvals`: `get_task_id_cols()` and `validate_model_oracle_out()` mirror its helpers, and a single supported `output_type` is enforced up front. `test-model_out.R` includes a test that one allocation problem routed through the hubverse API and through the core API gives the same score. Co-Authored-By: Claude Opus 5 (1M context) --- DESCRIPTION | 4 + NAMESPACE | 3 + NEWS.md | 6 + R/model_out.R | 361 ++++++++++++++++++++++++++++++++ R/validate.R | 176 ++++++++++++++++ man/allocate_model_out.Rd | 60 ++++++ man/alloscore_model_out.Rd | 83 ++++++++ man/as_alloscore_df.Rd | 53 +++++ tests/testthat/test-model_out.R | 259 +++++++++++++++++++++++ tests/testthat/test-validate.R | 147 +++++++++++++ 10 files changed, 1152 insertions(+) create mode 100644 R/model_out.R create mode 100644 R/validate.R create mode 100644 man/allocate_model_out.Rd create mode 100644 man/alloscore_model_out.Rd create mode 100644 man/as_alloscore_df.Rd create mode 100644 tests/testthat/test-model_out.R create mode 100644 tests/testthat/test-validate.R diff --git a/DESCRIPTION b/DESCRIPTION index 3873ba4..9ce6459 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -27,6 +27,7 @@ Imports: cli, distfromq, dplyr, + hubUtils, purrr, rlang, stats, @@ -35,7 +36,10 @@ Imports: tidyselect Suggests: covr, + hubExamples, testthat (>= 3.0.0) +Remotes: + hubverse-org/hubExamples Depends: R (>= 4.1.0) LazyData: true diff --git a/NAMESPACE b/NAMESPACE index 7e07d27..3df41f4 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -6,7 +6,10 @@ S3method(alloscore,slim) S3method(weights,allocated) export(add_pdqr_funs) export(allocate) +export(allocate_model_out) export(alloscore) +export(alloscore_model_out) +export(as_alloscore_df) export(dexp_gpl_df) export(dexp_gpl_loss) export(dexp_over_loss) diff --git a/NEWS.md b/NEWS.md index e437455..4216601 100644 --- a/NEWS.md +++ b/NEWS.md @@ -42,3 +42,9 @@ * `weights.allocated()` and `alloscore.slim()` now match the signatures of their generics. + +* Added `as_alloscore_df()`, `allocate_model_out()` and `alloscore_model_out()`, + which take a hubverse `model_out_tbl` and `oracle_output` directly. Predictive + quantiles are turned into cdfs and quantile functions with `distfromq`, so the + conversion that callers used to hand-roll is no longer needed. Only the + `quantile` output type is supported. diff --git a/R/model_out.R b/R/model_out.R new file mode 100644 index 0000000..5d950b2 --- /dev/null +++ b/R/model_out.R @@ -0,0 +1,361 @@ +#' Convert hubverse model output into forecasts alloscore2 can allocate against +#' +#' Groups a hubverse model output table into allocation problems, turns each +#' target's predictive quantiles into a cdf and quantile function using +#' [distfromq::make_p_fn()] and [distfromq::make_q_fn()], and attaches the +#' observed outcomes from `oracle_output`. +#' +#' An allocation problem is a set of targets that share one budget. +#' `target_cols` names the task ID columns whose combinations enumerate those +#' targets; every other task ID column, together with `model_id`, defines the +#' *allocation unit*, and one allocation problem is solved per combination of +#' those. This mirrors the way a hubverse `compound_taskid_set` distinguishes +#' task IDs that vary within a group from those held constant. +#' +#' @param model_out_tbl a hubverse model output table containing a single +#' `output_type`, which must be `"quantile"`. +#' @param oracle_output hubverse oracle output, holding the observed values in +#' an `oracle_value` column. Optional when only allocating. +#' @param target_cols character vector naming the task ID columns whose +#' combinations enumerate the targets that share a budget, for example +#' `"location"`. +#' +#' @return A tibble with one row per allocation unit, the allocation unit +#' columns, and a `forecasts` list column. Each element of `forecasts` is a +#' tibble with one row per target holding the `target_cols`, a +#' `target_names` key, the predictive quantile levels `ps` and values `qs`, +#' the cdf `F` and quantile function `Q`, and -- when `oracle_output` was +#' supplied -- the observed outcome `y`. +#' +#' @examplesIf requireNamespace("hubExamples", quietly = TRUE) +#' mot <- dplyr::filter(hubExamples::forecast_outputs, output_type == "quantile") +#' adf <- as_alloscore_df( +#' mot, +#' oracle_output = hubExamples::forecast_oracle_output, +#' target_cols = "location" +#' ) +#' adf +#' adf$forecasts[[1]] +#' +#' @export +as_alloscore_df <- function(model_out_tbl, oracle_output = NULL, target_cols) { + validate_output_type(model_out_tbl) + units <- resolve_allocation_unit(model_out_tbl, target_cols) + target_cols <- units$target_cols + allocation_unit <- units$allocation_unit + + if (!is.null(oracle_output)) { + model_out_tbl <- validate_model_oracle_out(model_out_tbl, oracle_output) + } else if (!inherits(model_out_tbl, "model_out_tbl")) { + model_out_tbl <- hubUtils::as_model_out_tbl(model_out_tbl) + } + + # a non-numeric output_type_id warns on coercion; the check below reports it + dat <- suppressWarnings(dplyr::mutate( + tibble::as_tibble(model_out_tbl), + output_type_id = as.numeric(.data$output_type_id) + )) + if (anyNA(dat[["output_type_id"]])) { + cli::cli_abort( + "Some {.field output_type_id} values are not numeric quantile levels." + ) + } + + # one row per target, holding that target's predictive quantiles + key_cols <- c(allocation_unit, target_cols) + dat <- dplyr::arrange( + dat, + dplyr::pick(tidyselect::all_of(key_cols)), + .data$output_type_id + ) + dup <- dplyr::filter( + dplyr::count( + dat, + dplyr::pick(tidyselect::all_of(c(key_cols, "output_type_id"))) + ), + .data$n > 1L + ) + if (nrow(dup) > 0) { + cli::cli_abort(c( + "{.arg model_out_tbl} has {nrow(dup)} duplicated forecast{?s}.", + "i" = "Each combination of {.val {key_cols}} and {.field output_type_id} must appear once." + )) + } + + nested <- dplyr::summarise( + dplyr::group_by(dat, dplyr::pick(tidyselect::all_of(key_cols))), + ps = list(.data$output_type_id), + qs = list(.data$value), + .groups = "drop" + ) + + if (!is.null(oracle_output)) { + oracle <- tibble::as_tibble(oracle_output) + if ("output_type" %in% colnames(oracle)) { + oracle <- dplyr::filter(oracle, .data$output_type == "quantile") + oracle <- dplyr::select( + oracle, + -tidyselect::any_of(c("output_type", "output_type_id")) + ) + } + join_cols <- intersect(key_cols, colnames(oracle)) + oracle <- dplyr::distinct( + dplyr::select(oracle, tidyselect::all_of(c(join_cols, "oracle_value"))) + ) + nested <- dplyr::left_join( + nested, + oracle, + by = join_cols, + relationship = "many-to-one" + ) + missing_rows <- which(is.na(nested[["oracle_value"]])) + if (length(missing_rows) > 0) { + cli::cli_abort(c( + "{length(missing_rows)} forecast{?s} {?has/have} no matching value in \ + {.arg oracle_output}.", + "i" = "Joined on {.val {join_cols}}." + )) + } + nested <- dplyr::rename(nested, y = "oracle_value") + } + + # build the cdf and quantile function for each target + nested[["F"]] <- purrr::map2( + nested[["ps"]], + nested[["qs"]], + function(ps, qs) distfromq::make_p_fn(ps = ps, qs = qs) + ) + nested[["Q"]] <- purrr::map2( + nested[["ps"]], + nested[["qs"]], + function(ps, qs) distfromq::make_q_fn(ps = ps, qs = qs) + ) + nested[["target_names"]] <- make_target_key(nested, target_cols) + + out <- dplyr::summarise( + dplyr::group_by(nested, dplyr::pick(tidyselect::all_of(allocation_unit))), + forecasts = list(dplyr::pick(tidyselect::everything())), + .groups = "drop" + ) + # keep the target-level columns in a predictable order + out[["forecasts"]] <- purrr::map(out[["forecasts"]], function(fc) { + dplyr::relocate( + fc, + tidyselect::all_of(c(target_cols, "target_names")), + tidyselect::any_of("y"), + "ps", + "qs", + "F", + "Q" + ) + }) + structure(out, target_cols = target_cols, allocation_unit = allocation_unit) +} + +#' Build a single character key naming each target +#' +#' @noRd +make_target_key <- function(dat, target_cols) { + if (length(target_cols) == 1L) { + return(as.character(dat[[target_cols]])) + } + do.call( + paste, + c(lapply(target_cols, function(col) as.character(dat[[col]])), sep = "|") + ) +} + +#' Allocate a budget across hubverse model output +#' +#' Runs [allocate()] once per allocation unit of a hubverse model output table. +#' See [as_alloscore_df()] for how allocation units are determined. +#' +#' @inheritParams as_alloscore_df +#' @inheritParams allocate +#' @param w allocation weights: a scalar, a vector named by target, a vector +#' ordered as the targets of each allocation problem, or the name of a column +#' of `model_out_tbl` holding a per-target weight. +#' @param ... further arguments passed to [allocate()]. +#' +#' @return A tibble with one row per allocation unit and value of `K`, holding +#' the allocation unit columns and the columns returned by [allocate()]. Unlike +#' [allocate()] this is a plain tibble rather than an `allocated` object, since +#' each unit has its own loss functions and weights. +#' +#' @examplesIf requireNamespace("hubExamples", quietly = TRUE) +#' mot <- dplyr::filter( +#' hubExamples::forecast_outputs, +#' output_type == "quantile", reference_date == "2022-11-19", horizon == 0 +#' ) +#' allocate_model_out(mot, K = c(100, 500), target_cols = "location") +#' +#' @export +allocate_model_out <- function( + model_out_tbl, + K, + target_cols, + w = 1, + kappa = 1, + alpha = 1, + g = "x", + ... +) { + adf <- as_alloscore_df( + model_out_tbl, + oracle_output = NULL, + target_cols = target_cols + ) + run_over_units( + adf, + function(forecasts, weights) { + allocate( + df = forecasts, + K = K, + target_names = "target_names", + w = weights, + kappa = kappa, + alpha = alpha, + g = g, + ... + ) + }, + w = w + ) +} + +#' Score hubverse model output with an allocation scoring rule +#' +#' For each allocation unit of a hubverse model output table, allocates a budget +#' under the model's forecasts and scores the realized loss of that allocation +#' relative to the loss of an oracle that knew the observed outcomes. See +#' [as_alloscore_df()] for how allocation units are determined, and +#' [alloscore()] for the score itself. +#' +#' @inheritParams as_alloscore_df +#' @inheritParams allocate_model_out +#' @param against_oracle logical; if `TRUE`, scores relative to the oracle +#' allocation are included. +#' @param summarize logical; if `TRUE`, average the scores over the columns +#' named in `by`. +#' @param by character vector naming the columns to summarize by. Must be a +#' subset of the allocation unit columns and `"K"`. +#' +#' @return A tibble of scores. Unsummarized, one row per allocation unit and +#' value of `K`, with columns `K`, `score`, `score_raw`, `score_oracle`, +#' `ytot` and a nested `xdf` of per-target detail. Summarized, one row per +#' combination of `by` with the mean of each score. +#' +#' @examplesIf requireNamespace("hubExamples", quietly = TRUE) +#' mot <- dplyr::filter(hubExamples::forecast_outputs, output_type == "quantile") +#' scores <- alloscore_model_out( +#' model_out_tbl = mot, +#' oracle_output = hubExamples::forecast_oracle_output, +#' K = c(500, 1000), +#' target_cols = "location", +#' by = c("model_id", "K") +#' ) +#' scores +#' +#' @export +alloscore_model_out <- function( + model_out_tbl, + oracle_output, + K, + target_cols, + w = 1, + kappa = 1, + alpha = 1, + g = "x", + against_oracle = TRUE, + summarize = TRUE, + by = c("model_id", "K"), + ... +) { + adf <- as_alloscore_df( + model_out_tbl, + oracle_output = oracle_output, + target_cols = target_cols + ) + allocation_unit <- attr(adf, "allocation_unit") + + scores <- run_over_units( + adf, + function(forecasts, weights) { + a <- allocate( + df = forecasts, + K = K, + target_names = "target_names", + w = weights, + kappa = kappa, + alpha = alpha, + g = g, + ... + ) + alloscore( + a, + y = rlang::set_names(forecasts[["y"]], forecasts[["target_names"]]), + against_oracle = against_oracle + ) + }, + w = w + ) + + score_cols <- intersect( + c("score", "score_raw", "score_oracle", "ytot"), + colnames(scores) + ) + scores <- dplyr::relocate( + scores, + tidyselect::all_of(c(allocation_unit, "K", score_cols)) + ) + scores <- dplyr::select( + scores, + tidyselect::all_of(c(allocation_unit, "K", score_cols)), + tidyselect::any_of("xdf") + ) + + if (!summarize) { + return(scores) + } + valid_by <- c(allocation_unit, "K") + unknown <- setdiff(by, valid_by) + if (length(unknown) > 0) { + cli::cli_abort(c( + "{.arg by} names {length(unknown)} column{?s} that {?is/are} not available.", + "x" = "Not available: {.val {unknown}}.", + "i" = "Expected a subset of {.val {valid_by}}." + )) + } + dplyr::summarise( + dplyr::group_by(scores, dplyr::pick(tidyselect::all_of(by))), + dplyr::across(tidyselect::all_of(score_cols), mean), + .groups = "drop" + ) +} + +#' Run an allocation or scoring function over each allocation unit +#' +#' @param adf output of [as_alloscore_df()]. +#' @param fn a function of `forecasts` and `weights` returning a tibble. +#' @param w weights, as accepted by [allocate_model_out()]. +#' +#' @return A tibble binding the allocation unit columns to each result. +#' +#' @noRd +run_over_units <- function(adf, fn, w) { + allocation_unit <- attr(adf, "allocation_unit") + results <- purrr::map( + seq_len(nrow(adf)), + function(i) { + forecasts <- adf[["forecasts"]][[i]] + weights <- resolve_weights(w, forecasts, forecasts[["target_names"]]) + res <- fn(forecasts, weights) + unit <- adf[i, allocation_unit, drop = FALSE] + dplyr::bind_cols( + unit[rep(1L, nrow(res)), , drop = FALSE], + tibble::as_tibble(res) + ) + } + ) + purrr::list_rbind(results) +} diff --git a/R/validate.R b/R/validate.R new file mode 100644 index 0000000..2af460c --- /dev/null +++ b/R/validate.R @@ -0,0 +1,176 @@ +#' Get the names of the task ID columns of a model output table +#' +#' @param model_out_tbl a hubverse model output table. +#' +#' @return A character vector of column names. +#' +#' @noRd +get_task_id_cols <- function(model_out_tbl) { + non_task_cols <- c( + "model_id", + "output_type", + "output_type_id", + "value", + "model_abbr", + "team_abbr" + ) + setdiff(colnames(model_out_tbl), non_task_cols) +} + +#' Check that a model output table holds a single supported output type +#' +#' @param model_out_tbl a hubverse model output table. +#' +#' @return The single `output_type` present, invisibly. +#' +#' @noRd +validate_output_type <- function(model_out_tbl) { + output_types <- unique(model_out_tbl[["output_type"]]) + if (length(output_types) != 1L) { + cli::cli_abort(c( + "{.arg model_out_tbl} must contain a single {.field output_type}, but it has \\ + {length(output_types)}: {.val {output_types}}.", + "i" = "Filter to one output type before scoring." + )) + } + if (!identical(output_types, "quantile")) { + cli::cli_abort(c( + "{.arg model_out_tbl} has {.field output_type} {.val {output_types}}, which is not supported.", + "i" = "{.pkg alloscore2} currently supports only the {.val quantile} output type." + )) + } + invisible(output_types) +} + +#' Validate a model output table and its oracle output +#' +#' @param model_out_tbl a hubverse model output table. +#' @param oracle_output hubverse oracle output, holding the observed values in +#' an `oracle_value` column. +#' +#' @return `model_out_tbl`, coerced to class `model_out_tbl` if it was not +#' already. +#' +#' @noRd +validate_model_oracle_out <- function(model_out_tbl, oracle_output) { + req_cols <- c("model_id", "output_type", "output_type_id", "value") + missing_cols <- setdiff(req_cols, colnames(model_out_tbl)) + if (length(missing_cols) > 0) { + cli::cli_abort( + "{.arg model_out_tbl} is missing required column{?s} {.val {missing_cols}}." + ) + } + if (!"oracle_value" %in% colnames(oracle_output)) { + cli::cli_abort( + "{.arg oracle_output} must have an {.field oracle_value} column." + ) + } + + task_id_cols <- get_task_id_cols(model_out_tbl) + join_cols <- intersect(task_id_cols, colnames(oracle_output)) + if (length(join_cols) == 0) { + cli::cli_abort( + "{.arg model_out_tbl} and {.arg oracle_output} have no task ID columns in common." + ) + } + expected_cols <- c( + task_id_cols, + "output_type", + "output_type_id", + "oracle_value" + ) + unexpected_cols <- setdiff(colnames(oracle_output), expected_cols) + if (length(unexpected_cols) > 0) { + cli::cli_abort( + "{.arg oracle_output} has {length(unexpected_cols)} unexpected column{?s} \\ + {.val {unexpected_cols}}; expected a subset of {.val {expected_cols}}." + ) + } + + if (!inherits(model_out_tbl, "model_out_tbl")) { + model_out_tbl <- hubUtils::as_model_out_tbl(model_out_tbl) + } + model_out_tbl +} + +#' Validate `target_cols` and derive the allocation unit +#' +#' An allocation problem is a set of targets that share one budget. `target_cols` +#' names the task ID columns whose combinations enumerate those targets; the +#' allocation unit is everything else, following the same logic as a hubverse +#' `compound_taskid_set`. +#' +#' @param model_out_tbl a hubverse model output table. +#' @param target_cols character vector of task ID columns identifying targets +#' that share a budget. +#' +#' @return A list with elements `target_cols` and `allocation_unit`. +#' +#' @noRd +resolve_allocation_unit <- function(model_out_tbl, target_cols) { + if ( + missing(target_cols) || is.null(target_cols) || length(target_cols) == 0 + ) { + cli::cli_abort(c( + "{.arg target_cols} must be supplied.", + "i" = "It names the task ID columns whose combinations enumerate the targets + that share a budget, for example {.code target_cols = \"location\"}." + )) + } + task_id_cols <- get_task_id_cols(model_out_tbl) + unknown <- setdiff(target_cols, task_id_cols) + if (length(unknown) > 0) { + cli::cli_abort(c( + "{.arg target_cols} must name task ID columns of {.arg model_out_tbl}.", + "x" = "Not a task ID column: {.val {unknown}}.", + "i" = "Available task ID columns: {.val {task_id_cols}}." + )) + } + allocation_unit <- c("model_id", setdiff(task_id_cols, target_cols)) + list(target_cols = target_cols, allocation_unit = allocation_unit) +} + +#' Resolve allocation weights against the targets of one allocation problem +#' +#' @param w scalar, vector, or the name of a column of `model_out_tbl`. +#' @param forecasts the target-level tibble for one allocation unit. +#' @param target_key character vector naming each target. +#' +#' @return A numeric vector of weights, one per target. +#' +#' @noRd +resolve_weights <- function(w, forecasts, target_key) { + n <- length(target_key) + if (is.character(w) && length(w) == 1L) { + if (!w %in% colnames(forecasts)) { + cli::cli_abort( + "{.arg w} names column {.val {w}}, which is not present in {.arg model_out_tbl}." + ) + } + return(forecasts[[w]]) + } + if (!is.numeric(w)) { + cli::cli_abort( + "{.arg w} must be numeric or the name of a column of {.arg model_out_tbl}." + ) + } + if (length(w) == 1L && is.null(names(w))) { + return(rep(w, n)) + } + if (!is.null(names(w))) { + missing_targets <- setdiff(target_key, names(w)) + if (length(missing_targets) > 0) { + cli::cli_abort( + "{.arg w} has no entry for target{?s} {.val {missing_targets}}." + ) + } + return(unname(w[target_key])) + } + if (length(w) != n) { + cli::cli_abort( + "{.arg w} has length {length(w)} but there are {n} targets; supply a scalar, a \\ + named vector, or a vector of length {n}." + ) + } + w +} diff --git a/man/allocate_model_out.Rd b/man/allocate_model_out.Rd new file mode 100644 index 0000000..79f6e9f --- /dev/null +++ b/man/allocate_model_out.Rd @@ -0,0 +1,60 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/model_out.R +\name{allocate_model_out} +\alias{allocate_model_out} +\title{Allocate a budget across hubverse model output} +\usage{ +allocate_model_out( + model_out_tbl, + K, + target_cols, + w = 1, + kappa = 1, + alpha = 1, + g = "x", + ... +) +} +\arguments{ +\item{model_out_tbl}{a hubverse model output table containing a single +\code{output_type}, which must be \code{"quantile"}.} + +\item{K}{vector of budgets. Cannot be supplied via \code{df}.} + +\item{target_cols}{character vector naming the task ID columns whose +combinations enumerate the targets that share a budget, for example +\code{"location"}.} + +\item{w}{allocation weights: a scalar, a vector named by target, a vector +ordered as the targets of each allocation problem, or the name of a column +of \code{model_out_tbl} holding a per-target weight.} + +\item{kappa}{scale factor.} + +\item{alpha}{normalized loss when the outcome \code{y} exceeds the allocation \code{x}. +Exactly one of \code{alpha} and \code{U} must be supplied.} + +\item{g}{a non-decreasing increment function, supplied either as a function +or as a string in the variable \code{x} such as \code{"log(x)"}.} + +\item{...}{further arguments passed to \code{\link[=allocate]{allocate()}}.} +} +\value{ +A tibble with one row per allocation unit and value of \code{K}, holding +the allocation unit columns and the columns returned by \code{\link[=allocate]{allocate()}}. Unlike +\code{\link[=allocate]{allocate()}} this is a plain tibble rather than an \code{allocated} object, since +each unit has its own loss functions and weights. +} +\description{ +Runs \code{\link[=allocate]{allocate()}} once per allocation unit of a hubverse model output table. +See \code{\link[=as_alloscore_df]{as_alloscore_df()}} for how allocation units are determined. +} +\examples{ +\dontshow{if (requireNamespace("hubExamples", quietly = TRUE)) withAutoprint(\{ # examplesIf} +mot <- dplyr::filter( + hubExamples::forecast_outputs, + output_type == "quantile", reference_date == "2022-11-19", horizon == 0 +) +allocate_model_out(mot, K = c(100, 500), target_cols = "location") +\dontshow{\}) # examplesIf} +} diff --git a/man/alloscore_model_out.Rd b/man/alloscore_model_out.Rd new file mode 100644 index 0000000..074dfc0 --- /dev/null +++ b/man/alloscore_model_out.Rd @@ -0,0 +1,83 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/model_out.R +\name{alloscore_model_out} +\alias{alloscore_model_out} +\title{Score hubverse model output with an allocation scoring rule} +\usage{ +alloscore_model_out( + model_out_tbl, + oracle_output, + K, + target_cols, + w = 1, + kappa = 1, + alpha = 1, + g = "x", + against_oracle = TRUE, + summarize = TRUE, + by = c("model_id", "K"), + ... +) +} +\arguments{ +\item{model_out_tbl}{a hubverse model output table containing a single +\code{output_type}, which must be \code{"quantile"}.} + +\item{oracle_output}{hubverse oracle output, holding the observed values in +an \code{oracle_value} column. Optional when only allocating.} + +\item{K}{vector of budgets. Cannot be supplied via \code{df}.} + +\item{target_cols}{character vector naming the task ID columns whose +combinations enumerate the targets that share a budget, for example +\code{"location"}.} + +\item{w}{allocation weights: a scalar, a vector named by target, a vector +ordered as the targets of each allocation problem, or the name of a column +of \code{model_out_tbl} holding a per-target weight.} + +\item{kappa}{scale factor.} + +\item{alpha}{normalized loss when the outcome \code{y} exceeds the allocation \code{x}. +Exactly one of \code{alpha} and \code{U} must be supplied.} + +\item{g}{a non-decreasing increment function, supplied either as a function +or as a string in the variable \code{x} such as \code{"log(x)"}.} + +\item{against_oracle}{logical; if \code{TRUE}, scores relative to the oracle +allocation are included.} + +\item{summarize}{logical; if \code{TRUE}, average the scores over the columns +named in \code{by}.} + +\item{by}{character vector naming the columns to summarize by. Must be a +subset of the allocation unit columns and \code{"K"}.} + +\item{...}{further arguments passed to \code{\link[=allocate]{allocate()}}.} +} +\value{ +A tibble of scores. Unsummarized, one row per allocation unit and +value of \code{K}, with columns \code{K}, \code{score}, \code{score_raw}, \code{score_oracle}, +\code{ytot} and a nested \code{xdf} of per-target detail. Summarized, one row per +combination of \code{by} with the mean of each score. +} +\description{ +For each allocation unit of a hubverse model output table, allocates a budget +under the model's forecasts and scores the realized loss of that allocation +relative to the loss of an oracle that knew the observed outcomes. See +\code{\link[=as_alloscore_df]{as_alloscore_df()}} for how allocation units are determined, and +\code{\link[=alloscore]{alloscore()}} for the score itself. +} +\examples{ +\dontshow{if (requireNamespace("hubExamples", quietly = TRUE)) withAutoprint(\{ # examplesIf} +mot <- dplyr::filter(hubExamples::forecast_outputs, output_type == "quantile") +scores <- alloscore_model_out( + model_out_tbl = mot, + oracle_output = hubExamples::forecast_oracle_output, + K = c(500, 1000), + target_cols = "location", + by = c("model_id", "K") +) +scores +\dontshow{\}) # examplesIf} +} diff --git a/man/as_alloscore_df.Rd b/man/as_alloscore_df.Rd new file mode 100644 index 0000000..013d27a --- /dev/null +++ b/man/as_alloscore_df.Rd @@ -0,0 +1,53 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/model_out.R +\name{as_alloscore_df} +\alias{as_alloscore_df} +\title{Convert hubverse model output into forecasts alloscore2 can allocate against} +\usage{ +as_alloscore_df(model_out_tbl, oracle_output = NULL, target_cols) +} +\arguments{ +\item{model_out_tbl}{a hubverse model output table containing a single +\code{output_type}, which must be \code{"quantile"}.} + +\item{oracle_output}{hubverse oracle output, holding the observed values in +an \code{oracle_value} column. Optional when only allocating.} + +\item{target_cols}{character vector naming the task ID columns whose +combinations enumerate the targets that share a budget, for example +\code{"location"}.} +} +\value{ +A tibble with one row per allocation unit, the allocation unit +columns, and a \code{forecasts} list column. Each element of \code{forecasts} is a +tibble with one row per target holding the \code{target_cols}, a +\code{target_names} key, the predictive quantile levels \code{ps} and values \code{qs}, +the cdf \code{F} and quantile function \code{Q}, and -- when \code{oracle_output} was +supplied -- the observed outcome \code{y}. +} +\description{ +Groups a hubverse model output table into allocation problems, turns each +target's predictive quantiles into a cdf and quantile function using +\code{\link[distfromq:make_p_fn]{distfromq::make_p_fn()}} and \code{\link[distfromq:make_q_fn]{distfromq::make_q_fn()}}, and attaches the +observed outcomes from \code{oracle_output}. +} +\details{ +An allocation problem is a set of targets that share one budget. +\code{target_cols} names the task ID columns whose combinations enumerate those +targets; every other task ID column, together with \code{model_id}, defines the +\emph{allocation unit}, and one allocation problem is solved per combination of +those. This mirrors the way a hubverse \code{compound_taskid_set} distinguishes +task IDs that vary within a group from those held constant. +} +\examples{ +\dontshow{if (requireNamespace("hubExamples", quietly = TRUE)) withAutoprint(\{ # examplesIf} +mot <- dplyr::filter(hubExamples::forecast_outputs, output_type == "quantile") +adf <- as_alloscore_df( + mot, + oracle_output = hubExamples::forecast_oracle_output, + target_cols = "location" +) +adf +adf$forecasts[[1]] +\dontshow{\}) # examplesIf} +} diff --git a/tests/testthat/test-model_out.R b/tests/testthat/test-model_out.R new file mode 100644 index 0000000..d6a4fed --- /dev/null +++ b/tests/testthat/test-model_out.R @@ -0,0 +1,259 @@ +test_that("as_alloscore_df nests one row per allocation unit", { + adf <- as_alloscore_df( + mini_model_out(), + mini_oracle_out(), + target_cols = "location" + ) + # 2 models x 1 reference_date x 1 horizon + expect_equal(nrow(adf), 2) + expect_true("forecasts" %in% names(adf)) + expect_setequal( + attr(adf, "allocation_unit"), + c("model_id", "reference_date", "target", "horizon", "target_end_date") + ) + expect_equal(attr(adf, "target_cols"), "location") +}) + +test_that("as_alloscore_df builds a cdf and quantile function per target", { + adf <- as_alloscore_df( + mini_model_out(), + mini_oracle_out(), + target_cols = "location" + ) + fc <- adf$forecasts[[1]] + expect_equal(nrow(fc), 2) + expect_true(all( + c("location", "target_names", "y", "ps", "qs", "F", "Q") %in% names(fc) + )) + expect_type(fc$F, "list") + # the predictive quantiles are reproduced by the interpolated functions + expect_equal(fc$Q[[1]](fc$ps[[1]]), fc$qs[[1]], tolerance = 1e-6) + expect_equal(fc$F[[1]](fc$qs[[1]]), fc$ps[[1]], tolerance = 1e-6) +}) + +test_that("as_alloscore_df sorts quantile levels ascending", { + shuffled <- mini_model_out()[sample(nrow(mini_model_out())), ] + adf <- as_alloscore_df(shuffled, mini_oracle_out(), target_cols = "location") + for (fc in adf$forecasts) { + for (ps in fc$ps) { + expect_equal(ps, sort(ps)) + } + for (qs in fc$qs) { + expect_equal(qs, sort(qs)) + } + } +}) + +test_that("as_alloscore_df attaches the observed outcomes", { + adf <- as_alloscore_df( + mini_model_out(), + mini_oracle_out(), + target_cols = "location" + ) + for (fc in adf$forecasts) { + expect_equal(fc$y[fc$location == "a"], 45) + expect_equal(fc$y[fc$location == "b"], 95) + } +}) + +test_that("as_alloscore_df works without oracle output", { + adf <- as_alloscore_df(mini_model_out(), target_cols = "location") + expect_false("y" %in% names(adf$forecasts[[1]])) + expect_true("F" %in% names(adf$forecasts[[1]])) +}) + +test_that("as_alloscore_df builds a compound key for several target columns", { + mot <- dplyr::bind_rows( + mini_model_out(), + dplyr::mutate(mini_model_out(), target = "inc death") + ) + oo <- dplyr::bind_rows( + mini_oracle_out(), + dplyr::mutate(mini_oracle_out(), target = "inc death") + ) + adf <- as_alloscore_df(mot, oo, target_cols = c("location", "target")) + # location and target now vary within an allocation unit + expect_equal(nrow(adf), 2) + fc <- adf$forecasts[[1]] + expect_equal(nrow(fc), 4) + expect_true(all(grepl("\\|", fc$target_names))) +}) + +test_that("as_alloscore_df rejects duplicated forecasts", { + mot <- dplyr::bind_rows(mini_model_out(), mini_model_out()) + expect_error( + as_alloscore_df(mot, mini_oracle_out(), target_cols = "location"), + regexp = "duplicated forecast" + ) +}) + +test_that("as_alloscore_df errors when the oracle output does not cover the forecasts", { + oo <- dplyr::filter(mini_oracle_out(), .data$location == "a") + expect_error( + as_alloscore_df(mini_model_out(), oo, target_cols = "location"), + regexp = "no matching value in" + ) +}) + +test_that("as_alloscore_df rejects non-numeric quantile levels", { + mot <- dplyr::mutate(mini_model_out(), output_type_id = "low") + expect_error( + as_alloscore_df(mot, mini_oracle_out(), target_cols = "location"), + regexp = "not numeric quantile levels" + ) +}) + +test_that("allocate_model_out allocates once per unit and budget", { + res <- allocate_model_out( + mini_model_out(), + K = c(50, 200), + target_cols = "location" + ) + expect_equal(nrow(res), 4) + expect_true(all(c("model_id", "K", "x", "xdf") %in% names(res))) + expect_setequal(res$K, c(50, 200)) + expect_setequal(res$model_id, c("m1", "m2")) +}) + +test_that("alloscore_model_out returns one row per unit and budget when unsummarized", { + res <- alloscore_model_out( + mini_model_out(), + mini_oracle_out(), + K = c(50, 200), + target_cols = "location", + summarize = FALSE + ) + expect_equal(nrow(res), 4) + expect_true(all( + c("model_id", "K", "score", "score_raw", "score_oracle", "ytot", "xdf") %in% + names(res) + )) + expect_equal(res$score, res$score_raw - res$score_oracle) +}) + +test_that("alloscore_model_out summarizes by the requested columns", { + res <- alloscore_model_out( + mini_model_out(), + mini_oracle_out(), + K = c(50, 200), + target_cols = "location", + by = c("model_id", "K") + ) + expect_equal(nrow(res), 4) + expect_setequal( + names(res), + c("model_id", "K", "score", "score_raw", "score_oracle", "ytot") + ) + # summarizing by model alone averages over budgets + by_model <- alloscore_model_out( + mini_model_out(), + mini_oracle_out(), + K = c(50, 200), + target_cols = "location", + by = "model_id" + ) + expect_equal(nrow(by_model), 2) +}) + +test_that("the summarized scores are the means of the unsummarized ones", { + args <- list( + mini_model_out(), + mini_oracle_out(), + K = c(50, 200), + target_cols = "location" + ) + raw <- do.call(alloscore_model_out, c(args, list(summarize = FALSE))) + summ <- do.call(alloscore_model_out, c(args, list(by = "model_id"))) + expected <- dplyr::summarise( + dplyr::group_by(raw, .data$model_id), + score = mean(.data$score), + .groups = "drop" + ) + expect_equal(summ$score, expected$score) +}) + +test_that("alloscore_model_out validates by", { + expect_error( + alloscore_model_out( + mini_model_out(), + mini_oracle_out(), + K = 50, + target_cols = "location", + by = "nonsense" + ), + regexp = "not available" + ) +}) + +test_that("alloscore_model_out passes weights through", { + # target b needs far more than target a, so weighting b up shifts the score + cheap <- alloscore_model_out( + mini_model_out(), + mini_oracle_out(), + K = 100, + target_cols = "location", + w = 1, + summarize = FALSE + ) + dear <- alloscore_model_out( + mini_model_out(), + mini_oracle_out(), + K = 100, + target_cols = "location", + w = c(a = 1, b = 5), + summarize = FALSE + ) + expect_false(isTRUE(all.equal(cheap$score_raw, dear$score_raw))) +}) + +test_that("alloscore_model_out honors against_oracle", { + res <- alloscore_model_out( + mini_model_out(), + mini_oracle_out(), + K = 50, + target_cols = "location", + against_oracle = FALSE, + summarize = FALSE + ) + expect_true("score_raw" %in% names(res)) + expect_false("score" %in% names(res)) +}) + +test_that("alloscore_model_out matches the core API on a single unit", { + # one allocation unit routed both ways must give the same score + mot <- dplyr::filter(mini_model_out(), .data$model_id == "m1") + hub <- alloscore_model_out( + mot, + mini_oracle_out(), + K = c(50, 200), + target_cols = "location", + summarize = FALSE + ) + adf <- as_alloscore_df(mot, mini_oracle_out(), target_cols = "location") + fc <- adf$forecasts[[1]] + core <- alloscore( + df = fc, + y = rlang::set_names(fc$y, fc$target_names), + K = c(50, 200), + target_names = "target_names" + ) + expect_equal(hub$score, core$score) + expect_equal(hub$score_raw, core$score_raw) + expect_equal(hub$ytot, core$ytot) +}) + +test_that("alloscore_model_out rejects unfiltered model output", { + mot <- dplyr::bind_rows( + mini_model_out(), + dplyr::mutate(mini_model_out(), output_type = "mean") + ) + expect_error( + alloscore_model_out( + mot, + mini_oracle_out(), + K = 50, + target_cols = "location" + ), + regexp = "must contain a single" + ) +}) diff --git a/tests/testthat/test-validate.R b/tests/testthat/test-validate.R new file mode 100644 index 0000000..5580746 --- /dev/null +++ b/tests/testthat/test-validate.R @@ -0,0 +1,147 @@ +test_that("get_task_id_cols excludes the model output representation columns", { + mot <- mini_model_out() + expect_setequal( + get_task_id_cols(mot), + c("reference_date", "target", "horizon", "location", "target_end_date") + ) +}) + +test_that("validate_output_type accepts a single quantile output type", { + expect_equal(validate_output_type(mini_model_out()), "quantile") +}) + +test_that("validate_output_type rejects several output types", { + mot <- mini_model_out() + mixed <- dplyr::bind_rows(mot, dplyr::mutate(mot, output_type = "mean")) + expect_error( + validate_output_type(mixed), + regexp = "must contain a single .*output_type" + ) +}) + +test_that("validate_output_type rejects unsupported output types", { + mot <- dplyr::mutate(mini_model_out(), output_type = "sample") + expect_error( + validate_output_type(mot), + regexp = "only the .*quantile.* output type" + ) +}) + +test_that("validate_model_oracle_out requires the model output columns", { + mot <- dplyr::select(mini_model_out(), -"value") + expect_error( + validate_model_oracle_out(mot, mini_oracle_out()), + regexp = "missing required column" + ) +}) + +test_that("validate_model_oracle_out requires oracle_value", { + oo <- dplyr::rename(mini_oracle_out(), observation = "oracle_value") + expect_error( + validate_model_oracle_out(mini_model_out(), oo), + regexp = "must have an .*oracle_value.* column" + ) +}) + +test_that("validate_model_oracle_out rejects unexpected oracle columns", { + oo <- dplyr::mutate(mini_oracle_out(), surprise = 1) + expect_error( + validate_model_oracle_out(mini_model_out(), oo), + regexp = "unexpected column" + ) +}) + +test_that("validate_model_oracle_out requires a shared task ID column", { + oo <- tibble::tibble(oracle_value = 1) + expect_error( + validate_model_oracle_out(mini_model_out(), oo), + regexp = "no task ID columns in common" + ) +}) + +test_that("validate_model_oracle_out returns a model_out_tbl", { + res <- validate_model_oracle_out(mini_model_out(), mini_oracle_out()) + expect_s3_class(res, "model_out_tbl") +}) + +test_that("resolve_allocation_unit derives the unit from target_cols", { + res <- resolve_allocation_unit(mini_model_out(), "location") + expect_equal(res$target_cols, "location") + expect_setequal( + res$allocation_unit, + c("model_id", "reference_date", "target", "horizon", "target_end_date") + ) +}) + +test_that("resolve_allocation_unit accepts several target columns", { + res <- resolve_allocation_unit(mini_model_out(), c("location", "target")) + expect_setequal( + res$allocation_unit, + c("model_id", "reference_date", "horizon", "target_end_date") + ) +}) + +test_that("resolve_allocation_unit requires target_cols", { + expect_error( + resolve_allocation_unit(mini_model_out()), + regexp = "must be supplied" + ) + expect_error( + resolve_allocation_unit(mini_model_out(), NULL), + regexp = "must be supplied" + ) +}) + +test_that("resolve_allocation_unit rejects columns that are not task IDs", { + expect_error( + resolve_allocation_unit(mini_model_out(), "not_a_column"), + regexp = "must name task ID columns" + ) + # model_id is not a task ID either + expect_error( + resolve_allocation_unit(mini_model_out(), "model_id"), + regexp = "must name task ID columns" + ) +}) + +test_that("resolve_weights accepts a scalar", { + fc <- tibble::tibble(target_names = c("a", "b")) + expect_equal(resolve_weights(1, fc, c("a", "b")), c(1, 1)) + expect_equal(resolve_weights(2.5, fc, c("a", "b")), c(2.5, 2.5)) +}) + +test_that("resolve_weights matches a named vector by target", { + fc <- tibble::tibble(target_names = c("a", "b")) + # order follows the targets, not the vector + expect_equal(resolve_weights(c(b = 3, a = 2), fc, c("a", "b")), c(2, 3)) + expect_error( + resolve_weights(c(a = 2), fc, c("a", "b")), + regexp = "no entry for target" + ) +}) + +test_that("resolve_weights accepts a positional vector of the right length", { + fc <- tibble::tibble(target_names = c("a", "b")) + expect_equal(resolve_weights(c(2, 3), fc, c("a", "b")), c(2, 3)) + expect_error( + resolve_weights(c(2, 3, 4), fc, c("a", "b")), + regexp = "has length 3 but there are 2 targets" + ) +}) + +test_that("resolve_weights takes a column name", { + fc <- tibble::tibble(target_names = c("a", "b"), cost = c(5, 7)) + expect_equal(resolve_weights("cost", fc, c("a", "b")), c(5, 7)) + expect_error( + resolve_weights("nope", fc, c("a", "b")), + regexp = "names column .*nope.*which is not present" + ) +}) + +test_that("resolve_weights rejects other types", { + fc <- tibble::tibble(target_names = c("a", "b")) + expect_error( + resolve_weights(TRUE, fc, c("a", "b")), + regexp = "must be numeric" + ) +}) From b4c4e315798d64199b959326cb238d41135a2059 Mon Sep 17 00:00:00 2001 From: Nicholas Reich Date: Wed, 2 Sep 2026 17:15:18 -0400 Subject: [PATCH 2/2] Pin numerical equivalence with the original alloscore The original package has no test suite, so there is no pre-existing ground truth to port. data-raw/make_legacy_reference.R runs the original code once over a fixed set of fixtures and freezes the results in tests/testthat/testdata/; test-legacy-equivalence.R checks that this package reproduces them. That keeps the original out of this package's dependencies while still pinning the numerics, so any future change in behaviour has to be deliberate. The fixture that matters most is legacy_hub_quantile_scores.csv: hubverse example data pushed through the original package's hand-rolled pipeline, which alloscore_model_out() must reproduce. It ships in the same pull request as that function so the interface does not land unproven. legacy_reference_meta.json records the original package's git SHA (0477ae9) and the R, distfromq and hubExamples versions everything was generated under. Tolerances are 1e-6 on allocations and 1e-8 on scores, loose enough to absorb uniroot noise and far tighter than any behavioural difference. Co-Authored-By: Claude Opus 5 (1M context) --- DESCRIPTION | 1 + NEWS.md | 3 + data-raw/make_legacy_reference.R | 237 ++++++++++++++++++ tests/testthat/test-legacy-equivalence.R | 229 +++++++++++++++++ .../legacy_hub_quantile_components.csv | 145 +++++++++++ .../testdata/legacy_hub_quantile_scores.csv | 73 ++++++ .../testdata/legacy_lnorm_components.csv | 51 ++++ .../testthat/testdata/legacy_lnorm_scores.csv | 6 + tests/testthat/testdata/legacy_lnorm_y.rds | Bin 0 -> 125 bytes .../testdata/legacy_reference_meta.json | 10 + .../testthat/testdata/legacy_slim_scores.csv | 9 + tests/testthat/testdata/legacy_slim_ys.rds | Bin 0 -> 420 bytes tests/testthat/testdata/legacy_zxh_beta.csv | 7 + tests/testthat/testdata/legacy_zxh_norm.csv | 18 ++ 14 files changed, 789 insertions(+) create mode 100644 data-raw/make_legacy_reference.R create mode 100644 tests/testthat/test-legacy-equivalence.R create mode 100644 tests/testthat/testdata/legacy_hub_quantile_components.csv create mode 100644 tests/testthat/testdata/legacy_hub_quantile_scores.csv create mode 100644 tests/testthat/testdata/legacy_lnorm_components.csv create mode 100644 tests/testthat/testdata/legacy_lnorm_scores.csv create mode 100644 tests/testthat/testdata/legacy_lnorm_y.rds create mode 100644 tests/testthat/testdata/legacy_reference_meta.json create mode 100644 tests/testthat/testdata/legacy_slim_scores.csv create mode 100644 tests/testthat/testdata/legacy_slim_ys.rds create mode 100644 tests/testthat/testdata/legacy_zxh_beta.csv create mode 100644 tests/testthat/testdata/legacy_zxh_norm.csv diff --git a/DESCRIPTION b/DESCRIPTION index 9ce6459..a3a1446 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -37,6 +37,7 @@ Imports: Suggests: covr, hubExamples, + jsonlite, testthat (>= 3.0.0) Remotes: hubverse-org/hubExamples diff --git a/NEWS.md b/NEWS.md index 4216601..c3ad32b 100644 --- a/NEWS.md +++ b/NEWS.md @@ -48,3 +48,6 @@ quantiles are turned into cdfs and quantile functions with `distfromq`, so the conversion that callers used to hand-roll is no longer needed. Only the `quantile` output type is supported. + +* Results agree with the original package to within root-finding tolerance; see + `tests/testthat/test-legacy-equivalence.R`. diff --git a/data-raw/make_legacy_reference.R b/data-raw/make_legacy_reference.R new file mode 100644 index 0000000..f6af108 --- /dev/null +++ b/data-raw/make_legacy_reference.R @@ -0,0 +1,237 @@ +## Generate frozen reference values from the ORIGINAL alloscore package. +## +## The original package (https://github.com/aaronger/alloscore) has no test +## suite, so there is no existing ground truth to port. This script runs the old +## code once over a fixed set of fixtures and writes the results to +## tests/testthat/testdata/, where test-legacy-equivalence.R compares them +## against alloscore2. That keeps the old package out of alloscore2's +## dependencies while still pinning the numerics. +## +## Run from the package root with the original alloscore checked out alongside: +## Rscript data-raw/make_legacy_reference.R +## +## Regenerate only when deliberately changing what is compared; a diff in these +## files is a behaviour change and should be reviewed as one. + +library(dplyr) +library(purrr) +library(tibble) +library(rlang) +library(magrittr) +library(tidyr) + +OLD_PKG <- Sys.getenv("ALLOSCORE_OLD_PATH", "../alloscore") +OUT_DIR <- "tests/testthat/testdata" +dir.create(OUT_DIR, showWarnings = FALSE, recursive = TRUE) + +stopifnot(dir.exists(OLD_PKG)) +suppressMessages(pkgload::load_all(OLD_PKG, quiet = TRUE, export_all = TRUE)) +message("Loaded original alloscore from ", normalizePath(OLD_PKG)) + +old_sha <- system2( + "git", + c("-C", OLD_PKG, "rev-parse", "HEAD"), + stdout = TRUE, + stderr = FALSE +) + +## ---------------------------------------------------------------- fixture 1 -- +## ZXH Table 2: Normal demand, K = 2500, unit cost as weight. +zxh_norm <- zxh_tab2 %>% + mutate(stdize_news_params(ax = c, a_minus = v, a_plus = h), .after = c) %>% + as_tibble() %>% + rename(mean = mu, sd = sigma) %>% + add_pdqr_funs(dist = "norm", types = c("p", "q")) +allo_norm <- allocate(zxh_norm, w = "c", K = 2500, target_names = "Product") +write.csv( + allo_norm$xdf[[1]] %>% select(Product, x), + file.path(OUT_DIR, "legacy_zxh_norm.csv"), + row.names = FALSE +) + +## ---------------------------------------------------------------- fixture 2 -- +## ZXH Table 3: Beta demand rescaled onto [x_min, x_max], K = 6500. +zxh_beta <- zxh_tab3 %>% + mutate(stdize_news_params(ax = c, a_minus = v, a_plus = h), .after = c) %>% + as_tibble() %>% + rename(shape1 = Balpha, shape2 = Bbeta) %>% + add_pdqr_funs( + types = c("p", "q"), + dist = "beta", + trans = function(x, x_min, x_max) (x - x_min) / (x_max - x_min), + trans_inv = function(q, x_min, x_max) x_min + (x_max - x_min) * q + ) +allo_beta <- allocate(zxh_beta, w = "c", K = 6500, target_names = "Product") +write.csv( + allo_beta$xdf[[1]] %>% select(Product, x), + file.path(OUT_DIR, "legacy_zxh_beta.csv"), + row.names = FALSE +) + +## ---------------------------------------------------------------- fixture 3 -- +## The basic_use vignette scenario: 10 lognormal forecasts against exponential +## outcomes, scored over a grid of budgets. +N <- 10 +y_mean <- 50 +lnorm_fc <- tibble( + target_names = LETTERS[1:N], + dist = "lnorm", + sdlog = 1, + meanlog = log(y_mean + 15 * (1:N)) - .5 * sdlog +) %>% + add_pdqr_funs(types = c("p", "q")) +y_lnorm <- withr::with_seed(8675309, rexp(N, rate = 1 / y_mean)) +saveRDS(y_lnorm, file.path(OUT_DIR, "legacy_lnorm_y.rds")) + +Ks_lnorm <- seq(200, 400, by = 50) +scored_lnorm <- alloscore(df = lnorm_fc, y = y_lnorm, K = Ks_lnorm) +write.csv( + scored_lnorm %>% select(K, score, score_raw, score_oracle, ytot), + file.path(OUT_DIR, "legacy_lnorm_scores.csv"), + row.names = FALSE +) +write.csv( + scored_lnorm %>% + select(K, xdf) %>% + unnest(xdf) %>% + select( + K, + target_names, + x, + y, + oracle, + components_raw, + components_oracle, + components + ), + file.path(OUT_DIR, "legacy_lnorm_components.csv"), + row.names = FALSE +) + +## ---------------------------------------------------------------- fixture 4 -- +## The critical fixture: hubverse quantile forecasts pushed through the OLD +## package's hand-rolled pipeline, exactly as utility-eval-papers/R/run-alloscore.R +## does it. alloscore2's as_alloscore_df() / alloscore_model_out() must +## reproduce these numbers. +mot <- hubExamples::forecast_outputs %>% + filter(output_type == "quantile") +oracle <- hubExamples::forecast_oracle_output %>% + filter(output_type == "quantile") %>% + select(-output_type, -output_type_id) + +unit_cols <- c( + "model_id", + "reference_date", + "target", + "horizon", + "target_end_date" +) +Ks_hub <- c(200, 800, 2000) + +hub_nested <- mot %>% + arrange( + across(all_of(c(unit_cols, "location"))), + as.numeric(output_type_id) + ) %>% + group_by(across(all_of(c(unit_cols, "location")))) %>% + summarise( + ps = list(as.numeric(output_type_id)), + qs = list(value), + .groups = "drop" + ) %>% + left_join(oracle, by = c("location", "target_end_date", "target")) %>% + rename(y = oracle_value) + +hub_rows <- hub_nested %>% + group_by(across(all_of(unit_cols))) %>% + group_split() + +hub_scores <- map(hub_rows, function(unit) { + fc <- unit %>% + mutate(target_names = as.character(location), dist = "distfromq") %>% + add_pdqr_funs(types = c("p", "q")) + scored <- alloscore( + df = fc, + y = set_names(fc$y, fc$target_names), + K = Ks_hub, + target_names = "target_names" + ) + bind_cols( + unit[rep(1L, nrow(scored)), unit_cols, drop = FALSE], + scored %>% select(K, score, score_raw, score_oracle, ytot) + ) +}) %>% + list_rbind() +write.csv( + hub_scores, + file.path(OUT_DIR, "legacy_hub_quantile_scores.csv"), + row.names = FALSE +) + +hub_components <- map(hub_rows, function(unit) { + fc <- unit %>% + mutate(target_names = as.character(location), dist = "distfromq") %>% + add_pdqr_funs(types = c("p", "q")) + scored <- alloscore( + df = fc, + y = set_names(fc$y, fc$target_names), + K = Ks_hub, + target_names = "target_names" + ) + detail <- scored %>% + select(K, xdf) %>% + unnest(xdf) %>% + select( + K, + target_names, + x, + y, + oracle, + components_raw, + components_oracle, + components + ) + bind_cols( + unit[rep(1L, nrow(detail)), unit_cols, drop = FALSE], + detail + ) +}) %>% + list_rbind() +write.csv( + hub_components, + file.path(OUT_DIR, "legacy_hub_quantile_components.csv"), + row.names = FALSE +) + +## ---------------------------------------------------------------- fixture 5 -- +## The Monte Carlo `slim` path over several outcome vectors. +Ks_slim <- c(150, 400) +allo_slim <- slim(allocate(lnorm_fc, K = Ks_slim)) +ys_slim <- withr::with_seed(20240101, { + map(1:4, function(i) set_names(rexp(N, rate = 1 / y_mean), LETTERS[1:N])) +}) +saveRDS(ys_slim, file.path(OUT_DIR, "legacy_slim_ys.rds")) +slim_scored <- alloscore(allo_slim, ys_slim) +write.csv( + slim_scored %>% select(samp, scores) %>% unnest(scores), + file.path(OUT_DIR, "legacy_slim_scores.csv"), + row.names = FALSE +) + +## ------------------------------------------------------------------ metadata -- +meta <- list( + generated_on = as.character(Sys.Date()), + old_alloscore_git_sha = old_sha, + r_version = R.version.string, + distfromq_version = as.character(packageVersion("distfromq")), + hubExamples_version = as.character(packageVersion("hubExamples")), + Ks_lnorm = Ks_lnorm, + Ks_hub = Ks_hub, + Ks_slim = Ks_slim +) +writeLines( + jsonlite::toJSON(meta, auto_unbox = TRUE, pretty = TRUE), + file.path(OUT_DIR, "legacy_reference_meta.json") +) + +message("Wrote reference fixtures to ", OUT_DIR) diff --git a/tests/testthat/test-legacy-equivalence.R b/tests/testthat/test-legacy-equivalence.R new file mode 100644 index 0000000..a14a08f --- /dev/null +++ b/tests/testthat/test-legacy-equivalence.R @@ -0,0 +1,229 @@ +# Equivalence with the original alloscore package. +# +# The original package (aaronger/alloscore) has no test suite, so there is no +# pre-existing ground truth to port. data-raw/make_legacy_reference.R runs the +# original code once over a fixed set of fixtures and freezes the results in +# testdata/; these tests check that alloscore2 reproduces them. That keeps the +# original package out of this one's dependencies while still pinning the +# numerics, so any future change in behaviour has to be deliberate. +# +# See testdata/legacy_reference_meta.json for the original package's git SHA and +# the versions everything was generated under. +# +# Tolerances are loose enough to absorb root-finding noise but far tighter than +# any behavioural difference would be: allocations to 1e-6, scores to 1e-8. +ALLOC_TOL <- 1e-6 +SCORE_TOL <- 1e-8 + +read_legacy <- function(file) { + read.csv(test_path("testdata", file), stringsAsFactors = FALSE) +} + +test_that("the reference fixtures are present and were generated together", { + meta <- jsonlite::fromJSON(test_path( + "testdata", + "legacy_reference_meta.json" + )) + expect_equal( + meta$old_alloscore_git_sha, + "0477ae9ed97da8fe1fa754e1c3452516aafb3fd9" + ) + expect_true(all(c("Ks_lnorm", "Ks_hub", "Ks_slim") %in% names(meta))) +}) + +test_that("ZXH Table 2 allocations match the original package", { + legacy <- read_legacy("legacy_zxh_norm.csv") + a <- allocate( + zxh_norm_forecasts(), + w = "c", + K = 2500, + target_names = "Product" + ) + ours <- a$xdf[[1]] + expect_equal(as.character(ours$Product), as.character(legacy$Product)) + expect_equal(ours$x, legacy$x, tolerance = ALLOC_TOL, ignore_attr = TRUE) +}) + +test_that("ZXH Table 3 allocations match the original package", { + legacy <- read_legacy("legacy_zxh_beta.csv") + a <- allocate( + zxh_beta_forecasts(), + w = "c", + K = 6500, + target_names = "Product" + ) + ours <- a$xdf[[1]] + expect_equal(as.character(ours$Product), as.character(legacy$Product)) + expect_equal(ours$x, legacy$x, tolerance = ALLOC_TOL, ignore_attr = TRUE) +}) + +test_that("lognormal scores match the original package", { + legacy <- read_legacy("legacy_lnorm_scores.csv") + y <- readRDS(test_path("testdata", "legacy_lnorm_y.rds")) + ours <- alloscore(lnorm_forecasts(), y = y, K = legacy$K) + expect_equal(ours$K, legacy$K) + expect_equal(ours$score, legacy$score, tolerance = SCORE_TOL) + expect_equal(ours$score_raw, legacy$score_raw, tolerance = SCORE_TOL) + expect_equal(ours$score_oracle, legacy$score_oracle, tolerance = SCORE_TOL) + expect_equal(ours$ytot, legacy$ytot, tolerance = SCORE_TOL) +}) + +test_that("lognormal per-target components match the original package", { + legacy <- read_legacy("legacy_lnorm_components.csv") + y <- readRDS(test_path("testdata", "legacy_lnorm_y.rds")) + ours <- alloscore(lnorm_forecasts(), y = y, K = unique(legacy$K)) + ours <- tidyr::unnest(dplyr::select(ours, "K", "xdf"), "xdf") + ours <- dplyr::arrange(ours, .data$K, .data$target_names) + legacy <- dplyr::arrange(legacy, .data$K, .data$target_names) + expect_equal(ours$target_names, legacy$target_names) + expect_equal(ours$x, legacy$x, tolerance = ALLOC_TOL, ignore_attr = TRUE) + expect_equal( + ours$oracle, + legacy$oracle, + tolerance = ALLOC_TOL, + ignore_attr = TRUE + ) + expect_equal( + ours$components_raw, + legacy$components_raw, + tolerance = SCORE_TOL, + ignore_attr = TRUE + ) + expect_equal( + ours$components_oracle, + legacy$components_oracle, + tolerance = SCORE_TOL, + ignore_attr = TRUE + ) + expect_equal( + ours$components, + legacy$components, + tolerance = SCORE_TOL, + ignore_attr = TRUE + ) +}) + +test_that("the Monte Carlo slim path matches the original package", { + legacy <- read_legacy("legacy_slim_scores.csv") + ys <- readRDS(test_path("testdata", "legacy_slim_ys.rds")) + a <- slim(allocate(lnorm_forecasts(), K = unique(legacy$K))) + ours <- alloscore(a, ys) + ours <- tidyr::unnest(dplyr::select(ours, "samp", "scores"), "scores") + ours <- dplyr::arrange(ours, .data$samp, .data$K) + legacy <- dplyr::arrange(legacy, .data$samp, .data$K) + expect_equal(as.character(ours$samp), as.character(legacy$samp)) + expect_equal(ours$K, legacy$K) + expect_equal(ours$score, legacy$score, tolerance = SCORE_TOL) + expect_equal(ours$score_raw, legacy$score_raw, tolerance = SCORE_TOL) + expect_equal(ours$score_oracle, legacy$score_oracle, tolerance = SCORE_TOL) +}) + +# The tests below are the point of the whole exercise: they check that the +# hubverse-native entry points reproduce what the original package produced when +# the same hub data was pushed through its hand-rolled pipeline (nest ps/qs, +# add_pdqr_funs(dist = "distfromq"), join truth, alloscore), as in +# utility-eval-papers/R/run-alloscore.R. + +skip_if_no_hub_examples <- function() { + skip_if_not_installed("hubExamples") +} + +hub_model_out <- function() { + dplyr::filter(hubExamples::forecast_outputs, .data$output_type == "quantile") +} + +test_that("alloscore_model_out matches the original hand-rolled hub pipeline", { + skip_if_no_hub_examples() + legacy <- read_legacy("legacy_hub_quantile_scores.csv") + meta <- jsonlite::fromJSON(test_path( + "testdata", + "legacy_reference_meta.json" + )) + + ours <- alloscore_model_out( + model_out_tbl = hub_model_out(), + oracle_output = hubExamples::forecast_oracle_output, + K = meta$Ks_hub, + target_cols = "location", + summarize = FALSE + ) + key <- c("model_id", "reference_date", "target", "horizon", "K") + ours <- dplyr::arrange( + dplyr::mutate(ours, reference_date = as.character(.data$reference_date)), + dplyr::pick(tidyselect::all_of(key)) + ) + legacy <- dplyr::arrange(legacy, dplyr::pick(tidyselect::all_of(key))) + + expect_equal(nrow(ours), nrow(legacy)) + expect_equal(ours$model_id, legacy$model_id) + expect_equal(ours$reference_date, legacy$reference_date) + expect_equal(ours$horizon, legacy$horizon) + expect_equal(ours$K, legacy$K) + expect_equal(ours$score, legacy$score, tolerance = SCORE_TOL) + expect_equal(ours$score_raw, legacy$score_raw, tolerance = SCORE_TOL) + expect_equal(ours$score_oracle, legacy$score_oracle, tolerance = SCORE_TOL) + expect_equal(ours$ytot, legacy$ytot, tolerance = SCORE_TOL) +}) + +test_that("per-target hub allocations match the original hand-rolled pipeline", { + skip_if_no_hub_examples() + legacy <- read_legacy("legacy_hub_quantile_components.csv") + meta <- jsonlite::fromJSON(test_path( + "testdata", + "legacy_reference_meta.json" + )) + + ours <- alloscore_model_out( + model_out_tbl = hub_model_out(), + oracle_output = hubExamples::forecast_oracle_output, + K = meta$Ks_hub, + target_cols = "location", + summarize = FALSE + ) + ours <- tidyr::unnest( + dplyr::select(ours, "model_id", "reference_date", "horizon", "K", "xdf"), + "xdf" + ) + key <- c("model_id", "reference_date", "horizon", "K", "target_names") + ours <- dplyr::arrange( + dplyr::mutate( + ours, + reference_date = as.character(.data$reference_date), + target_names = as.character(.data$target_names) + ), + dplyr::pick(tidyselect::all_of(key)) + ) + legacy <- dplyr::arrange( + dplyr::mutate(legacy, target_names = as.character(.data$target_names)), + dplyr::pick(tidyselect::all_of(key)) + ) + + expect_equal(nrow(ours), nrow(legacy)) + expect_equal(ours$target_names, legacy$target_names) + expect_equal(ours$x, legacy$x, tolerance = ALLOC_TOL, ignore_attr = TRUE) + expect_equal(ours$y, legacy$y, tolerance = SCORE_TOL, ignore_attr = TRUE) + expect_equal( + ours$oracle, + legacy$oracle, + tolerance = ALLOC_TOL, + ignore_attr = TRUE + ) + expect_equal( + ours$components_raw, + legacy$components_raw, + tolerance = SCORE_TOL, + ignore_attr = TRUE + ) + expect_equal( + ours$components_oracle, + legacy$components_oracle, + tolerance = SCORE_TOL, + ignore_attr = TRUE + ) + expect_equal( + ours$components, + legacy$components, + tolerance = SCORE_TOL, + ignore_attr = TRUE + ) +}) diff --git a/tests/testthat/testdata/legacy_hub_quantile_components.csv b/tests/testthat/testdata/legacy_hub_quantile_components.csv new file mode 100644 index 0000000..a1efaf7 --- /dev/null +++ b/tests/testthat/testdata/legacy_hub_quantile_components.csv @@ -0,0 +1,145 @@ +"model_id","reference_date","target","horizon","target_end_date","K","target_names","x","y","oracle","components_raw","components_oracle","components" +"Flusight-baseline",2022-11-19,"wk inc flu hosp",0,2022-11-19,200,"25",0.0304513920615024,79,17.9749673019421,78.9695486079385,61.0250326980579,17.9445159098806 +"Flusight-baseline",2022-11-19,"wk inc flu hosp",0,2022-11-19,200,"48",200,1230,182.025032698058,1030,1047.97496730194,-17.9749673019421 +"Flusight-baseline",2022-11-19,"wk inc flu hosp",0,2022-11-19,800,"25",3.6049281346405,79,71.8998692077684,75.3950718653595,7.10013079223157,68.2949410731279 +"Flusight-baseline",2022-11-19,"wk inc flu hosp",0,2022-11-19,800,"48",796.672506486363,1230,728.100130792232,433.327493513637,501.899869207768,-68.5723756941312 +"Flusight-baseline",2022-11-19,"wk inc flu hosp",0,2022-11-19,2000,"25",184.661355737283,79,693.222216017544,0,0,0 +"Flusight-baseline",2022-11-19,"wk inc flu hosp",0,2022-11-19,2000,"48",1815.33864426272,1230,1306.77778398991,0,0,0 +"Flusight-baseline",2022-11-19,"wk inc flu hosp",1,2022-11-26,200,"25",0,221,20.5581327652605,221,200.441867234739,20.5581327652606 +"Flusight-baseline",2022-11-19,"wk inc flu hosp",1,2022-11-26,200,"48",200,1929,179.441867234739,1729,1749.55813276526,-20.5581327652606 +"Flusight-baseline",2022-11-19,"wk inc flu hosp",1,2022-11-26,800,"25",0.0138225321441638,221,82.2325310610422,220.986177467856,138.767468938958,82.218708528898 +"Flusight-baseline",2022-11-19,"wk inc flu hosp",1,2022-11-26,800,"48",800,1929,717.767468938958,1129,1211.23253106104,-82.2325310610422 +"Flusight-baseline",2022-11-19,"wk inc flu hosp",1,2022-11-26,2000,"25",234.473646203051,221,205.581327652605,0,15.4186723473945,-15.4186723473945 +"Flusight-baseline",2022-11-19,"wk inc flu hosp",1,2022-11-26,2000,"48",1765.52635379695,1929,1794.41867234739,163.473646203051,134.581327652606,28.8923185504457 +"Flusight-baseline",2022-11-19,"wk inc flu hosp",2,2022-12-03,200,"25",0,446,36.4677012279843,446,409.532298772016,36.4677012279843 +"Flusight-baseline",2022-11-19,"wk inc flu hosp",2,2022-12-03,200,"48",200,2139,163.532298772016,1939,1975.46770122798,-36.4677012279844 +"Flusight-baseline",2022-11-19,"wk inc flu hosp",2,2022-12-03,800,"25",0,446,145.870804911937,446,300.129195088063,145.870804911937 +"Flusight-baseline",2022-11-19,"wk inc flu hosp",2,2022-12-03,800,"48",800,2139,654.129195088063,1339,1484.87080491194,-145.870804911937 +"Flusight-baseline",2022-11-19,"wk inc flu hosp",2,2022-12-03,2000,"25",233.333325743739,446,364.677012279843,212.666674256261,81.3229877201566,131.343686536105 +"Flusight-baseline",2022-11-19,"wk inc flu hosp",2,2022-12-03,2000,"48",1766.66667425626,2139,1635.32298772016,372.333325743738,503.677012279843,-131.343686536105 +"Flusight-baseline",2022-11-19,"wk inc flu hosp",3,2022-12-10,200,"25",0,578,49.0038137366568,578,528.996186263343,49.0038137366569 +"Flusight-baseline",2022-11-19,"wk inc flu hosp",3,2022-12-10,200,"48",200,1781,150.996186263343,1581,1630.00381373666,-49.0038137366569 +"Flusight-baseline",2022-11-19,"wk inc flu hosp",3,2022-12-10,800,"25",0.154699231044563,578,196.015254946627,577.845300768955,381.984745053373,195.860555715583 +"Flusight-baseline",2022-11-19,"wk inc flu hosp",3,2022-12-10,800,"48",799.815652953934,1781,603.984745053373,981.184347046066,1177.01525494663,-195.830907900562 +"Flusight-baseline",2022-11-19,"wk inc flu hosp",3,2022-12-10,2000,"25",231.588392034307,578,490.038137366568,346.411607965693,87.9618626334316,258.449745332262 +"Flusight-baseline",2022-11-19,"wk inc flu hosp",3,2022-12-10,2000,"48",1768.41160796569,1781,1509.96186263343,12.5883920343076,271.038137366569,-258.449745332261 +"Flusight-baseline",2022-12-17,"wk inc flu hosp",0,2022-12-17,200,"25",63.6863317527284,694,64.3784768968923,630.313668247272,629.621523103108,0.692145144163987 +"Flusight-baseline",2022-12-17,"wk inc flu hosp",0,2022-12-17,200,"48",136.313668247272,1462,135.621523103108,1325.68633175273,1326.37847689689,-0.692145144163987 +"Flusight-baseline",2022-12-17,"wk inc flu hosp",0,2022-12-17,800,"25",254.745327010913,694,257.513907587569,439.254672989087,436.486092412431,2.76858057665589 +"Flusight-baseline",2022-12-17,"wk inc flu hosp",0,2022-12-17,800,"48",545.254672989087,1462,542.486092412431,916.745327010913,919.513907587569,-2.76858057665595 +"Flusight-baseline",2022-12-17,"wk inc flu hosp",0,2022-12-17,2000,"25",503.437994562332,694,643.784768968923,190.562005437668,50.2152310310765,140.346774406592 +"Flusight-baseline",2022-12-17,"wk inc flu hosp",0,2022-12-17,2000,"48",1496.83239744629,1462,1356.21523103108,0,105.784768968923,-105.784768968923 +"Flusight-baseline",2022-12-17,"wk inc flu hosp",1,2022-12-24,200,"25",48.2114777177733,769,98.024218940161,720.788522282227,670.975781059839,49.8127412223877 +"Flusight-baseline",2022-12-17,"wk inc flu hosp",1,2022-12-24,200,"48",151.788522282227,1225,101.975781059839,1073.21147771777,1123.02421894016,-49.8127412223878 +"Flusight-baseline",2022-12-17,"wk inc flu hosp",1,2022-12-24,800,"25",192.845910871093,769,392.096875760644,576.154089128907,376.903124239356,199.250964889551 +"Flusight-baseline",2022-12-17,"wk inc flu hosp",1,2022-12-24,800,"48",607.154089128907,1225,407.903124239356,617.845910871093,817.096875760644,-199.250964889551 +"Flusight-baseline",2022-12-17,"wk inc flu hosp",1,2022-12-24,2000,"25",483.376369334152,769,768.999085428917,285.623630665848,0.000914571082603288,285.622716094766 +"Flusight-baseline",2022-12-17,"wk inc flu hosp",1,2022-12-24,2000,"48",1516.48561799443,1225,1224.9990950182,0,0.000904981801113536,-0.000904981801113536 +"Flusight-baseline",2022-12-17,"wk inc flu hosp",2,2022-12-31,200,"25",15.4465768796131,733,67.6511303638082,717.553423120387,665.348869636192,52.2045534841951 +"Flusight-baseline",2022-12-17,"wk inc flu hosp",2,2022-12-31,200,"48",184.553423120387,1434,132.348869636192,1249.44657687961,1301.65113036381,-52.2045534841952 +"Flusight-baseline",2022-12-17,"wk inc flu hosp",2,2022-12-31,800,"25",61.7863075184524,733,270.604521455233,671.213692481548,462.395478544767,208.81821393678 +"Flusight-baseline",2022-12-17,"wk inc flu hosp",2,2022-12-31,800,"48",738.213692481548,1434,529.395478544767,695.786307518452,904.604521455233,-208.81821393678 +"Flusight-baseline",2022-12-17,"wk inc flu hosp",2,2022-12-31,2000,"25",494.430123207851,733,676.511303638082,238.569876792149,56.4886963619177,182.081180430231 +"Flusight-baseline",2022-12-17,"wk inc flu hosp",2,2022-12-31,2000,"48",1505.57273626555,1434,1323.48869636192,0,110.511303638082,-110.511303638082 +"Flusight-baseline",2022-12-17,"wk inc flu hosp",3,2023-01-07,200,"25",20.5547910702393,466,73.6176879037166,445.445208929761,392.382312096283,53.0628968334773 +"Flusight-baseline",2022-12-17,"wk inc flu hosp",3,2023-01-07,200,"48",179.445208929761,1170,126.382312096283,990.554791070239,1043.61768790372,-53.0628968334772 +"Flusight-baseline",2022-12-17,"wk inc flu hosp",3,2023-01-07,800,"25",82.1965112186504,466,294.470751614866,383.80348878135,171.529248385134,212.274240396216 +"Flusight-baseline",2022-12-17,"wk inc flu hosp",3,2023-01-07,800,"48",717.583072410008,1170,505.529248385134,452.416927589992,664.470751614866,-212.053824024874 +"Flusight-baseline",2022-12-17,"wk inc flu hosp",3,2023-01-07,2000,"25",492.49608701616,466,647.999992594123,0,0,0 +"Flusight-baseline",2022-12-17,"wk inc flu hosp",3,2023-01-07,2000,"48",1507.51473186549,1170,1352.00000739098,0,0,0 +"MOBS-GLEAM_FLUH",2022-11-19,"wk inc flu hosp",0,2022-11-19,200,"25",4.94390220000684,79,17.9749673019421,74.0560977999932,61.0250326980579,13.0310651019353 +"MOBS-GLEAM_FLUH",2022-11-19,"wk inc flu hosp",0,2022-11-19,200,"48",195.056097799993,1230,182.025032698058,1034.94390220001,1047.97496730194,-13.0310651019352 +"MOBS-GLEAM_FLUH",2022-11-19,"wk inc flu hosp",0,2022-11-19,800,"25",19.9039569486559,79,71.8998692077684,59.0960430513441,7.10013079223157,51.9959122591125 +"MOBS-GLEAM_FLUH",2022-11-19,"wk inc flu hosp",0,2022-11-19,800,"48",785.288222970613,1230,728.100130792232,444.711777029387,501.899869207768,-57.1880921783812 +"MOBS-GLEAM_FLUH",2022-11-19,"wk inc flu hosp",0,2022-11-19,2000,"25",151.143562035376,79,693.222216017544,0,0,0 +"MOBS-GLEAM_FLUH",2022-11-19,"wk inc flu hosp",0,2022-11-19,2000,"48",1848.85643796463,1230,1306.77778398991,0,0,0 +"MOBS-GLEAM_FLUH",2022-11-19,"wk inc flu hosp",1,2022-11-26,200,"25",17.2714619905452,221,20.5581327652605,203.728538009455,200.441867234739,3.28667077471539 +"MOBS-GLEAM_FLUH",2022-11-19,"wk inc flu hosp",1,2022-11-26,200,"48",182.728538009455,1929,179.441867234739,1746.27146199055,1749.55813276526,-3.2866707747155 +"MOBS-GLEAM_FLUH",2022-11-19,"wk inc flu hosp",1,2022-11-26,800,"25",26.941140234666,221,82.2325310610422,194.058859765334,138.767468938958,55.2913908263762 +"MOBS-GLEAM_FLUH",2022-11-19,"wk inc flu hosp",1,2022-11-26,800,"48",772.762990898821,1929,717.767468938958,1156.23700910118,1211.23253106104,-54.9955219598635 +"MOBS-GLEAM_FLUH",2022-11-19,"wk inc flu hosp",1,2022-11-26,2000,"25",172.23527340467,221,205.581327652605,48.7647265953296,15.4186723473945,33.3460542479351 +"MOBS-GLEAM_FLUH",2022-11-19,"wk inc flu hosp",1,2022-11-26,2000,"48",1827.76472659533,1929,1794.41867234739,101.235273404673,134.581327652606,-33.3460542479331 +"MOBS-GLEAM_FLUH",2022-11-19,"wk inc flu hosp",2,2022-12-03,200,"25",8.48088382127624,446,36.4677012279843,437.519116178724,409.532298772016,27.9868174067081 +"MOBS-GLEAM_FLUH",2022-11-19,"wk inc flu hosp",2,2022-12-03,200,"48",191.519116178724,2139,163.532298772016,1947.48088382128,1975.46770122798,-27.9868174067083 +"MOBS-GLEAM_FLUH",2022-11-19,"wk inc flu hosp",2,2022-12-03,800,"25",37.7563169887656,446,145.870804911937,408.243683011234,300.129195088063,108.114487923172 +"MOBS-GLEAM_FLUH",2022-11-19,"wk inc flu hosp",2,2022-12-03,800,"48",762.236477111037,2139,654.129195088063,1376.76352288896,1484.87080491194,-108.107282022975 +"MOBS-GLEAM_FLUH",2022-11-19,"wk inc flu hosp",2,2022-12-03,2000,"25",203.620579347936,446,364.677012279843,242.379420652064,81.3229877201566,161.056432931908 +"MOBS-GLEAM_FLUH",2022-11-19,"wk inc flu hosp",2,2022-12-03,2000,"48",1796.37942065207,2139,1635.32298772016,342.620579347935,503.677012279843,-161.056432931909 +"MOBS-GLEAM_FLUH",2022-11-19,"wk inc flu hosp",3,2022-12-10,200,"25",6.62918735063729,578,49.0038137366568,571.370812649363,528.996186263343,42.3746263860196 +"MOBS-GLEAM_FLUH",2022-11-19,"wk inc flu hosp",3,2022-12-10,200,"48",193.370812649363,1781,150.996186263343,1587.62918735064,1630.00381373666,-42.3746263860196 +"MOBS-GLEAM_FLUH",2022-11-19,"wk inc flu hosp",3,2022-12-10,800,"25",50.2535379677492,578,196.015254946627,527.746462032251,381.984745053373,145.761716978878 +"MOBS-GLEAM_FLUH",2022-11-19,"wk inc flu hosp",3,2022-12-10,800,"48",749.737800648365,1781,603.984745053373,1031.26219935163,1177.01525494663,-145.753055594993 +"MOBS-GLEAM_FLUH",2022-11-19,"wk inc flu hosp",3,2022-12-10,2000,"25",261.039379559644,578,490.038137366568,316.960620440356,87.9618626334316,228.998757806924 +"MOBS-GLEAM_FLUH",2022-11-19,"wk inc flu hosp",3,2022-12-10,2000,"48",1738.96062044036,1781,1509.96186263343,42.0393795596428,271.038137366569,-228.998757806926 +"MOBS-GLEAM_FLUH",2022-12-17,"wk inc flu hosp",0,2022-12-17,200,"25",57.7993811525329,694,64.3784768968923,636.200618847467,629.621523103108,6.57909574435951 +"MOBS-GLEAM_FLUH",2022-12-17,"wk inc flu hosp",0,2022-12-17,200,"48",142.200618847467,1462,135.621523103108,1319.79938115253,1326.37847689689,-6.57909574435939 +"MOBS-GLEAM_FLUH",2022-12-17,"wk inc flu hosp",0,2022-12-17,800,"25",231.197524610132,694,257.513907587569,462.802475389868,436.486092412431,26.3163829774377 +"MOBS-GLEAM_FLUH",2022-12-17,"wk inc flu hosp",0,2022-12-17,800,"48",568.802475389868,1462,542.486092412431,893.197524610132,919.513907587569,-26.3163829774378 +"MOBS-GLEAM_FLUH",2022-12-17,"wk inc flu hosp",0,2022-12-17,2000,"25",486.090414587336,694,643.784768968923,207.909585412664,50.2152310310765,157.694354381588 +"MOBS-GLEAM_FLUH",2022-12-17,"wk inc flu hosp",0,2022-12-17,2000,"48",1514.01004095682,1462,1356.21523103108,0,105.784768968923,-105.784768968923 +"MOBS-GLEAM_FLUH",2022-12-17,"wk inc flu hosp",1,2022-12-24,200,"25",100,769,98.024218940161,669,670.975781059839,-1.97578105983905 +"MOBS-GLEAM_FLUH",2022-12-17,"wk inc flu hosp",1,2022-12-24,200,"48",100,1225,101.975781059839,1125,1123.02421894016,1.97578105983894 +"MOBS-GLEAM_FLUH",2022-12-17,"wk inc flu hosp",1,2022-12-24,800,"25",240.055849374625,769,392.096875760644,528.944150625375,376.903124239356,152.041026386019 +"MOBS-GLEAM_FLUH",2022-12-17,"wk inc flu hosp",1,2022-12-24,800,"48",561.027133737376,1225,407.903124239356,663.972866262624,817.096875760644,-153.12400949802 +"MOBS-GLEAM_FLUH",2022-12-17,"wk inc flu hosp",1,2022-12-24,2000,"25",585.402391993638,769,768.999085428917,183.597608006362,0.000914571082603288,183.59669343528 +"MOBS-GLEAM_FLUH",2022-12-17,"wk inc flu hosp",1,2022-12-24,2000,"48",1414.64511329539,1225,1224.9990950182,0,0.000904981801113536,-0.000904981801113536 +"MOBS-GLEAM_FLUH",2022-12-17,"wk inc flu hosp",2,2022-12-31,200,"25",67.1463160957422,733,67.6511303638082,665.853683904258,665.348869636192,0.50481426806607 +"MOBS-GLEAM_FLUH",2022-12-17,"wk inc flu hosp",2,2022-12-31,200,"48",132.853683904258,1434,132.348869636192,1301.14631609574,1301.65113036381,-0.504814268066184 +"MOBS-GLEAM_FLUH",2022-12-17,"wk inc flu hosp",2,2022-12-31,800,"25",272.899996887217,733,270.604521455233,460.100003112783,462.395478544767,-2.29547543198419 +"MOBS-GLEAM_FLUH",2022-12-17,"wk inc flu hosp",2,2022-12-31,800,"48",527.354232691284,1434,529.395478544767,906.645767308716,904.604521455233,2.04124585348359 +"MOBS-GLEAM_FLUH",2022-12-17,"wk inc flu hosp",2,2022-12-31,2000,"25",682.395799428468,733,676.511303638082,50.6042005715318,56.4886963619177,-5.88449579038593 +"MOBS-GLEAM_FLUH",2022-12-17,"wk inc flu hosp",2,2022-12-31,2000,"48",1317.62555167536,1434,1323.48869636192,116.374448324642,110.511303638082,5.86314468655974 +"MOBS-GLEAM_FLUH",2022-12-17,"wk inc flu hosp",3,2023-01-07,200,"25",39.1796097581527,466,73.6176879037166,426.820390241847,392.382312096283,34.4380781455639 +"MOBS-GLEAM_FLUH",2022-12-17,"wk inc flu hosp",3,2023-01-07,200,"48",160.820390241847,1170,126.382312096283,1009.17960975815,1043.61768790372,-34.4380781455638 +"MOBS-GLEAM_FLUH",2022-12-17,"wk inc flu hosp",3,2023-01-07,800,"25",298.900126260804,466,294.470751614866,167.099873739196,171.529248385134,-4.4293746459378 +"MOBS-GLEAM_FLUH",2022-12-17,"wk inc flu hosp",3,2023-01-07,800,"48",501.07245356601,1170,505.529248385134,668.92754643399,664.470751614866,4.45679481912327 +"MOBS-GLEAM_FLUH",2022-12-17,"wk inc flu hosp",3,2023-01-07,2000,"25",749.020261850236,466,647.999992594123,0,0,0 +"MOBS-GLEAM_FLUH",2022-12-17,"wk inc flu hosp",3,2023-01-07,2000,"48",1250.98024982892,1170,1352.00000739098,0,0,0 +"PSI-DICE",2022-11-19,"wk inc flu hosp",0,2022-11-19,200,"25",0,79,17.9749673019421,79,61.0250326980579,17.9749673019421 +"PSI-DICE",2022-11-19,"wk inc flu hosp",0,2022-11-19,200,"48",200,1230,182.025032698058,1030,1047.97496730194,-17.9749673019421 +"PSI-DICE",2022-11-19,"wk inc flu hosp",0,2022-11-19,800,"25",3.68975282452671,79,71.8998692077684,75.3102471754733,7.10013079223157,68.2101163832417 +"PSI-DICE",2022-11-19,"wk inc flu hosp",0,2022-11-19,800,"48",795.143516903397,1230,728.100130792232,434.856483096603,501.899869207768,-67.0433861111659 +"PSI-DICE",2022-11-19,"wk inc flu hosp",0,2022-11-19,2000,"25",213.599497143032,79,693.222216017544,0,0,0 +"PSI-DICE",2022-11-19,"wk inc flu hosp",0,2022-11-19,2000,"48",1786.40050285697,1230,1306.77778398991,0,0,0 +"PSI-DICE",2022-11-19,"wk inc flu hosp",1,2022-11-26,200,"25",0,221,20.5581327652605,221,200.441867234739,20.5581327652606 +"PSI-DICE",2022-11-19,"wk inc flu hosp",1,2022-11-26,200,"48",200,1929,179.441867234739,1729,1749.55813276526,-20.5581327652606 +"PSI-DICE",2022-11-19,"wk inc flu hosp",1,2022-11-26,800,"25",0,221,82.2325310610422,221,138.767468938958,82.2325310610422 +"PSI-DICE",2022-11-19,"wk inc flu hosp",1,2022-11-26,800,"48",800,1929,717.767468938958,1129,1211.23253106104,-82.2325310610422 +"PSI-DICE",2022-11-19,"wk inc flu hosp",1,2022-11-26,2000,"25",249.132467560618,221,205.581327652605,0,15.4186723473945,-15.4186723473945 +"PSI-DICE",2022-11-19,"wk inc flu hosp",1,2022-11-26,2000,"48",1750.86753243938,1929,1794.41867234739,178.132467560619,134.581327652606,43.5511399080131 +"PSI-DICE",2022-11-19,"wk inc flu hosp",2,2022-12-03,200,"25",0,446,36.4677012279843,446,409.532298772016,36.4677012279843 +"PSI-DICE",2022-11-19,"wk inc flu hosp",2,2022-12-03,200,"48",200,2139,163.532298772016,1939,1975.46770122798,-36.4677012279844 +"PSI-DICE",2022-11-19,"wk inc flu hosp",2,2022-12-03,800,"25",1.23063244352411,446,145.870804911937,444.769367556476,300.129195088063,144.640172468413 +"PSI-DICE",2022-11-19,"wk inc flu hosp",2,2022-12-03,800,"48",800,2139,654.129195088063,1339,1484.87080491194,-145.870804911937 +"PSI-DICE",2022-11-19,"wk inc flu hosp",2,2022-12-03,2000,"25",342.345611028742,446,364.677012279843,103.654388971258,81.3229877201566,22.3314012511019 +"PSI-DICE",2022-11-19,"wk inc flu hosp",2,2022-12-03,2000,"48",1657.65438897126,2139,1635.32298772016,481.345611028742,503.677012279843,-22.3314012511009 +"PSI-DICE",2022-11-19,"wk inc flu hosp",3,2022-12-10,200,"25",0,578,49.0038137366568,578,528.996186263343,49.0038137366569 +"PSI-DICE",2022-11-19,"wk inc flu hosp",3,2022-12-10,200,"48",200,1781,150.996186263343,1581,1630.00381373666,-49.0038137366569 +"PSI-DICE",2022-11-19,"wk inc flu hosp",3,2022-12-10,800,"25",0,578,196.015254946627,578,381.984745053373,196.015254946627 +"PSI-DICE",2022-11-19,"wk inc flu hosp",3,2022-12-10,800,"48",800,1781,603.984745053373,981,1177.01525494663,-196.015254946627 +"PSI-DICE",2022-11-19,"wk inc flu hosp",3,2022-12-10,2000,"25",457.368601917772,578,490.038137366568,120.631398082228,87.9618626334316,32.6695354487963 +"PSI-DICE",2022-11-19,"wk inc flu hosp",3,2022-12-10,2000,"48",1542.63139808223,1781,1509.96186263343,238.368601917769,271.038137366569,-32.6695354487993 +"PSI-DICE",2022-12-17,"wk inc flu hosp",0,2022-12-17,200,"25",0,694,64.3784768968923,694,629.621523103108,64.3784768968924 +"PSI-DICE",2022-12-17,"wk inc flu hosp",0,2022-12-17,200,"48",200,1462,135.621523103108,1262,1326.37847689689,-64.3784768968924 +"PSI-DICE",2022-12-17,"wk inc flu hosp",0,2022-12-17,800,"25",1.60640784583241,694,257.513907587569,692.393592154168,436.486092412431,255.907499741737 +"PSI-DICE",2022-12-17,"wk inc flu hosp",0,2022-12-17,800,"48",800,1462,542.486092412431,662,919.513907587569,-257.513907587569 +"PSI-DICE",2022-12-17,"wk inc flu hosp",0,2022-12-17,2000,"25",390.64161046894,694,643.784768968923,303.35838953106,50.2152310310765,253.143158499984 +"PSI-DICE",2022-12-17,"wk inc flu hosp",0,2022-12-17,2000,"48",1609.28037907203,1462,1356.21523103108,0,105.784768968923,-105.784768968923 +"PSI-DICE",2022-12-17,"wk inc flu hosp",1,2022-12-24,200,"25",0,769,98.024218940161,769,670.975781059839,98.024218940161 +"PSI-DICE",2022-12-17,"wk inc flu hosp",1,2022-12-24,200,"48",200,1225,101.975781059839,1025,1123.02421894016,-98.0242189401611 +"PSI-DICE",2022-12-17,"wk inc flu hosp",1,2022-12-24,800,"25",0.597281146153994,769,392.096875760644,768.402718853846,376.903124239356,391.49959461449 +"PSI-DICE",2022-12-17,"wk inc flu hosp",1,2022-12-24,800,"48",800,1225,407.903124239356,425,817.096875760644,-392.096875760644 +"PSI-DICE",2022-12-17,"wk inc flu hosp",1,2022-12-24,2000,"25",519.323454022046,769,768.999085428917,249.676545977954,0.000914571082603288,249.675631406871 +"PSI-DICE",2022-12-17,"wk inc flu hosp",1,2022-12-24,2000,"48",1480.63650403677,1225,1224.9990950182,0,0.000904981801113536,-0.000904981801113536 +"PSI-DICE",2022-12-17,"wk inc flu hosp",2,2022-12-31,200,"25",0,733,67.6511303638082,733,665.348869636192,67.6511303638082 +"PSI-DICE",2022-12-17,"wk inc flu hosp",2,2022-12-31,200,"48",200,1434,132.348869636192,1234,1301.65113036381,-67.6511303638083 +"PSI-DICE",2022-12-17,"wk inc flu hosp",2,2022-12-31,800,"25",0.153369559700542,733,270.604521455233,732.8466304403,462.395478544767,270.451151895532 +"PSI-DICE",2022-12-17,"wk inc flu hosp",2,2022-12-31,800,"48",800,1434,529.395478544767,634,904.604521455233,-270.604521455233 +"PSI-DICE",2022-12-17,"wk inc flu hosp",2,2022-12-31,2000,"25",595.141449693784,733,676.511303638082,137.858550306216,56.4886963619177,81.3698539442987 +"PSI-DICE",2022-12-17,"wk inc flu hosp",2,2022-12-31,2000,"48",1404.86375787182,1434,1323.48869636192,29.1362421281781,110.511303638082,-81.3750615099041 +"PSI-DICE",2022-12-17,"wk inc flu hosp",3,2023-01-07,200,"25",0,466,73.6176879037166,466,392.382312096283,73.6176879037166 +"PSI-DICE",2022-12-17,"wk inc flu hosp",3,2023-01-07,200,"48",200,1170,126.382312096283,970,1043.61768790372,-73.6176879037166 +"PSI-DICE",2022-12-17,"wk inc flu hosp",3,2023-01-07,800,"25",0,466,294.470751614866,466,171.529248385134,294.470751614866 +"PSI-DICE",2022-12-17,"wk inc flu hosp",3,2023-01-07,800,"48",800,1170,505.529248385134,370,664.470751614866,-294.470751614866 +"PSI-DICE",2022-12-17,"wk inc flu hosp",3,2023-01-07,2000,"25",783.567762787459,466,647.999992594123,0,0,0 +"PSI-DICE",2022-12-17,"wk inc flu hosp",3,2023-01-07,2000,"48",1216.39856318535,1170,1352.00000739098,0,0,0 diff --git a/tests/testthat/testdata/legacy_hub_quantile_scores.csv b/tests/testthat/testdata/legacy_hub_quantile_scores.csv new file mode 100644 index 0000000..61a2f74 --- /dev/null +++ b/tests/testthat/testdata/legacy_hub_quantile_scores.csv @@ -0,0 +1,73 @@ +"model_id","reference_date","target","horizon","target_end_date","K","score","score_raw","score_oracle","ytot" +"Flusight-baseline",2022-11-19,"wk inc flu hosp",0,2022-11-19,200,-0.0304513920614227,1108.96954860794,1109,1309 +"Flusight-baseline",2022-11-19,"wk inc flu hosp",0,2022-11-19,800,-0.277434621003238,508.722565378997,509,1309 +"Flusight-baseline",2022-11-19,"wk inc flu hosp",0,2022-11-19,2000,0,0,0,1309 +"Flusight-baseline",2022-11-19,"wk inc flu hosp",1,2022-11-26,200,0,1950,1950,2150 +"Flusight-baseline",2022-11-19,"wk inc flu hosp",1,2022-11-26,800,-0.0138225321441041,1349.98617746786,1350,2150 +"Flusight-baseline",2022-11-19,"wk inc flu hosp",1,2022-11-26,2000,13.4736462030512,163.473646203051,150,2150 +"Flusight-baseline",2022-11-19,"wk inc flu hosp",2,2022-12-03,200,0,2385,2385,2585 +"Flusight-baseline",2022-11-19,"wk inc flu hosp",2,2022-12-03,800,0,1785,1785,2585 +"Flusight-baseline",2022-11-19,"wk inc flu hosp",2,2022-12-03,2000,-6.82121026329696e-13,584.999999999999,585,2585 +"Flusight-baseline",2022-11-19,"wk inc flu hosp",3,2022-12-10,200,0,2159,2159,2359 +"Flusight-baseline",2022-11-19,"wk inc flu hosp",3,2022-12-10,800,0.0296478150210078,1559.02964781502,1559,2359 +"Flusight-baseline",2022-11-19,"wk inc flu hosp",3,2022-12-10,2000,4.54747350886464e-13,359.000000000001,359,2359 +"Flusight-baseline",2022-12-17,"wk inc flu hosp",0,2022-12-17,200,0,1956,1956,2156 +"Flusight-baseline",2022-12-17,"wk inc flu hosp",0,2022-12-17,800,0,1356,1356,2156 +"Flusight-baseline",2022-12-17,"wk inc flu hosp",0,2022-12-17,2000,34.5620054376685,190.562005437668,156,2156 +"Flusight-baseline",2022-12-17,"wk inc flu hosp",1,2022-12-24,200,0,1794,1794,1994 +"Flusight-baseline",2022-12-17,"wk inc flu hosp",1,2022-12-24,800,0,1194,1194,1994 +"Flusight-baseline",2022-12-17,"wk inc flu hosp",1,2022-12-24,2000,285.621811112964,285.623630665848,0.00181955288371682,1994 +"Flusight-baseline",2022-12-17,"wk inc flu hosp",2,2022-12-31,200,0,1967,1967,2167 +"Flusight-baseline",2022-12-17,"wk inc flu hosp",2,2022-12-31,800,0,1367,1367,2167 +"Flusight-baseline",2022-12-17,"wk inc flu hosp",2,2022-12-31,2000,71.5698767921488,238.569876792149,167,2167 +"Flusight-baseline",2022-12-17,"wk inc flu hosp",3,2023-01-07,200,0,1436,1436,1636 +"Flusight-baseline",2022-12-17,"wk inc flu hosp",3,2023-01-07,800,0.220416371341798,836.220416371342,836,1636 +"Flusight-baseline",2022-12-17,"wk inc flu hosp",3,2023-01-07,2000,0,0,0,1636 +"MOBS-GLEAM_FLUH",2022-11-19,"wk inc flu hosp",0,2022-11-19,200,0,1109,1109,1309 +"MOBS-GLEAM_FLUH",2022-11-19,"wk inc flu hosp",0,2022-11-19,800,-5.19217991926865,503.807820080731,509,1309 +"MOBS-GLEAM_FLUH",2022-11-19,"wk inc flu hosp",0,2022-11-19,2000,0,0,0,1309 +"MOBS-GLEAM_FLUH",2022-11-19,"wk inc flu hosp",1,2022-11-26,200,0,1950,1950,2150 +"MOBS-GLEAM_FLUH",2022-11-19,"wk inc flu hosp",1,2022-11-26,800,0.295868866512819,1350.29586886651,1350,2150 +"MOBS-GLEAM_FLUH",2022-11-19,"wk inc flu hosp",1,2022-11-26,2000,2.01794136955868e-12,150.000000000002,150,2150 +"MOBS-GLEAM_FLUH",2022-11-19,"wk inc flu hosp",2,2022-12-03,200,0,2385,2385,2585 +"MOBS-GLEAM_FLUH",2022-11-19,"wk inc flu hosp",2,2022-12-03,800,0.00720590019705014,1785.0072059002,1785,2585 +"MOBS-GLEAM_FLUH",2022-11-19,"wk inc flu hosp",2,2022-12-03,2000,-9.09494701772928e-13,584.999999999999,585,2585 +"MOBS-GLEAM_FLUH",2022-11-19,"wk inc flu hosp",3,2022-12-10,200,0,2159,2159,2359 +"MOBS-GLEAM_FLUH",2022-11-19,"wk inc flu hosp",3,2022-12-10,800,0.0086613838852827,1559.00866138389,1559,2359 +"MOBS-GLEAM_FLUH",2022-11-19,"wk inc flu hosp",3,2022-12-10,2000,-1.59161572810262e-12,358.999999999999,359,2359 +"MOBS-GLEAM_FLUH",2022-12-17,"wk inc flu hosp",0,2022-12-17,200,0,1956,1956,2156 +"MOBS-GLEAM_FLUH",2022-12-17,"wk inc flu hosp",0,2022-12-17,800,0,1356,1356,2156 +"MOBS-GLEAM_FLUH",2022-12-17,"wk inc flu hosp",0,2022-12-17,2000,51.9095854126643,207.909585412664,156,2156 +"MOBS-GLEAM_FLUH",2022-12-17,"wk inc flu hosp",1,2022-12-24,200,0,1794,1794,1994 +"MOBS-GLEAM_FLUH",2022-12-17,"wk inc flu hosp",1,2022-12-24,800,-1.0829831120011,1192.917016888,1194,1994 +"MOBS-GLEAM_FLUH",2022-12-17,"wk inc flu hosp",1,2022-12-24,2000,183.595788453479,183.597608006362,0.00181955288371682,1994 +"MOBS-GLEAM_FLUH",2022-12-17,"wk inc flu hosp",2,2022-12-31,200,0,1967,1967,2167 +"MOBS-GLEAM_FLUH",2022-12-17,"wk inc flu hosp",2,2022-12-31,800,-0.254229578500599,1366.7457704215,1367,2167 +"MOBS-GLEAM_FLUH",2022-12-17,"wk inc flu hosp",2,2022-12-31,2000,-0.0213511038261913,166.978648896174,167,2167 +"MOBS-GLEAM_FLUH",2022-12-17,"wk inc flu hosp",3,2023-01-07,200,0,1436,1436,1636 +"MOBS-GLEAM_FLUH",2022-12-17,"wk inc flu hosp",3,2023-01-07,800,0.0274201731854191,836.027420173185,836,1636 +"MOBS-GLEAM_FLUH",2022-12-17,"wk inc flu hosp",3,2023-01-07,2000,0,0,0,1636 +"PSI-DICE",2022-11-19,"wk inc flu hosp",0,2022-11-19,200,0,1109,1109,1309 +"PSI-DICE",2022-11-19,"wk inc flu hosp",0,2022-11-19,800,1.16673027207588,510.166730272076,509,1309 +"PSI-DICE",2022-11-19,"wk inc flu hosp",0,2022-11-19,2000,0,0,0,1309 +"PSI-DICE",2022-11-19,"wk inc flu hosp",1,2022-11-26,200,0,1950,1950,2150 +"PSI-DICE",2022-11-19,"wk inc flu hosp",1,2022-11-26,800,0,1350,1350,2150 +"PSI-DICE",2022-11-19,"wk inc flu hosp",1,2022-11-26,2000,28.1324675606186,178.132467560619,150,2150 +"PSI-DICE",2022-11-19,"wk inc flu hosp",2,2022-12-03,200,0,2385,2385,2585 +"PSI-DICE",2022-11-19,"wk inc flu hosp",2,2022-12-03,800,-1.23063244352397,1783.76936755648,1785,2585 +"PSI-DICE",2022-11-19,"wk inc flu hosp",2,2022-12-03,2000,9.09494701772928e-13,585.000000000001,585,2585 +"PSI-DICE",2022-11-19,"wk inc flu hosp",3,2022-12-10,200,0,2159,2159,2359 +"PSI-DICE",2022-11-19,"wk inc flu hosp",3,2022-12-10,800,0,1559,1559,2359 +"PSI-DICE",2022-11-19,"wk inc flu hosp",3,2022-12-10,2000,-2.95585778076202e-12,358.999999999997,359,2359 +"PSI-DICE",2022-12-17,"wk inc flu hosp",0,2022-12-17,200,0,1956,1956,2156 +"PSI-DICE",2022-12-17,"wk inc flu hosp",0,2022-12-17,800,-1.60640784583256,1354.39359215417,1356,2156 +"PSI-DICE",2022-12-17,"wk inc flu hosp",0,2022-12-17,2000,147.35838953106,303.35838953106,156,2156 +"PSI-DICE",2022-12-17,"wk inc flu hosp",1,2022-12-24,200,0,1794,1794,1994 +"PSI-DICE",2022-12-17,"wk inc flu hosp",1,2022-12-24,800,-0.597281146154046,1193.40271885385,1194,1994 +"PSI-DICE",2022-12-17,"wk inc flu hosp",1,2022-12-24,2000,249.67472642507,249.676545977954,0.00181955288371682,1994 +"PSI-DICE",2022-12-17,"wk inc flu hosp",2,2022-12-31,200,0,1967,1967,2167 +"PSI-DICE",2022-12-17,"wk inc flu hosp",2,2022-12-31,800,-0.153369559700423,1366.8466304403,1367,2167 +"PSI-DICE",2022-12-17,"wk inc flu hosp",2,2022-12-31,2000,-0.00520756560536029,166.994792434395,167,2167 +"PSI-DICE",2022-12-17,"wk inc flu hosp",3,2023-01-07,200,0,1436,1436,1636 +"PSI-DICE",2022-12-17,"wk inc flu hosp",3,2023-01-07,800,0,836,836,1636 +"PSI-DICE",2022-12-17,"wk inc flu hosp",3,2023-01-07,2000,0,0,0,1636 diff --git a/tests/testthat/testdata/legacy_lnorm_components.csv b/tests/testthat/testdata/legacy_lnorm_components.csv new file mode 100644 index 0000000..684226f --- /dev/null +++ b/tests/testthat/testdata/legacy_lnorm_components.csv @@ -0,0 +1,51 @@ +"K","target_names","x","y","oracle","components_raw","components_oracle","components" +200,"A",9.80853065474633,83.1081720027042,51.9003247051485,73.2996413479578,31.2078472975557,42.0917940504022 +200,"B",12.072037766677,61.1632620408499,38.1959237946417,49.0912242741729,22.9673382462083,26.1238860279646 +200,"C",14.3355448776114,38.3670584743327,23.9599040151198,24.0315135967213,14.4071544592129,9.62435913750839 +200,"D",16.5990519879569,17.3045910429209,10.8065456595817,0.70553905496401,6.49804538333927,-5.79250632837526 +200,"E",18.8625590979256,15.4280751375967,9.63469141774452,0,5.7933837198522,-5.7933837198522 +200,"F",21.1260662076386,33.8232535636052,21.1223313431638,12.6971873559667,12.7009222204414,-0.00373486447471194 +200,"G",23.38957331717,8.33169200923294,5.20305750529424,0,3.1286345039387,-3.1286345039387 +200,"H",25.653080426568,1.09511546324939,0.683870079721273,0,0.411245383528112,-0.411245383528112 +200,"I",27.916587535865,36.6909868994212,22.9132078081527,8.77439936355626,13.7777790912685,-5.00337972771221 +200,"J",30.1800946450837,24.9485736014321,15.5801436714318,0,9.36842993000037,-9.36842993000037 +250,"A",12.2677050070404,83.1081720027042,64.8754058814356,70.8404669956638,18.2327661212685,52.6077008743953 +250,"B",15.0987139504976,61.1632620408499,47.7449047433021,46.0645480903523,13.4183572975479,32.6461907928045 +250,"C",17.9297228925305,38.3670584743327,29.9498800188997,20.4373355818022,8.41717845543297,12.0201571263693 +250,"D",20.7607318333805,17.3045910429209,13.5081820744771,0,3.79640896844385,-3.79640896844385 +250,"E",23.5917407732417,15.4280751375967,12.0433642721807,0,3.38471086541607,-3.38471086541607 +250,"F",26.4227497122665,33.8232535636052,26.4029141789548,7.40050385133878,7.42033938465045,-0.0198355333116638 +250,"G",29.2537586505746,8.33169200923294,6.5038218816178,0,1.82787012761514,-1.82787012761514 +250,"H",32.0847675882603,1.09511546324939,0.854837599651591,0,0.240277863597794,-0.240277863597794 +250,"I",34.9157765253986,36.6909868994212,28.6415097601909,1.77521037402264,8.04947713923029,-6.27426676520765 +250,"J",37.7467854620498,24.9485736014321,19.4751795892897,0,5.47339401214243,-5.47339401214243 +300,"A",14.7131623615289,83.1081720027042,77.8504870577228,68.3950096411753,5.25768494498139,63.1373246961939 +300,"B",18.1087382911125,61.1632620408499,57.2938856919625,43.0545237497374,3.86937634888744,39.18514740085 +300,"C",21.5043142206961,38.3670584743327,35.9398560226797,16.8627442536366,2.42720245165302,14.4355418019836 +300,"D",24.8998901502797,17.3045910429209,16.2098184893725,0,1.09477255354843,-1.09477255354843 +300,"E",28.2954660798633,15.4280751375967,14.4520371266168,0,0.976038010979943,-0.976038010979943 +300,"F",31.6910420094468,33.8232535636052,31.6834970147458,2.13221155415841,2.13975654885948,-0.00754499470107817 +300,"G",35.0866179390305,8.33169200923294,7.80458625794136,0,0.527105751291579,-0.527105751291579 +300,"H",38.4821938686141,1.09511546324939,1.02580511958191,0,0.0693103436674756,-0.0693103436674756 +300,"I",41.8777697981976,36.6909868994212,34.3698117122291,0,2.3211751871921,-2.3211751871921 +300,"J",45.2733457277812,24.9485736014321,23.3702155071477,0,1.57835809428448,-1.57835809428448 +350,"A",17.1701084283076,83.1081720027042,93.2328883388545,65.9380635743966,0,65.9380635743966 +350,"B",21.1326719117538,61.1632620408499,62.4289122386836,40.0305901290961,0,40.0305901290961 +350,"C",25.0952353952007,38.3670584743327,39.6326902772998,13.271823079132,0,13.271823079132 +350,"D",29.0577988786479,17.3045910429209,18.5702800147119,0,0,0 +350,"E",33.0203623620953,15.4280751375967,16.6937040588236,0,0,0 +350,"F",36.9829258455429,33.8232535636052,35.088888123224,0,0,0 +350,"G",40.9454893289907,8.33169200923294,9.59734579247015,0,0,0 +350,"H",44.9080528124385,1.09511546324939,10.5845073598066,0,0,0 +350,"I",48.8706162958864,36.6909868994212,37.9565958549501,0,0,0 +350,"J",52.8331797793344,24.9485736014321,26.214187941252,0,0,0 +400,"A",19.6251175212424,83.1081720027042,110.25558526814,63.4830544814617,0,63.4830544814617 +400,"B",24.154221564606,61.1632620408499,64.5567493548151,37.0090404762439,0,37.0090404762439 +400,"C",28.6833256079697,38.3670584743327,41.7605273935478,9.683732866363,0,9.683732866363 +400,"D",33.2124296513334,17.3045910429209,20.6981171308435,0,0,0 +400,"E",37.741533694697,15.4280751375967,18.8215411749552,0,0,0 +400,"F",42.2706377380606,33.8232535636052,37.2167252394138,0,0,0 +400,"G",46.7997417814242,8.33169200923294,11.7251829086454,0,0,0 +400,"H",51.3288458247879,1.09511546324939,26.5391135013597,0,0,0 +400,"I",55.8579498681515,36.6909868994212,40.0844329710817,0,0,0 +400,"J",60.3870539115151,24.9485736014321,28.3420250574127,0,0,0 diff --git a/tests/testthat/testdata/legacy_lnorm_scores.csv b/tests/testthat/testdata/legacy_lnorm_scores.csv new file mode 100644 index 0000000..e82958c --- /dev/null +++ b/tests/testthat/testdata/legacy_lnorm_scores.csv @@ -0,0 +1,6 @@ +"K","score","score_raw","score_oracle","ytot" +200,48.3387247579936,168.599504993339,120.260780235345,320.260780235345 +250,76.2572846578344,146.51806489318,70.2607802353454,320.260780235345 +300,110.183708963362,130.444489198708,20.2607802353454,320.260780235345 +350,119.240476782625,119.240476782625,0,320.260780235345 +400,110.175827824069,110.175827824069,0,320.260780235345 diff --git a/tests/testthat/testdata/legacy_lnorm_y.rds b/tests/testthat/testdata/legacy_lnorm_y.rds new file mode 100644 index 0000000000000000000000000000000000000000..e39a6c7b5d58e5e5ab17ef50145789788973ee5a GIT binary patch literal 125 zcmb2|=3oE==I#ec2?+^l35m%m32CfGk`d0%cS>{{c(U8J9c{KwVMa?PhDPo zj_E1ixvKsBdhDNgYwOqDJ$T^3g7WC}sV>)4+1saceynuZcj(k27>`NI= cly_+qZDL!s{YOb@J0~+UL(-g?cKJYC0gl-;nE(I) literal 0 HcmV?d00001 diff --git a/tests/testthat/testdata/legacy_reference_meta.json b/tests/testthat/testdata/legacy_reference_meta.json new file mode 100644 index 0000000..ddd3798 --- /dev/null +++ b/tests/testthat/testdata/legacy_reference_meta.json @@ -0,0 +1,10 @@ +{ + "generated_on": "2026-09-02", + "old_alloscore_git_sha": "0477ae9ed97da8fe1fa754e1c3452516aafb3fd9", + "r_version": "R version 4.6.1 (2026-06-24)", + "distfromq_version": "1.0.4", + "hubExamples_version": "1.0.1", + "Ks_lnorm": [200, 250, 300, 350, 400], + "Ks_hub": [200, 800, 2000], + "Ks_slim": [150, 400] +} diff --git a/tests/testthat/testdata/legacy_slim_scores.csv b/tests/testthat/testdata/legacy_slim_scores.csv new file mode 100644 index 0000000..7c068ef --- /dev/null +++ b/tests/testthat/testdata/legacy_slim_scores.csv @@ -0,0 +1,9 @@ +"samp","K","ytot","score_raw","score_oracle","score" +"1",150,391.128258794535,276.407449586467,241.128258794535,35.2791907919314 +"1",400,391.128258794535,180.547889841629,0,180.547889841629 +"2",150,428.900121918159,278.979780574544,278.900121918159,0.0796586563856057 +"2",400,428.900121918159,119.614506955218,28.900121918159,90.7143850370588 +"3",150,357.558147297456,229.282204167897,207.558147297456,21.7240568704411 +"3",400,357.558147297456,99.1486635405122,0,99.1486635405122 +"4",150,660.043996446886,528.007663833769,510.043996446886,17.9636673868826 +"4",400,660.043996446886,345.829571635782,260.043996446886,85.7855751888964 diff --git a/tests/testthat/testdata/legacy_slim_ys.rds b/tests/testthat/testdata/legacy_slim_ys.rds new file mode 100644 index 0000000000000000000000000000000000000000..ef58d7b4ec8e8f5af17d30dd2117e7b9bb3362a4 GIT binary patch literal 420 zcmV;V0bBkbiwFP!000001B>8dU|?WoU}0lqU}gm}8CXL@+;lB~3}GNSrVYE-;2GjNt}jxWgD8Foq|L;RPiA!M$YOFKD`xq1!<<(%H8I=(HgJ^S88tP7hVM z`tUfAHgwOOmzv${VEg=^$^xLg^@jdA7l5?4b3l`Ot&2mDT#q$Ky<_w0#da6|L%hWJ zpU%E;nSW(!tb!+olts2y-ynBMg*ho86z- z0P=;|K1XYyFAR7BR|f$3CR2`vv49f5fcU~rC$hH$=nBR7o@B9As~n=`o^4z!!0W)T zSh_X=l&t0j%rXYL-c3#5ct)SPL)h>BiOE2|O4qqFFM#r~8fvbI{~H~W)3&`?UF%F= OUjP7oY;Jo*0{{R^S;@oz literal 0 HcmV?d00001 diff --git a/tests/testthat/testdata/legacy_zxh_beta.csv b/tests/testthat/testdata/legacy_zxh_beta.csv new file mode 100644 index 0000000..4aeafd2 --- /dev/null +++ b/tests/testthat/testdata/legacy_zxh_beta.csv @@ -0,0 +1,7 @@ +"Product","x" +"1",207.9270789835 +"2",96.7251320468424 +"3",90.3393829658162 +"4",100.77854954193 +"5",90.5490106068021 +"6",211.689762496459 diff --git a/tests/testthat/testdata/legacy_zxh_norm.csv b/tests/testthat/testdata/legacy_zxh_norm.csv new file mode 100644 index 0000000..d94f2ce --- /dev/null +++ b/tests/testthat/testdata/legacy_zxh_norm.csv @@ -0,0 +1,18 @@ +"Product","x" +"1",0 +"2",0 +"3",0 +"4",0 +"5",0 +"6",106.856347554936 +"7",0 +"8",14.02132022895 +"9",0 +"10",0 +"11",15.70733211563 +"12",42.2548280047259 +"13",34.5969264061139 +"14",0 +"15",0 +"16",0 +"17",15.1847499600554