Size score-column precision to the column's variation, not its minimum

README.md

@@ -1,6 +1,10 @@
 # predevals
 A JavaScript module for interactive exploration of forecast evaluations.
 
+## Score rounding
+
+Each score column in the table gets one number of decimal places, chosen from the values in that column so the decimal points line up. The rule is: measure the spread of the middle half of the column (its interquartile range) and use just enough decimals to show that spread to two digits. A column whose models fall between 0.19 and 1.27 gets 2 decimals (`0.27`, `0.31`, `0.30`); one whose models fall between 1247 and 15679 gets none (`1247`, `15679`). Two guards sit on top: no column shows more than 5 significant figures, and rounding never touches digits left of the decimal point, so 12345.6 displays as `12346`, never `12300`. Using the middle half rather than the smallest and largest values is what keeps one blown-up model from flattening the whole column and one near-zero model from padding every other row with meaningless decimals; a value too small to show at the column's width displays as less than the smallest value that width can show -- `<0.01` in a 2-decimal column, `<0.0001` in a 4-decimal one -- rather than as `0`. Relative skill columns always use 2 decimals and coverage columns 1, since those scales are known in advance. Rounding is display-only: sorting and downloaded data always use the full values.
+
 # Development
 
 ## Installing dev requirements

src/predevals.js

@@ -3,7 +3,7 @@
  */
 
 import * as d3 from "d3";
-import {axis_kind, base_col_name, convertDataColumnTypes, get_round_decimals, hexToRGB, is_coverage_col, is_n_col, is_relative_skill_col, parse_coverage_rate, reference_line_value, score_col_name_to_text, toArray} from "./utils.js";
+import {axis_kind, base_col_name, convertDataColumnTypes, get_round_decimals, hexToRGB, is_coverage_col, is_n_col, is_relative_skill_col, parse_coverage_rate, reference_line_value, render_score, score_col_name_to_text, toArray} from "./utils.js";
 import {nDefinition, metricDefinitions} from "./metric-definitions.js";
 
 
@@ -652,12 +652,17 @@ const App = {
                 return {data: columnName, name: columnName};
             } else {
                 // format score columns: relative_skill → 2 dp, interval_coverage → 1 dp,
-                // all others → auto-detected minimum decimals needed (see get_round_decimals).
+                // all others → the decimals that resolve the column's variation (see get_round_decimals).
                 // Note: we only build tables if disaggregate_by is '(None)', so we can assume that all
                 // columns other than model_id and the `n` column(s) are scores
                 const colValues = thisState.scores_table.map(row => row[columnName]);
                 const decimals = get_round_decimals(columnName, colValues);
-                return {data: columnName, name: columnName, render: (d) => d.toFixed(decimals)};
+
+                // only format for display: render_score() can emit '<0.01', which DataTables would
+                // then type-detect as a string and sort lexicographically. Sorting gets the number.
+                const render = (d, type) => (type === 'display' || type === 'filter')
+                    ? render_score(columnName, d, decimals) : d;
+                return {data: columnName, name: columnName, render: render};
             }
         });
         const targetObj = this.getSelectedTargetObj();
@@ -903,7 +908,14 @@ const App = {
                 // their extremes, so nothing sits on the edge for them and this is a no-op.
                 cliponaxis: false,
                 hovermode: false,
-                hovertemplate: `model: %{data.name}<br>${thisState.selected_disaggregate_by}: %{x}<br>${score_col_name_to_text(this.state.selected_metric)}: %{y:.${metricDecimals}f}<extra></extra>`,
+
+                // pre-format through render_score() rather than let Plotly round, so hover goes
+                // through the same rule as every other score display -- including its '<0.01'
+                // floor and its '' for a missing score, where `%{y:.Nf}` rendered a bare 'NaN'.
+                // (The table's own decimals come from the non-disaggregated `scores_table`, so
+                // while disaggregation is on the two panes can still differ in decimals.)
+                customdata: y.map(v => render_score(thisState.selected_metric, v, metricDecimals)),
+                hovertemplate: `model: %{data.name}<br>${thisState.selected_disaggregate_by}: %{x}<br>${score_col_name_to_text(this.state.selected_metric)}: %{customdata}<extra></extra>`,
                 opacity: 0.7,
             };
             pd.push(line_data);
@@ -1068,9 +1080,12 @@ const App = {
             x: x,
             y: y,
             z: z,
-            customdata: z_orig,
+
+            // pre-format through render_score() rather than let Plotly round, so hover goes through
+            // the same rule as every other score display (see the line plot's note on this)
+            customdata: z_orig.map(row => row.map(v => v === null ? null : render_score(thisState.selected_metric, v, metricDecimals))),
             type: 'heatmap',
-            hovertemplate: `${thisState.selected_disaggregate_by}: %{x}<br>model: %{y}<br>${score_col_name_to_text(this.state.selected_metric)}: %{customdata:.${metricDecimals}f}<extra></extra>`,
+            hovertemplate: `${thisState.selected_disaggregate_by}: %{x}<br>model: %{y}<br>${score_col_name_to_text(this.state.selected_metric)}: %{customdata}<extra></extra>`,
             hoverongaps: false
         };
 

src/utils.js

@@ -13,10 +13,9 @@ function hexToRGB(hex) {
  * Return the number of decimal places to use when rendering a score column.
  *
  * - `*_scaled_relative_skill` columns always use 2 decimal places.
- * - `interval_coverage_*` columns always use 1 (values are 0–100 percentages).
- * - All other columns: when `values` is supplied, returns the minimum decimals
- *   needed so that no non-zero value rounds to zero (see `min_decimals_for_values`);
- *   otherwise falls back to 1.
+ * - `interval_coverage_*` columns always use 1 (values are 0-100 percentages).
+ * - All other columns: when `values` is supplied, returns the decimals that resolve the column's
+ *   variation (see `score_decimals`); otherwise falls back to 1.
  *
  * @param {string} col_name
  * @param {Array<number>|null} [values=null]
@@ -27,30 +26,90 @@ function get_round_decimals(col_name, values = null) {
         return 2;
     }
     if (!is_coverage_col(col_name) && values !== null) {
-        return min_decimals_for_values(values);
+        return score_decimals(values);
     }
     return 1;
 }
 
 /**
- * Return the minimum number of decimal places needed so that every non-zero
- * value in `values` renders as non-zero (i.e., doesn't round to "0.000…0").
- * Null, undefined, non-finite, and zero entries are ignored.
- * Returns 1 when the array is empty or contains only zeros/non-finite values.
+ * Decimals for a score column, chosen so the displayed digits resolve the column's variation
+ * ("two effective digits", Ehrenberg 1977, JRSS A 140(3), 277-297), capped so the column never
+ * carries more than `maxSigFigs`. Display only: never applied to sort keys or downloaded data.
+ *
+ * Both anchors are quantiles rather than extremes: the range is set by the worst model, but the
+ * comparison that matters is among the contenders at the top, so neither one blown-up submission
+ * (which would flatten the leaderboard through the cap) nor one near-zero score (which would pad
+ * every other row with spurious decimals) gets to set the whole column's precision.
+ *
+ * Because the rule is decimals-based (`toFixed`) rather than significant-figure-based (R's
+ * `signif`), it never rounds digits left of the decimal point: a national-scale MAE of 12345.6
+ * renders as `12346`, never `12300`. `maxSigFigs` only ever removes decimals, and it bottoms out
+ * at 0.
+ *
+ * Note: `maxSigFigs` is applied as a hard minimum against the spread, so it wins where the two
+ * disagree. Above ~1e4 that can flatten neighbors the spread would have resolved: [12345.1,
+ * 12345.2] wants 1 decimal and gets 0. The loss is the sixth significant figure rather than the
+ * first, which is the trade the cap is there to make; raise `maxSigFigs` if a hub's scores live at
+ * that scale and the distinction matters.
+ *
+ * Note: depends on the rows currently in `scores_table`, so precision can shift under filtering.
+ * That is inherent to any column-wide rule, and is the price of keeping decimal points aligned.
  *
  * @param {Array<number|null|undefined>} values
+ * @param {Object} [options]
  * @returns {number}
  */
-function min_decimals_for_values(values) {
-    const nonZeroAbs = values
-        .filter(v => v !== null && v !== undefined && isFinite(v) && v !== 0)
-        .map(v => Math.abs(v));
-    if (nonZeroAbs.length === 0) return 1;
-    const minVal = Math.min(...nonZeroAbs);
-    // ceil(-log10(minVal)) = decimal places needed to show 1 significant figure for the smallest value.
-    // Subtract a tiny epsilon before ceil to guard against floating-point overshoot
-    // (e.g. -log10(0.001) can return 2.9999... instead of 3 exactly).
-    return Math.max(1, Math.ceil(-Math.log10(minVal) - 1e-10));
+function score_decimals(values, {effDigits = 2, fallbackSigFigs = 3, maxSigFigs = 5, maxDecimals = 6} = {}) {
+    const vals = values.filter(v => v !== null && v !== undefined && isFinite(v));
+    const nonZeroAbs = vals.filter(v => v !== 0).map(Math.abs);
+    if (nonZeroAbs.length === 0) return 0;
+
+    // epsilon guards floating-point overshoot, e.g. log10(0.001) === -3.0000000000000004
+    const mag = (x) => Math.floor(Math.log10(x) + 1e-10);
+    const quantile = (sorted, p) => sorted[Math.min(sorted.length - 1, Math.floor(p * (sorted.length - 1)))];
+    const sorted = [...vals].sort((a, b) => a - b);
+
+    // Widen the window in rungs when the IQR is degenerate (few rows, or many ties). Going straight
+    // from the IQR to the full range would hand the column back to the outliers the IQR anchoring
+    // exists to keep out: eight tied 0.5s next to one 52000 would render as "1" nine times over,
+    // which is the very failure this function was written to fix. The full range is still the last
+    // rung, but by then the column is tied from P10 to P90 and there is nothing else left to measure.
+    const spread_between = (lo, hi) => quantile(sorted, hi) - quantile(sorted, lo);
+    const spread = spread_between(0.25, 0.75) || spread_between(0.10, 0.90) || spread_between(0, 1);
+    const large = quantile([...nonZeroAbs].sort((a, b) => a - b), 0.75);
+    const d_spread = spread > 0 ? effDigits - 1 - mag(spread)
+                                : fallbackSigFigs - 1 - mag(large);
+    const d_cap = maxSigFigs - 1 - mag(large);
+    return Math.min(Math.max(d_spread, 0), Math.max(d_cap, 0), maxDecimals);
+}
+
+/**
+ * Render one score cell as a string. `decimals` comes from `get_round_decimals()`, computed once
+ * per column so that decimal points line up down the column.
+ *
+ * Values too small to survive the column's rounding render as `<0.01` (or `>-0.01`) rather than as
+ * a bare `0`, which is what keeps a column-wide rule from claiming a real non-zero score is zero.
+ *
+ * @param {string} col_name
+ * @param {number|null|undefined} value
+ * @param {number} decimals
+ * @returns {string}
+ */
+function render_score(col_name, value, decimals) {
+    if (value === null || value === undefined || !isFinite(value)) {
+        return '';
+    }
+    if (is_relative_skill_col(col_name)) {
+        return value.toFixed(2);
+    }
+    if (is_coverage_col(col_name)) {
+        return value.toFixed(1);
+    }
+    const smallest = Math.pow(10, -decimals);
+    if (value !== 0 && Math.abs(value) < smallest / 2) {
+        return (value < 0 ? '>-' : '<') + smallest.toFixed(decimals);
+    }
+    return value.toFixed(decimals);
 }
 
 
@@ -238,4 +297,4 @@ function axis_kind(values) {
     return 'category';
 }
 
-export {titleCase, hexToRGB, min_decimals_for_values, get_round_decimals, parse_coverage_rate, split_transformed_col_name, base_col_name, is_n_col, is_coverage_col, is_relative_skill_col, reference_line_value, score_col_name_to_text, convertDataColumnTypes, toArray, axis_kind}
+export {titleCase, hexToRGB, score_decimals, render_score, get_round_decimals, parse_coverage_rate, split_transformed_col_name, base_col_name, is_n_col, is_coverage_col, is_relative_skill_col, reference_line_value, score_col_name_to_text, convertDataColumnTypes, toArray, axis_kind}

test/ui.js

@@ -65,6 +65,30 @@ function useStubbedApp(hooks) {
     });
 }
 
+// DataTables isn't loaded in the jsdom test env; stub the pieces updateTable() touches so the
+// <thead> is still assembled (that DOM is built before the DataTable() hand-off). Returns a getter
+// for the config updateTable() handed over, which is where the column render functions live.
+function stubDataTable(hooks) {
+    let originalDataTable;
+    let capturedConfig;
+    hooks.beforeEach(() => {
+        originalDataTable = $.fn.DataTable;
+        capturedConfig = null;
+        const DataTableStub = function (config) {
+            capturedConfig = config;
+            return {destroy() {}};
+        };
+        DataTableStub.isDataTable = function () {
+            return false;
+        };
+        $.fn.DataTable = DataTableStub;
+    });
+    hooks.afterEach(() => {
+        $.fn.DataTable = originalDataTable;
+    });
+    return () => capturedConfig;
+}
+
 // A scores_table stand-in carrying only the `.columns` property the render paths read.
 function scoresWithColumns(columns) {
     const scores = [];
@@ -215,23 +239,7 @@ QUnit.module('metric definitions glossary', (hooks) => {
 
 QUnit.module('scores table headers', (hooks) => {
     useStubbedApp(hooks);
-
-    // DataTables isn't loaded in the jsdom test env; stub the pieces updateTable() touches so the
-    // <thead> is still assembled (that DOM is built before the DataTable() hand-off).
-    let originalDataTable;
-    hooks.beforeEach(() => {
-        originalDataTable = $.fn.DataTable;
-        const DataTableStub = function () {
-            return {destroy() {}};
-        };
-        DataTableStub.isDataTable = function () {
-            return false;
-        };
-        $.fn.DataTable = DataTableStub;
-    });
-    hooks.afterEach(() => {
-        $.fn.DataTable = originalDataTable;
-    });
+    stubDataTable(hooks);
 
     const headerTexts = () => textsOf('#predeval_table thead th');
 
@@ -262,6 +270,89 @@ QUnit.module('scores table headers', (hooks) => {
 });
 
 
+//
+// scores table cell rendering tests
+//
+
+// These tests cover the whole per-column path the scores table takes: read the column's values ->
+// pick one decimal count for the column (get_round_decimals()) -> turn each value into a cell
+// string (render_score()). test/utils.js pins down the two functions in isolation; what is asserted
+// here is that updateTable() actually wires them together, on the DataTables column config, with
+// the column's real values rather than the fixed 1 decimal that predates issue #88.
+
+QUnit.module('scores table cell rendering', (hooks) => {
+    useStubbedApp(hooks);
+    const dtConfig = stubDataTable(hooks);
+
+    // the flusight `wis__log` column from issue #88, trimmed to the rows that make the point: under
+    // the old min-anchored rule every one of these rendered as "0.3"
+    const WIS_LOG_ROWS = [0.265444756757736, 0.305843546429542, 0.296765334110182, 0.194580477360387,
+        0.42445635328479, 0.690585048746816, 1.27387287336059, 0.343356886197072];
+
+    // Build a scores_table of real rows (not the header-only `scoresWithColumns` stand-in) so the
+    // column-values → decimals → cell-string path runs for real.
+    function tableWith(columnValues) {
+        const columns = ['model_id', ...Object.keys(columnValues)];
+        const nRows = Object.values(columnValues)[0].length;
+        const scores = Array.from({length: nRows}, (_, i) => {
+            const row = {model_id: `model-${i}`};
+            for (const [col, values] of Object.entries(columnValues)) {
+                row[col] = values[i];
+            }
+            return row;
+        });
+        scores.columns = columns;
+        return scores;
+    }
+
+    const renderFor = (columnName) => dtConfig().columns.find((c) => c.name === columnName).render;
+
+    test('renders score cells at the decimals that resolve the column (issue #88)', assert => {
+        // Case: the table is handed the flusight `wis__log` column, whose values all sit
+        // between 0.19 and 1.27.
+        // Desired: the cells come back at the 2 decimals that separate these models. Rendering at
+        // the old fixed 1 decimal would print all three of these rows as "0.3", which is the
+        // screenshot in issue #88.
+        App.state.scores_table = tableWith({wis__log: WIS_LOG_ROWS});
+        App.updateTable();
+
+        const render = renderFor('wis__log');
+        assert.equal(render(0.265444756757736, 'display'), '0.27');
+        assert.equal(render(0.305843546429542, 'display'), '0.31');
+        assert.equal(render(0.296765334110182, 'display'), '0.30');
+    });
+
+    test('hands DataTables the raw number for sorting', assert => {
+        // Case: DataTables calls the same render() for every data type it needs, not just for
+        // display - 'sort' and 'type' among them.
+        // Desired: only 'display' and 'filter' get the rounded string. Sorting and type detection
+        // get the untouched number, so the column orders by value; a string such as '<0.01' in a
+        // sort key would silently switch the whole column to lexicographic ordering. 'filter' gets
+        // the string on purpose, so that a search matches what the reader can actually see.
+        App.state.scores_table = tableWith({wis__log: WIS_LOG_ROWS});
+        App.updateTable();
+
+        const render = renderFor('wis__log');
+        assert.strictEqual(render(0.305843546429542, 'sort'), 0.305843546429542);
+        assert.strictEqual(render(0.305843546429542, 'type'), 0.305843546429542);
+        assert.equal(render(0.305843546429542, 'filter'), '0.31', 'search matches what is displayed');
+    });
+
+    test('renders whole-number-scale columns without a decimal place', assert => {
+        // Case: an ae_median column in the tens-to-hundreds, spread wide enough that a tenth
+        // of a case carries no information.
+        // Desired: whole numbers, and every integer digit kept - "291", not "290.7" and not the
+        // "290" that a significant-figures rule would produce. The decimal count is per column, so
+        // this and the wis__log column above coexist in one table at different widths.
+        App.state.scores_table = tableWith({ae_median: [290.690197703552, 101.287927350427, 76.9, 498.2, 11.3]});
+        App.updateTable();
+
+        const render = renderFor('ae_median');
+        assert.equal(render(290.690197703552, 'display'), '291');
+        assert.equal(render(11.3, 'display'), '11');
+    });
+});
+
 //
 // plot x-axis tests
 //