diff --git a/packages/data-generator/.mocharc.json b/packages/data-generator/.mocharc.json new file mode 100644 index 00000000..6d0022d7 --- /dev/null +++ b/packages/data-generator/.mocharc.json @@ -0,0 +1,5 @@ +{ + "extension": ["ts"], + "require": "ts-node/register", + "spec": "test/**/*.spec.ts" +} diff --git a/packages/data-generator/README.md b/packages/data-generator/README.md new file mode 100644 index 00000000..1132c0eb --- /dev/null +++ b/packages/data-generator/README.md @@ -0,0 +1,614 @@ +# Data Generator + +Internal test utility package for generating data that conforms to any given Lectern Data Dictionary. + +Used by `packages/validation` and other packages that need programmatically generated test data for performance and correctness testing. + +This package is **private** and not published to NPM. + +--- + +## Usage + +### Generating records in memory + +```ts +import { + generateRecord, + generateSchemaRecords, + generateDictionaryRecords, +} from '@overture-stack/lectern-data-generator'; +import { donorSchema, sampleSchema, myDictionary } from './fixtures'; + +// Generate a single record conforming to a schema +const record = generateRecord(donorSchema, { seed: 42 }); + +// Generate a record with a specific field value forced +const recordWithOverride = generateRecord(donorSchema, { + overrides: { id: 'DUPLICATE-ID' }, +}); + +// Generate a child record whose foreign key fields are drawn from an existing set of parent rows +const parentRows = [{ id: 'P001' }, { id: 'P002' }]; +const childRecord = generateRecord(sampleSchema, { + seed: 42, + foreignKeyPool: new Map([['donor', parentRows]]), +}); + +// Lazily generate 10,000 records for a schema — one at a time, without buffering +for (const record of generateSchemaRecords(donorSchema, { count: 10_000, seed: 42 })) { + process(record); +} + +// Lazily generate records for an entire dictionary in foreign key dependency order +// Parent schema records are yielded before their child schema records +for (const { schemaName, record } of generateDictionaryRecords(myDictionary, { + counts: { donor: 1_000, sample: 5_000 }, + seed: 42, +})) { + process(schemaName, record); +} +``` + +### Writing generated data to files + +```ts +import { generateSchemaFile, generateDictionaryFiles } from '@overture-stack/lectern-data-generator'; +import { donorSchema, myDictionary } from './fixtures'; + +// Write 10,000 records to a new TSV file — streams to disk without buffering +const schemaResult = await generateSchemaFile(donorSchema, '/tmp/output', 'tsv', { + count: 10_000, + seed: 42, +}); +if (!schemaResult.success) { + // schemaResult.data.error is 'DIRECTORY_NOT_FOUND' | 'FILE_ALREADY_EXISTS' + console.error(schemaResult.data.error); +} + +// Write records for every schema in a dictionary to separate files in one call +// All file paths are checked before any writing begins — no partial output on failure +const dictionaryResult = await generateDictionaryFiles(myDictionary, '/tmp/output', 'tsv', { + counts: { donor: 1_000, sample: 5_000 }, + seed: 42, +}); +if (!dictionaryResult.success) { + console.error(dictionaryResult.data.error); +} +// Produces: /tmp/output/donor.tsv, /tmp/output/sample.tsv +``` + +--- + +## API Documentation + +### Table of Contents + +- [Data Generator](#data-generator) + - [Usage](#usage) + - [API Documentation](#api-documentation) + - [Table of Contents](#table-of-contents) + - [Generator Behaviour](#generator-behaviour) + - [Seeded generation](#seeded-generation) + - [Field Generation](#field-generation) + - [Conditional restrictions](#conditional-restrictions) + - [Empty fields](#empty-fields) + - [Generator failures](#generator-failures) + - [Reference tags in restrictions](#reference-tags-in-restrictions) + - [Record Generation](#record-generation) + - [Field dependency ordering](#field-dependency-ordering) + - [Foreign key constraints](#foreign-key-constraints) + - [Schema and Dictionary Generation](#schema-and-dictionary-generation) + - [Unique field constraints](#unique-field-constraints) + - [Unique key constraints](#unique-key-constraints) + - [Foreign Key dependency ordering across schemas](#fk-dependency-ordering-across-schemas) + - [API - Functions](#api---functions) + - [`generateStringValue`](#generatestringvalue) + - [`generateIntegerValue`](#generateintegervalue) + - [`generateNumberValue`](#generatenumbervalue) + - [`generateBooleanValue`](#generatebooleanvalue) + - [`generateRecord`](#generaterecord) + - [`generateSchemaRecords`](#generateschemrecords) + - [`generateDictionaryRecords`](#generatedictionaryrecords) + - [`generateSchemaFile`](#generateschemafile) + - [`generateDictionaryFiles`](#generatedictionaryfiles) + - [API - Types](#api---types) + - [`FieldGenerator`](#fieldgenerator) + - [`FieldGeneratorOptions`](#fieldgeneratoroptions) + - [`FieldGeneratorResult`](#fieldgeneratorresult) + - [`ForeignKeyPool`](#foreignkeypool) + - [`RecordGeneratorOptions`](#recordgeneratoroptions) + - [`SchemaGeneratorOptions`](#schemageneratoroptions) + - [`DictionaryGeneratorOptions`](#dictionarygeneratoroptions) + - [`DictionaryRecord`](#dictionaryrecord) + - [`DataFileFormat`](#datafileformat) + - [`GenerateFileError`](#generatefileerror) + +--- + +## Generator Behaviour + +### Seeded generation + +All generators accept an optional `seed` number. When provided, the same seed and the same schema always produce the same output — useful for snapshot tests and repeatable performance benchmarks. + +**Important:** reproducibility depends on the schema remaining unchanged. Modifying a field's restrictions (adding a `codeList`, tightening a `range`, etc.) will produce different values even with the same seed. + +When no seed is provided, a random seed is chosen at runtime and output is non-deterministic. + +### Field Generation + +#### Conditional restrictions + +Field definitions in Lectern schemas can include conditional restrictions (`if/then/else` blocks) that activate different restrictions depending on the values of other fields in the same record. Each field generator accepts an optional `record` parameter - a partial copy of the record being built - so that the correct restriction branch can be resolved before generating the value. + +When `record` is omitted or a referenced field is absent from the provided record, the missing field is treated as `undefined`. + +#### Empty fields + +By default, each generator has a 25% chance of returning `undefined` instead of a generated value for any field that does not carry a `required: true` restriction. This reflects the reality that optional fields in real datasets are frequently absent. + +The rate is controlled by the `emptyRate` option on `FieldGeneratorOptions`, `RecordGeneratorOptions`, `SchemaGeneratorOptions`, and `DictionaryGeneratorOptions`: + +- `emptyRate: 0` — never produce empty fields; always generate a value. +- `emptyRate: 1` — always produce empty fields for non-required fields. +- `emptyRate: 0.25` — default; approximately one field in four is left empty. + +The empty check is seeded alongside the value draw, so the same seed always produces the same empty/non-empty outcome. + +#### Generator failures + +Dictionaries can specify restrictions that are contradictory, making it impossible to generate a valid value. In these cases the generator returns a failure result rather than throwing. The returned object contains: + +- A fallback value (best-effort, may not satisfy all restrictions). +- A list of conflicts describing which restrictions could not be reconciled. + +Restriction combinations that can produce a failure: + +- **Multiple `codeList` restrictions with no common values** — the intersection of two or more code lists is empty. +- **Multiple `range` restrictions that do not overlap** — the merged lower bound exceeds the merged upper bound, or both bounds are equal and at least one is exclusive. +- **`codeList` and `range` together with no intersection** — none of the code list values fall within the specified range. +- **`codeList` and `regex` together with no intersection** — none of the code list values match the specified regex pattern. + +Callers should check `result.success` before using the value in a context that requires a valid record. + +#### Reference tags in restrictions + +A resolved Lectern dictionary replaces all references with their concrete values before use. If a dictionary is passed to the generators with unresolved references still present (strings starting with `#/`), those entries are silently skipped. The generator continues with whichever concrete values remain. + +If all entries in a `codeList` are reference tags, the generator falls back to producing an arbitrary value of the appropriate type, as though no `codeList` restriction were present. + +### Record Generation + +#### Field dependency ordering + +`generateRecord` statically analyses the schema's conditional restrictions to build a dependency map between fields. Fields are then generated in topological order: any field that conditionally references another field is always generated after the field it depends on, so the partial record available to each generator reflects the correct values when evaluating conditional branches. + +If a dependency cycle exists (field A conditionally references field B, and field B conditionally references field A), all fields involved in the cycle are placed in the same generation tier and generated with an incomplete partial record. + +#### Foreign key constraints + +Lectern schemas can declare foreign key constraints at the schema level (`schema.restrictions.foreignKey`). Each foreign key rule names a parent schema and maps one or more local field names to corresponding field names in the parent schema. + +When generating child records, pass a `ForeignKeyPool` to `generateRecord` so that foreign key constrained fields are populated from actual parent rows rather than generated freely: + +```ts +const pool: ForeignKeyPool = new Map([['donor', [{ id: 'D001' }, { id: 'D002' }, { id: 'D003' }]]]); +const childRecord = generateRecord(sampleSchema, { seed: 42, foreignKeyPool: pool }); +// childRecord.donor_id will be one of 'D001', 'D002', or 'D003' +``` + +For composite foreign keys (a single rule with multiple field mappings), all local fields are drawn from the **same** randomly selected parent row, preserving relational consistency. + +See the `ForeignKeyPool` type documentation below for the expected pool structure. + +### Schema and Dictionary Generation + +#### Unique field constraints + +When a field carries `unique: true`, `generateSchemaRecords` tracks every value it has generated for that field and excludes already-seen values from subsequent draws. This ensures all generated values for that field are distinct across the full set of yielded records. + +If the field's value space is exhausted (e.g. a `codeList` with five entries and six records requested), the generator yields a best-effort value that may duplicate an earlier one rather than throwing. + +#### Unique key constraints + +When a schema declares `restrictions.uniqueKey`, `generateSchemaRecords` tracks the composite key tuple for every record and retries generation (up to 10 times) when a collision is detected. Retry seeds are derived deterministically from the record seed and the retry count, so retries do not affect the seed sequence for non-colliding records. + +`initialUniqueValues` can be used to pre-populate both trackers (for `unique` fields and `uniqueKey` tuples) when appending to an existing dataset, so the generator avoids colliding with already-written values. + +#### Foreign key dependency ordering across schemas + +`generateDictionaryRecords` resolves schema generation order using a topological sort on the dictionary's foreign key relationships. Parent schemas are always fully generated before any dependent child schemas begin. As each parent schema finishes, its generated records are collected into a pool that child schemas draw from to populate their foreign key fields — ensuring every child record references a value that actually exists in the parent. + +Schemas with no foreign key relationships between them may appear in the same tier and are generated sequentially within that tier. + +--- + +### API - Functions + +#### `generateStringValue` + +Generates a value for a `SchemaStringField`. Returns a single `string` or `undefined`, or `string[]` when the field has `isArray: true`. + +The generator reads the field's restrictions (including conditional branches, resolved against the provided `record`) and produces a value satisfying all active restrictions: + +- If the field is not `required` and the empty check fires (see `emptyRate`), returns `undefined`. +- If `codeList` is present, picks a random element from the list. +- If `regex` is present, generates a string matching the pattern. +- If both `codeList` and `regex` are active, filters the code list to values that also satisfy the regex; returns a failure if the intersection is empty. +- Otherwise, returns an arbitrary human-readable string. +- If `field.isArray` is `true`, returns an array of generated values. Array length is controlled by `options.arrayLength` (default 1–3). + +**Parameters** + +| Parameter | Type | Description | +| --------- | ---------------------------------- | --------------------------------------------------- | +| `field` | `SchemaStringField` | The field definition to generate a value for. | +| `options` | `FieldGeneratorOptions` (optional) | Seed, record context, array length, and empty rate. | + +**Returns:** `FieldGeneratorResult` — success wrapping `string | string[] | undefined`, or failure with conflict details. + +--- + +#### `generateIntegerValue` + +Generates a value for a `SchemaIntegerField`. Returns a single `number` (integer) or `undefined`, or `number[]` when `isArray: true`. + +- If the field is not `required` and the empty check fires (see `emptyRate`), returns `undefined`. +- If `codeList` is present, picks a random element from the list. +- If `range` is present, generates an integer within the bounds (`min`/`max`/`exclusiveMin`/`exclusiveMax`). +- If both `codeList` and `range` are active, filters the code list to values within the range; returns a failure if none qualify. +- If multiple `range` restrictions are active, they are intersected; returns a failure if the intersection is empty. +- Otherwise, returns an arbitrary integer. +- If `field.isArray` is `true`, returns an array. Length is controlled by `options.arrayLength` (default 1–3). + +**Parameters** + +| Parameter | Type | Description | +| --------- | ---------------------------------- | --------------------------------------------------- | +| `field` | `SchemaIntegerField` | The field definition to generate a value for. | +| `options` | `FieldGeneratorOptions` (optional) | Seed, record context, array length, and empty rate. | + +**Returns:** `FieldGeneratorResult` — success wrapping `number | number[] | undefined`, or failure with conflict details. + +--- + +#### `generateNumberValue` + +Generates a value for a `SchemaNumberField`. Returns a single `number` (may be floating-point) or `undefined`, or `number[]` when `isArray: true`. + +Behaviour mirrors `generateIntegerValue`. The difference is that when no `codeList` or `range` constrains the output, the generated value may be a floating-point number rather than an integer. + +- If the field is not `required` and the empty check fires (see `emptyRate`), returns `undefined`. +- If `codeList` is present, picks a random element from the list. +- If `range` is present, generates a float within the bounds. +- If both `codeList` and `range` are active, filters the code list to values within the range; returns a failure if none qualify. +- If multiple `range` restrictions are active, they are intersected; returns a failure if the intersection is empty. +- Otherwise, returns an arbitrary floating-point number. +- If `field.isArray` is `true`, returns an array. Length is controlled by `options.arrayLength` (default 1–3). + +**Parameters** + +| Parameter | Type | Description | +| --------- | ---------------------------------- | --------------------------------------------------- | +| `field` | `SchemaNumberField` | The field definition to generate a value for. | +| `options` | `FieldGeneratorOptions` (optional) | Seed, record context, array length, and empty rate. | + +**Returns:** `FieldGeneratorResult` — success wrapping `number | number[] | undefined`, or failure with conflict details. + +--- + +#### `generateBooleanValue` + +Generates a value for a `SchemaBooleanField`. Returns a single `boolean` or `undefined`, or `boolean[]` when `isArray: true`. + +Returns `true` or `false` at random. If the field is not `required` and the empty check fires (see `emptyRate`), returns `undefined` instead. If `field.isArray` is `true`, returns an array. Length is controlled by `options.arrayLength` (default 1–3). + +`required: true` combined with `empty: true` across the active restrictions is a conflict and produces a failure result, though a value is still generated. + +**Parameters** + +| Parameter | Type | Description | +| --------- | ---------------------------------- | --------------------------------------------------- | +| `field` | `SchemaBooleanField` | The field definition to generate a value for. | +| `options` | `FieldGeneratorOptions` (optional) | Seed, record context, array length, and empty rate. | + +**Returns:** `FieldGeneratorResult` — success wrapping `boolean | boolean[] | undefined`, or failure if `required` and `empty` conflict. + +--- + +#### `generateRecord` + +Assembles a complete `DataRecord` for a given `Schema` by calling the appropriate field generator for each field. + +Fields are generated in dependency order (see [Field dependency ordering](#field-dependency-ordering)). Each field generator receives the partial record built so far so that conditional restrictions referencing earlier fields resolve correctly. + +If `options.overrides` is provided, fields with a matching key use the override value directly and are not generated. If `options.foreignKeyPool` is provided, foreign key constrained fields are populated from the pool before generation begins. Explicit overrides take priority over pool values. + +If `seed` is provided, the same seed and schema always produce the same `DataRecord`. Per-field seeds are derived from the record-level seed by definition-order index in `schema.fields`, preserving seed stability even when generation order differs from definition order. + +**Parameters** + +| Parameter | Type | Description | +| --------- | ------------------------------------- | ------------------------------------ | +| `schema` | `Schema` | The schema to generate a record for. | +| `options` | `RecordGeneratorOptions` _(optional)_ | Generation options — see type below. | + +**Returns:** `DataRecord` — a record with a value (or `undefined`) for every field in the schema. + +--- + +#### `generateSchemaRecords` + +A synchronous generator that lazily yields `DataRecord` values for a given `Schema`. Records are produced one at a time — none are buffered in memory. + +Enforces `unique` field constraints by excluding already-seen values from each field generator. Enforces `uniqueKey` constraints by retrying generation (up to 10 times) with a deterministically derived seed when a composite key tuple collides. + +**Parameters** + +| Parameter | Type | Description | +| --------- | ------------------------------------- | -------------------------------------------------------- | +| `schema` | `Schema` | The schema to generate records for. | +| `options` | `SchemaGeneratorOptions` _(optional)_ | Count, seed, foreign key pool, empty rate, initial unique values. | + +**Returns:** `Generator` — yields one record per iteration. + +--- + +#### `generateDictionaryRecords` + +A synchronous generator that lazily yields `DictionaryRecord` values (tagged `{ schemaName, record }` pairs) for all schemas in a dictionary that have a non-zero count. + +Schemas are generated in foreign key dependency order. Parent schemas are fully generated and their records held in a pool before any child records are yielded. Child records stream out one at a time without being retained in memory. + +**Parameters** + +| Parameter | Type | Description | +| ------------ | ---------------------------- | --------------------------------------------------------- | +| `dictionary` | `Dictionary` | The dictionary to generate records for. | +| `options` | `DictionaryGeneratorOptions` | Counts per schema, seed, and empty rate — see type below. | + +**Returns:** `Generator` — yields one tagged record per iteration, parents before children. + +--- + +#### `generateSchemaFile` + +Generates records for a schema and writes them to a new file in the given output directory. The file is named `.` (e.g. `donor.tsv`). Records are streamed to disk without buffering. + +Fails before writing if the output directory does not exist or if the output file already exists. + +**Parameters** + +| Parameter | Type | Description | +| ----------- | ------------------------------------- | ------------------------------------------------------------- | +| `schema` | `Schema` | The schema to generate records for. | +| `outputDir` | `string` | Path to an existing directory where the file will be written. | +| `format` | `DataFileFormat` | Column delimiter format: `'tsv'` or `'csv'`. | +| `options` | `SchemaGeneratorOptions` _(optional)_ | Count, seed, empty rate, and uniqueness options. | + +**Returns:** `Promise>` — resolves to a success result on completion, or a failure with `DIRECTORY_NOT_FOUND` or `FILE_ALREADY_EXISTS`. + +--- + +#### `generateDictionaryFiles` + +Generates records for all schemas in a dictionary with a non-zero count and writes each to a separate file in the output directory, named `.`. Records stream to disk via the `generateDictionaryRecords` generator — parent schema records are always written before child records. + +All expected output file paths are checked before any writing begins. If any file already exists or the directory is missing, the function returns a failure without creating or modifying any files. + +**Parameters** + +| Parameter | Type | Description | +| ------------ | ---------------------------- | ---------------------------------------------------------- | +| `dictionary` | `Dictionary` | The dictionary to generate records for. | +| `outputDir` | `string` | Path to an existing directory where files will be written. | +| `format` | `DataFileFormat` | Column delimiter format: `'tsv'` or `'csv'`. | +| `options` | `DictionaryGeneratorOptions` | Counts per schema, seed, and empty rate — see type below. | + +**Returns:** `Promise>` — resolves to a success result on completion, or a failure with `DIRECTORY_NOT_FOUND` or `FILE_ALREADY_EXISTS`. + +--- + +### API - Types + +#### `FieldGenerator` + +Function type for a field value generator. + +```ts +type FieldGenerator = ( + field: TField, + options?: FieldGeneratorOptions, +) => FieldGeneratorResult; +``` + +| Type parameter | Description | +| -------------- | ---------------------------------------------------------- | +| `TField` | The specific `SchemaField` subtype this generator handles. | + +--- + +#### `FieldGeneratorOptions` + +Options accepted by all field generator functions. + +```ts +type FieldGeneratorOptions = { + seed?: number; + record?: DataRecord; + arrayLength?: number | RestrictionRange; + emptyRate?: number; + excludeValues?: Set; +}; +``` + +| Property | Type | Default | Description | +| --------------- | ---------------------------- | ----------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `seed` | `number` | random | RNG seed for deterministic output. When omitted, a random seed is chosen. | +| `record` | `DataRecord` | `{}` | Partial record context used to resolve conditional restrictions. Fields absent from this record are treated as `undefined` during evaluation. | +| `arrayLength` | `number \| RestrictionRange` | `undefined` | Controls array length for `isArray` fields. A number specifies the exact length; a `RestrictionRange` provides integer bounds to sample from. When omitted, length is chosen randomly between 1 and 3. Ignored for non-array fields. | +| `emptyRate` | `number` | `0.25` | Probability (0–1) that a non-required field returns `undefined` instead of a generated value. Values outside [0, 1] are clamped. Has no effect when the field's active restrictions include `required: true`. | +| `excludeValues` | `Set` | `undefined` | Values the generator must not produce. Used by `generateSchemaRecords` to enforce `unique` field constraints across records. | + +--- + +#### `FieldGeneratorResult` + +Return type of all field generator functions. + +```ts +type FieldGeneratorResult = Result; +``` + +On success, `.data` holds the generated `DataRecordValue` (which may be `undefined` when the empty check fires). On failure, `.data` contains a `FieldGeneratorFailureData` object: + +```ts +type FieldGeneratorFailureData = { + value: DataRecordValue; + conflicts: RestrictionConflict[]; +}; +``` + +| Property | Type | Description | +| ----------- | ----------------------- | ------------------------------------------------------------------------ | +| `value` | `DataRecordValue` | Best-effort fallback value; may not satisfy all restrictions. | +| `conflicts` | `RestrictionConflict[]` | List of restriction pairs that could not be reconciled during reduction. | + +Check `result.success` to narrow the type before accessing `.data`. + +--- + +#### `ForeignKeyPool` + +```ts +type ForeignKeyPool = Map; +``` + +Supplies the set of valid parent rows for each foreign key relationship when generating child records. + +The map is keyed by the **parent schema name** (matching `ForeignKeyRestriction.schema`). Each value is an array of partial `DataRecord` objects — one entry per available parent row. + +Each partial record need only contain the fields named in the foreign key mappings' `foreign` side for the relevant rule. It does not need to be a complete record from the parent schema; any fields not referenced by foreign key mappings on the child schema are ignored. + +**Example:** if the child schema has a foreign key to `"donor"` with mapping `{ local: "donor_id", foreign: "id" }`, the pool entry for `"donor"` must include at least `{ id: someValue }` for each available parent row. + +For composite foreign key rules (multiple mappings in a single `ForeignKeyRestriction`), all mapped local fields are assigned from the **same** selected parent row, preserving relational consistency. + +When a parent schema name has no entry in the map, fields referencing that schema are generated normally — field-level restrictions apply and no foreign key constraint is enforced. + +--- + +#### `RecordGeneratorOptions` + +Options accepted by `generateRecord`. + +```ts +type RecordGeneratorOptions = { + overrides?: DataRecord; + seed?: number; + foreignKeyPool?: ForeignKeyPool; + emptyRate?: number; + fieldExclusions?: Record>; +}; +``` + +| Property | Type | Default | Description | +| ----------------- | -------------------------------------- | ----------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `overrides` | `DataRecord` | `undefined` | Field values to use directly, bypassing generation. Keys not in `schema.fields` are ignored. Takes priority over `foreignKeyPool` values. | +| `seed` | `number` | `undefined` | RNG seed for deterministic output. When omitted, output is non-deterministic. | +| `foreignKeyPool` | `ForeignKeyPool` | `undefined` | Pool of available parent rows for foreign key constrained fields. See `ForeignKeyPool` for the expected structure. | +| `emptyRate` | `number` | `0.25` | Probability (0–1) that any non-required field is left empty (`undefined`). Passed through to each field generator unchanged. See `FieldGeneratorOptions.emptyRate`. | +| `fieldExclusions` | `Record>` | `undefined` | Per-field sets of values the generator must not produce. Used internally by `generateSchemaRecords` to enforce `unique` constraints. | + +--- + +#### `SchemaGeneratorOptions` + +Options accepted by `generateSchemaRecords` and `generateSchemaFile`. + +```ts +type SchemaGeneratorOptions = { + count: number; + seed?: number; + foreignKeyPool?: ForeignKeyPool; + emptyRate?: number; + initialUniqueValues?: { + fields?: Record; + keys?: string[]; + }; +}; +``` + +| Property | Type | Default | Description | +| --------------------- | -------------------- | ------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `count` | `number` | _(required)_ | Number of records to generate. | +| `seed` | `number` | `undefined` | RNG seed for deterministic output. | +| `foreignKeyPool` | `ForeignKeyPool` | `undefined` | Pool of available parent rows for foreign key constrained fields. | +| `emptyRate` | `number` | `0.25` | Probability (0–1) that any non-required field is left empty. | +| `initialUniqueValues` | `{ fields?, keys? }` | `undefined` | Pre-populates uniqueness trackers to avoid collisions with records already written elsewhere. `fields` maps field names to pre-seen values; `keys` is an array of pre-seen serialized `uniqueKey` tuples (`JSON.stringify(keyFieldValues)`). | + +--- + +#### `DictionaryGeneratorOptions` + +Options accepted by `generateDictionaryRecords` and `generateDictionaryFiles`. + +```ts +type DictionaryGeneratorOptions = { + counts: Record; + seed?: number; + emptyRate?: number; +}; +``` + +| Property | Type | Default | Description | +| ----------- | ------------------------ | ------------ | -------------------------------------------------------------------------------------------------- | +| `counts` | `Record` | _(required)_ | Maps schema name to the number of records to generate. Schemas with count 0 or absent are skipped. | +| `seed` | `number` | `undefined` | RNG seed for deterministic output across all generated schemas. | +| `emptyRate` | `number` | `0.25` | Probability (0–1) that any non-required field is left empty. | + +--- + +#### `DictionaryRecord` + +The value yielded by each iteration of `generateDictionaryRecords`. + +```ts +type DictionaryRecord = { + schemaName: string; + record: DataRecord; +}; +``` + +| Property | Type | Description | +| ------------ | ------------ | ----------------------------------------------------- | +| `schemaName` | `string` | The name of the schema this record was generated for. | +| `record` | `DataRecord` | The generated record. | + +--- + +#### `DataFileFormat` + +```ts +type DataFileFormat = 'tsv' | 'csv'; +``` + +Specifies the column delimiter used when writing data files. `'tsv'` uses tab (`\t`); `'csv'` uses comma (`,`). + +--- + +#### `GenerateFileError` + +Discriminated union of failure reasons returned by `generateSchemaFile` and `generateDictionaryFiles`. + +```ts +type GenerateFileError = + | { error: 'DIRECTORY_NOT_FOUND'; directory: string } + | { error: 'FILE_ALREADY_EXISTS'; filePath: string }; +``` + +| Variant | Additional field | Description | +| --------------------- | ---------------- | -------------------------------------------------------------------------- | +| `DIRECTORY_NOT_FOUND` | `directory` | The specified output directory does not exist. | +| `FILE_ALREADY_EXISTS` | `filePath` | A file at the expected output path already exists and was not overwritten. | diff --git a/packages/data-generator/docs/resolving-restrictions.md b/packages/data-generator/docs/resolving-restrictions.md new file mode 100644 index 00000000..ba7c009b --- /dev/null +++ b/packages/data-generator/docs/resolving-restrictions.md @@ -0,0 +1,62 @@ +# Resolving Restrictions for Field Generators + +Field generators attempt to generate a value that will be valid given the restrictions defined for the field in the schema. To accomplish this, the generators act in two phases: first collecting all active restrictions from the field definition, then merging each restriction type down to a set of effective constraints used for generation. + +## Phase 1 - Collecting active restrictions + +A field's restrictions may be a single restriction object, an array of restriction objects, or a mix of plain and conditional objects. + +Conditional restrictions (`if/then/else` blocks) are evaluated against the `record` argument passed to the generator. When a condition passes, the `then` branch is used; otherwise the `else` branch is used. Fields absent from `record` are treated as `undefined`, which typically causes an `exists: true` condition to fail and an `exists: false` condition to pass. + +All active (non-conditional) restriction values are collected by type before generation begins. + +## Phase 2 - Merging restrictions + +Each restriction type is merged independently. + +### `codeList` + +Multiple code lists are intersected. The generated value is drawn from the set of elements present in every active code list. + +- **No conflict:** the intersection is non-empty. +- **Conflict:** the intersection is empty. The generator returns a failure result with a best-effort value drawn from the first code list. + +### `range` (integer and number fields) + +Multiple ranges are intersected to produce the tightest overlapping subrange. The most restrictive bound from each side (lower and upper) is kept. + +- **No conflict:** a valid subrange exists. +- **Conflict:** the merged lower bound exceeds the upper bound, or both bounds are equal but at least one is exclusive. The generator returns a failure result with a best-effort value drawn from the first range. + +### `regex` (string fields) + +Multiple regex patterns are combined using lookahead conjunction so that the generated string must match every pattern. This is always syntactically valid, so no conflict is reported at merge time. Patterns that are semantically incompatible (i.e. can never simultaneously match) are not detected; generation will produce a value that matches the combined expression as best it can. + +### `codeList` + `range` together (integer and number fields) + +When both are present, the code list is filtered to values that fall within the merged range. + +- **No conflict:** at least one code list value satisfies the range. +- **Conflict:** no code list value satisfies the range. The generator returns a failure result with a best-effort value drawn from the unfiltered code list. + +### `codeList` + `regex` together (string fields) + +When both are present, the code list is filtered to values that match the merged regex pattern. + +- **No conflict:** at least one code list value matches the pattern. +- **Conflict:** no code list value matches the pattern. The generator returns a failure result with a best-effort value drawn from the unfiltered code list. + +### `required` and `empty` + +These restrictions do not constrain the generated value itself. However, `required: true` combined with `empty: true` across the active restrictions is a conflict - it is impossible for a field to be both required and empty. The generator returns a failure result but still produces a value. + +## Failure results + +All conflicts produce a failure result rather than throwing an error. The failure includes: + +- A best-effort fallback value that may not satisfy all restrictions. +- A list of conflicts, one per irreconcilable pair of restrictions, each describing the restriction type and the reason the conflict occurred. + +When multiple conflicts occur on the same field (e.g. two incompatible code lists and an incompatible range), all conflicts are collected and returned together in a single failure result. + +Check `result.success` to distinguish success from failure before using the result value. diff --git a/packages/data-generator/package.json b/packages/data-generator/package.json new file mode 100644 index 00000000..d8e03cc7 --- /dev/null +++ b/packages/data-generator/package.json @@ -0,0 +1,23 @@ +{ + "name": "@overture-stack/lectern-data-generator", + "version": "0.0.0", + "description": "Test data generation utilities for Lectern schemas", + "private": true, + "main": "dist/index.js", + "scripts": { + "build": "pnpm nuke:build && tsc -p ./tsconfig.build.json", + "format": "prettier --write .", + "nuke:build": "npx rimraf dist", + "test": "nyc mocha" + }, + "keywords": [], + "author": "Ontario Institute for Cancer Research", + "license": "AGPL-3.0", + "dependencies": { + "@overture-stack/lectern-dictionary": "workspace:^", + "@overture-stack/lectern-validation": "workspace:^" + }, + "devDependencies": { + "fast-check": "^4.9.0" + } +} diff --git a/packages/data-generator/src/common/fileTypes.ts b/packages/data-generator/src/common/fileTypes.ts new file mode 100644 index 00000000..0f77ea78 --- /dev/null +++ b/packages/data-generator/src/common/fileTypes.ts @@ -0,0 +1,14 @@ +/** Column delimiter format for data files. `'tsv'` uses tab; `'csv'` uses comma. */ +export type DataFileFormat = 'tsv' | 'csv'; + +/** Maps each `DataFileFormat` to its column delimiter character. */ +export const COLUMN_DELIMITER = { + tsv: '\t', + csv: ',', +}; + +/** Maps each `DataFileFormat` to its file extension, including the leading dot. */ +export const FILE_EXTENSION = { + tsv: '.tsv', + csv: '.csv', +} as const satisfies Record; diff --git a/packages/data-generator/src/common/hash.ts b/packages/data-generator/src/common/hash.ts new file mode 100644 index 00000000..53bc8d76 --- /dev/null +++ b/packages/data-generator/src/common/hash.ts @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2026 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +// Knuth multiplicative hash constant (2^32 / golden ratio, nearest odd integer). +const KNUTH_MULTIPLIER = 2654435761; + +/** + * Maps `seed` to a well-distributed 32-bit unsigned integer using a Knuth multiplicative hash + * followed by one round of xorshift32. Produces independent draws without fast-check overhead. + * + * Used as the shared primitive for `shouldGenerateEmpty`, `seededIndexInRange`, and + * `deriveRetrySeed` — all of which need a single cheap, seeded, non-colliding hash draw. + */ +export const knuthHash = (seed: number): number => { + let hash = (seed * KNUTH_MULTIPLIER + 1) >>> 0; + hash ^= hash << 13; + hash ^= hash >>> 17; + hash ^= hash << 5; + return hash >>> 0; +}; diff --git a/packages/data-generator/src/dataFile/dataFileGenerator.ts b/packages/data-generator/src/dataFile/dataFileGenerator.ts new file mode 100644 index 00000000..19b8ef0d --- /dev/null +++ b/packages/data-generator/src/dataFile/dataFileGenerator.ts @@ -0,0 +1,157 @@ +/* + * Copyright (c) 2026 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import type { Dictionary, Result, Schema } from '@overture-stack/lectern-dictionary'; +import { failWith, success } from '@overture-stack/lectern-dictionary'; +import { + type DictionaryGeneratorOptions, + generateDictionaryRecords, +} from '../dataGeneration/dictionary/dictionaryGenerator'; +import { type SchemaGeneratorOptions, generateSchemaRecords } from '../dataGeneration/records/schemaGenerator'; +import { closeDataFile, openDataFile, writeRecord } from './dataFileWriter'; +import { FILE_EXTENSION, type DataFileFormat } from '../common/fileTypes'; + +/** Failure reasons returned by `generateSchemaFile` and `generateDictionaryFiles`. */ +export type GenerateFileError = + | { error: 'DIRECTORY_NOT_FOUND'; directory: string } + | { error: 'FILE_ALREADY_EXISTS'; filePath: string }; + +const resolveOutputPath = (outputDir: string, schemaName: string, format: DataFileFormat): string => + path.join(outputDir, schemaName + FILE_EXTENSION[format]); + +const DIRECTORY_NOT_FOUND = 'DIRECTORY_NOT_FOUND' as const; +const FILE_ALREADY_EXISTS = 'FILE_ALREADY_EXISTS' as const; + +const checkDirectory = (outputDir: string): Result => { + if (!fs.existsSync(outputDir) || !fs.statSync(outputDir).isDirectory()) { + return failWith(`Output directory does not exist: ${outputDir}`, { + error: DIRECTORY_NOT_FOUND, + directory: outputDir, + }); + } + return success(undefined); +}; + +const checkFileAbsent = (filePath: string): Result => { + if (fs.existsSync(filePath)) { + return failWith(`File already exists: ${filePath}`, { error: FILE_ALREADY_EXISTS, filePath }); + } + return success(undefined); +}; + +/** + * Generates records for `schema` and writes them to a new file in `outputDir`. + * + * The output file is named `.`. Fails without writing if the directory + * does not exist or if the file already exists. + */ +export const generateSchemaFile = async ( + schema: Schema, + outputDir: string, + format: DataFileFormat, + options?: Omit & { count: number }, +): Promise> => { + const directoryCheck = checkDirectory(outputDir); + if (!directoryCheck.success) { + return directoryCheck; + } + + const filePath = resolveOutputPath(outputDir, schema.name, format); + const fileCheck = checkFileAbsent(filePath); + if (!fileCheck.success) { + return fileCheck; + } + + const handle = await openDataFile(schema, filePath, format); + try { + for (const record of generateSchemaRecords(schema, options)) { + const writeResult = await writeRecord(handle, record); + if (!writeResult.success) { + throw new Error(`Failed to write record: ${writeResult.data.error}`); + } + } + } finally { + await closeDataFile(handle); + } + + return success(undefined); +}; + +/** + * Generates records for all schemas in `dictionary` with a non-zero count and writes each to a + * separate file in `outputDir`, named `.`. + * + * All output file paths are checked before any writing begins. Fails without writing any files + * if the directory does not exist or if any expected output file already exists. + */ +export const generateDictionaryFiles = async ( + dictionary: Dictionary, + outputDir: string, + format: DataFileFormat, + options: DictionaryGeneratorOptions, +): Promise> => { + const directoryCheck = checkDirectory(outputDir); + if (!directoryCheck.success) { + return directoryCheck; + } + + const includedSchemaNames = Object.entries(options.counts) + .filter(([, count]) => count > 0) + .map(([name]) => name); + + for (const schemaName of includedSchemaNames) { + const filePath = resolveOutputPath(outputDir, schemaName, format); + const fileCheck = checkFileAbsent(filePath); + if (!fileCheck.success) { + return fileCheck; + } + } + + const schemaByName = new Map(dictionary.schemas.map((schema) => [schema.name, schema])); + const handles = new Map>>(); + + for (const schemaName of includedSchemaNames) { + const schema = schemaByName.get(schemaName); + if (schema === undefined) { + continue; + } + const filePath = resolveOutputPath(outputDir, schemaName, format); + handles.set(schemaName, await openDataFile(schema, filePath, format)); + } + + try { + for (const { schemaName, record } of generateDictionaryRecords(dictionary, options)) { + const handle = handles.get(schemaName); + if (handle !== undefined) { + const writeResult = await writeRecord(handle, record); + if (!writeResult.success) { + throw new Error(`Failed to write record for schema '${schemaName}': ${writeResult.data.error}`); + } + } + } + } finally { + for (const handle of handles.values()) { + await closeDataFile(handle); + } + } + + return success(undefined); +}; diff --git a/packages/data-generator/src/dataFile/dataFileWriter.ts b/packages/data-generator/src/dataFile/dataFileWriter.ts new file mode 100644 index 00000000..0e7ea964 --- /dev/null +++ b/packages/data-generator/src/dataFile/dataFileWriter.ts @@ -0,0 +1,150 @@ +/* + * Copyright (c) 2026 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import fs from 'node:fs'; +import type { DataRecord, DataRecordValue, Result, Schema, SchemaField } from '@overture-stack/lectern-dictionary'; +import { DEFAULT_DELIMITER, failWith, success } from '@overture-stack/lectern-dictionary'; +import { COLUMN_DELIMITER, type DataFileFormat } from '../common/fileTypes'; + +const STREAM_CLOSED = 'STREAM_CLOSED' as const; + +/** Failure data returned by `writeRecord` when the file handle has already been closed. */ +export type WriteRecordError = { error: typeof STREAM_CLOSED }; + +/** + * An open handle to a data file being written. Created by `openDataFile`, `openTsvFile`, or + * `openCsvFile`. Pass to `writeRecord` to append rows and to `closeDataFile` when done. + * + * The underlying write stream is managed internally; do not retain references beyond the + * lifetime of the file handle. + */ +export type DataFileHandle = { + readonly schema: Schema; + readonly format: DataFileFormat; +}; + +const streamRegistry = new Map(); + +const serializeValue = (value: DataRecordValue, field: SchemaField): string => { + if (value === undefined) { + return ''; + } + if (Array.isArray(value)) { + return value.map(String).join(field.delimiter ?? DEFAULT_DELIMITER); + } + return String(value); +}; + +const serializeRecord = (record: DataRecord, schema: Schema, columnDelimiter: string): string => { + const values = schema.fields.map((field) => serializeValue(record[field.name], field)); + return values.join(columnDelimiter) + '\n'; +}; + +const writeToStream = (stream: fs.WriteStream, data: string): Promise => + new Promise((resolve, reject) => { + // Attach the error listener before calling write so that errors emitted synchronously + // or before the write callback fires are not missed. + stream.once('error', reject); + const canContinue = stream.write(data, (writeError) => { + if (writeError !== undefined && writeError !== null) { + reject(writeError); + } + }); + if (canContinue) { + stream.off('error', reject); + resolve(); + } else { + stream.once('drain', () => { + stream.off('error', reject); + resolve(); + }); + } + }); + +/** + * Opens a new data file for writing and writes the header row. The header columns are the + * field names from `schema.fields` in definition order, separated by the format delimiter. + * + * Returns a `DataFileHandle` that must be passed to `writeRecord` and `closeDataFile`. + * Always call `closeDataFile` when done to flush and close the underlying stream. + */ +export const openDataFile = async ( + schema: Schema, + filePath: string, + format: DataFileFormat, +): Promise => { + const stream = fs.createWriteStream(filePath); + const columnDelimiter = COLUMN_DELIMITER[format]; + + const header = schema.fields.map((field) => field.name).join(columnDelimiter) + '\n'; + await writeToStream(stream, header); + + const handle: DataFileHandle = { schema, format }; + streamRegistry.set(handle, stream); + return handle; +}; + +/** Convenience wrapper for `openDataFile` that opens the file in TSV (tab-separated) format. */ +export const openTsvFile = (schema: Schema, filePath: string): Promise => + openDataFile(schema, filePath, 'tsv'); + +/** Convenience wrapper for `openDataFile` that opens the file in CSV (comma-separated) format. */ +export const openCsvFile = (schema: Schema, filePath: string): Promise => + openDataFile(schema, filePath, 'csv'); + +/** + * Serializes `record` and appends it as a row to the file associated with `handle`. + * + * Fields absent from `record`are written as empty strings. Fields present in `record` + * but absent from the schema are ignored. Array field values are joined with + * `field.delimiter` (falling back to `DEFAULT_DELIMITER`) — distinct from the column + * delimiter set by the file format. + * + * Returns a failure with `{ error: 'STREAM_CLOSED' }` if `handle` has already been closed. + */ +export const writeRecord = async ( + handle: DataFileHandle, + record: DataRecord, +): Promise> => { + const stream = streamRegistry.get(handle); + if (stream === undefined) { + return failWith('Cannot write to a closed file handle.', { error: STREAM_CLOSED }); + } + const columnDelimiter = COLUMN_DELIMITER[handle.format]; + const row = serializeRecord(record, handle.schema, columnDelimiter); + await writeToStream(stream, row); + return success(undefined); +}; + +/** + * Flushes and closes the file associated with `handle`. Resolves once the underlying stream + * has finished writing. Calling `closeDataFile` on an already-closed handle is a no-op. + */ +export const closeDataFile = (handle: DataFileHandle): Promise => { + const stream = streamRegistry.get(handle); + if (stream === undefined) { + return Promise.resolve(); + } + streamRegistry.delete(handle); + return new Promise((resolve, reject) => { + stream.on('finish', resolve); + stream.on('error', reject); + stream.end(); + }); +}; diff --git a/packages/data-generator/src/dataGeneration/dictionary/dictionaryGenerator.ts b/packages/data-generator/src/dataGeneration/dictionary/dictionaryGenerator.ts new file mode 100644 index 00000000..66336150 --- /dev/null +++ b/packages/data-generator/src/dataGeneration/dictionary/dictionaryGenerator.ts @@ -0,0 +1,193 @@ +/* + * Copyright (c) 2026 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import type { DataRecord, Dictionary, Schema } from '@overture-stack/lectern-dictionary'; +import { type ForeignKeyPool } from '../records/recordGenerator'; +import { generateSchemaRecords } from '../records/schemaGenerator'; + +/** Maps schema name to the number of records to generate. Schemas absent from this map or with count 0 are skipped. */ +export type DictionarySchemaCount = Record; + +/** Options for `generateDictionaryRecords`. */ +export type DictionaryGeneratorOptions = { + counts: DictionarySchemaCount; + seed?: number; + emptyRate?: number; +}; + +/** A single record yielded by `generateDictionaryRecords`, tagged with its originating schema name. */ +export type DictionaryRecord = { + schemaName: string; + record: DataRecord; +}; + +// Builds a tier-ordered list of schema names using Kahn's algorithm on FK edges. +// Child schemas depend on parent schemas (FK target must be generated first). +// Schemas not in `includedNames` are excluded from the graph entirely. +const resolveSchemaGenerationOrder = (schemas: Schema[], includedNames: Set): string[][] => { + const included = schemas.filter((schema) => includedNames.has(schema.name)); + + const inDegree = new Map(included.map((schema) => [schema.name, 0])); + // dependents[parent] = set of child schema names that depend on parent + const dependents = new Map>(included.map((schema) => [schema.name, new Set()])); + + for (const schema of included) { + for (const fkRule of schema.restrictions?.foreignKey ?? []) { + if (!includedNames.has(fkRule.schema)) { + continue; + } + inDegree.set(schema.name, (inDegree.get(schema.name) ?? 0) + 1); + dependents.get(fkRule.schema)?.add(schema.name); + } + } + + const readyQueue = included.map((schema) => schema.name).filter((name) => (inDegree.get(name) ?? 0) === 0); + const processed = new Set(); + const order: string[][] = []; + + while (processed.size < included.length) { + if (readyQueue.length === 0) { + // Cycle among remaining schemas — place all remaining in one tier. + const cyclic = included.map((schema) => schema.name).filter((name) => !processed.has(name)); + order.push(cyclic); + break; + } + + const currentTier = readyQueue.splice(0); + order.push(currentTier); + + for (const schemaName of currentTier) { + processed.add(schemaName); + for (const dependent of dependents.get(schemaName) ?? new Set()) { + const newDegree = (inDegree.get(dependent) ?? 1) - 1; + inDegree.set(dependent, newDegree); + if (newDegree === 0) { + readyQueue.push(dependent); + } + } + } + } + + return order; +}; + +// Projects each record down to only the foreign fields referenced by child schemas pointing at `parentSchemaName`. +const extractFkPool = (parentSchemaName: string, records: DataRecord[], childSchemas: Schema[]): DataRecord[] => { + const foreignFieldNames = new Set(); + for (const childSchema of childSchemas) { + for (const fkRule of childSchema.restrictions?.foreignKey ?? []) { + if (fkRule.schema === parentSchemaName) { + for (const mapping of fkRule.mappings) { + foreignFieldNames.add(mapping.foreign); + } + } + } + } + + return records.map((record) => { + const projected: DataRecord = {}; + for (const fieldName of foreignFieldNames) { + if (Object.hasOwn(record, fieldName)) { + projected[fieldName] = record[fieldName]; + } + // If the field is absent from the record (e.g. generated as undefined and not set), + // it is omitted from the pool entry. Child records that draw from this pool will find + // no value for that mapping and fall back to unconstrained generation for that field. + } + return projected; + }); +}; + +/** + * Lazily generates records for all schemas in `dictionary` that have a non-zero count in + * `options.counts`, yielding one `DictionaryRecord` at a time. + * + * Schemas are generated in FK dependency order. Parent schemas are fully generated and held in + * memory as a FK pool before any child records are yielded, ensuring child FK fields always + * reference valid parent values. Child records are streamed out one at a time and not retained. + */ +export function* generateDictionaryRecords( + dictionary: Dictionary, + options: DictionaryGeneratorOptions, +): Generator { + const { counts, seed, emptyRate } = options; + + const includedNames = new Set( + Object.entries(counts) + .filter(([, count]) => count > 0) + .map(([name]) => name), + ); + + const schemaByName = new Map(dictionary.schemas.map((schema) => [schema.name, schema])); + const generationOrder = resolveSchemaGenerationOrder(dictionary.schemas, includedNames); + + // Assign a stable index to each schema in generation order for deterministic per-schema seeds. + const schemaGenerationIndex = new Map(); + let generationIndex = 0; + for (const tier of generationOrder) { + for (const schemaName of tier) { + schemaGenerationIndex.set(schemaName, generationIndex++); + } + } + + const foreignKeyPool: ForeignKeyPool = new Map(); + + // Pre-compute which included schemas have at least one included child schema depending on them. + // Only schemas with included dependents need their records collected into the FK pool. + const schemasWithDependents = new Set(); + for (const schema of dictionary.schemas) { + if (!includedNames.has(schema.name)) { + continue; + } + for (const fkRule of schema.restrictions?.foreignKey ?? []) { + if (includedNames.has(fkRule.schema)) { + schemasWithDependents.add(fkRule.schema); + } + } + } + + for (const tier of generationOrder) { + for (const schemaName of tier) { + const schema = schemaByName.get(schemaName); + if (schema === undefined) { + continue; + } + + const count = counts[schemaName] ?? 0; + const schemaIndex = schemaGenerationIndex.get(schemaName) ?? 0; + const schemaSeed = seed !== undefined ? seed + schemaIndex : undefined; + const schemaGenerator = generateSchemaRecords(schema, { count, seed: schemaSeed, foreignKeyPool, emptyRate }); + + if (schemasWithDependents.has(schemaName)) { + // Collect fully into the FK pool before yielding, so child schemas can reference these records. + const records = [...schemaGenerator]; + const poolEntry = extractFkPool(schemaName, records, dictionary.schemas); + foreignKeyPool.set(schemaName, poolEntry); + for (const record of records) { + yield { schemaName, record }; + } + } else { + // No children depend on this schema — stream records out directly without collecting. + for (const record of schemaGenerator) { + yield { schemaName, record }; + } + } + } + } +} diff --git a/packages/data-generator/src/dataGeneration/fields/fieldGenerators.ts b/packages/data-generator/src/dataGeneration/fields/fieldGenerators.ts new file mode 100644 index 00000000..c4490661 --- /dev/null +++ b/packages/data-generator/src/dataGeneration/fields/fieldGenerators.ts @@ -0,0 +1,568 @@ +/* + * Copyright (c) 2026 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { + failWith, + success, + type DataRecord, + type DataRecordValue, + type RestrictionRange, + type Result, + type SchemaBooleanField, + type SchemaField, + type SchemaIntegerField, + type SchemaNumberField, + type SchemaStringField, + type SingleDataValue, +} from '@overture-stack/lectern-dictionary'; +import fc from 'fast-check'; +import { knuthHash } from '../../common/hash'; +import { collectRestrictions, type CollectedRestrictions } from './resolveRestrictions'; +import { + filterCodeListByRange, + filterCodeListByRegex, + reduceCodeLists, + reduceEmpty, + reduceRanges, + reduceRegex, + reduceRequired, + type RestrictionConflict, +} from './restrictionReducers'; + +/* ************************** * + * Result Types * + * ************************** */ + +/** + * Payload carried in the failure case of a `FieldGeneratorResult`. The generated value is still + * present - it was produced using the non-conflicting subset of restrictions and will not satisfy + * all restrictions. The `conflicts` array describes each pair of restrictions that could not be + * reconciled. + */ +export type FieldGeneratorFailureData = { + value: TValue; + conflicts: RestrictionConflict[]; +}; + +/** + * Return type of all field generator functions. + * + * - Success: the generated value satisfies all active restrictions. + * - Failure: one or more restrictions are in conflict and cannot be satisfied simultaneously. + * The failure `.data.value` is still a usable value generated from the non-conflicting subset, + * but it will not pass validation. The failure `.data.conflicts` describes what conflicted. + */ +export type FieldGeneratorResult = Result< + TValue, + FieldGeneratorFailureData +>; + +/** + * Options accepted by all field generator functions. + * + * `seed` controls the RNG - the same field, seed, and record always produce the same output. + * If omitted, a random seed is used. + * + * `record` provides already-generated sibling field values used to evaluate conditional + * restrictions. Fields absent from `record` are treated as undefined. Defaults to an empty record. + * + * `arrayLength` controls the number of elements generated for array fields (`field.isArray === true`). + * Ignored for non-array fields. May be a fixed number or a `RestrictionRange` used to generate + * the length as an integer within those bounds. If omitted, the length is 1–3. + * + * `emptyRate` is the probability (0–1) that a non-required field returns `undefined` instead of a + * generated value. Values outside [0, 1] are clamped. Defaults to `0.25`. Has no effect when the + * field's active restrictions include `required: true`. + * + * `excludeValues` is an optional set of values that the generator must not produce. When provided, + * the underlying `fast-check` arbitrary is filtered with `fc.filter` to reject excluded values + * before sampling. If the arbitrary cannot find a non-excluded value within its retry limit, a + * best-effort fallback value is returned (same failure path as a restriction conflict). + */ +export type FieldGeneratorOptions = { + seed?: number; + record?: DataRecord; + arrayLength?: number | RestrictionRange; + emptyRate?: number; + excludeValues?: Set; +}; + +/** + * Function signature for a field value generator. Accepts a schema field definition and an optional + * `FieldGeneratorOptions` object. Returns a `FieldGeneratorResult`. + * + * On success, `result.data` is the generated value. On failure (conflicting restrictions), + * `result.data.value` is a best-effort value and `result.data.conflicts` lists the conflicts. + */ +export type FieldGenerator = ( + field: TField, + options?: FieldGeneratorOptions, +) => FieldGeneratorResult; + +/* ************************** * + * Internal Helpers * + * ************************** */ + +const DEFAULT_ARRAY_MIN = 1; +const DEFAULT_ARRAY_MAX = 3; + +const DEFAULT_EMPTY_RATE = 0.25; + +/* + * A ReferenceTag is a string starting with `#/`. When a codeList or regex contains one it means the + * dictionary still has unresolved references. Generators skip these entries and use only concrete values. + */ +const isReferenceTag = (value: string): boolean => value.startsWith('#/'); + +const randomSeed = (): number => Math.floor(Math.random() * 2 ** 32); + +/** + * Draws a single value from a `fast-check` arbitrary using a deterministic seed. + * + * Wraps `fc.sample` with `numRuns: 1` and unwraps the result. Throws if `fast-check` produces no + * value, which should not occur for well-formed arbitraries but is guarded against explicitly + * because the return type of `fc.sample` does not exclude empty arrays. + * + * When `excludeValues` is provided, the arbitrary is filtered with `fc.filter` before sampling. + * fast-check retries internally up to its default max-tries limit; if all retries are exhausted + * the sample still returns the first candidate (which may be in the exclusion set). This is + * intentional: the generator falls back gracefully rather than throwing. + */ +const sampleFCGenerator = (arbitrary: fc.Arbitrary, seed: number, excludeValues?: Set): T => { + const filtered = + excludeValues !== undefined && excludeValues.size > 0 ? + arbitrary.filter((value) => !excludeValues.has(value as DataRecordValue)) + : arbitrary; + const [value] = fc.sample(filtered, { seed, numRuns: 1 }); + if (value === undefined) { + throw new Error('fast-check sample produced no value'); + } + return value; +}; + +// Maps the hash to [0, 1) and compares against the threshold to decide whether to emit undefined. +const shouldGenerateEmpty = (seed: number, emptyRate: number): boolean => knuthHash(seed) / 4294967296 < emptyRate; + +const resolveArrayLength = (arrayLength: number | RestrictionRange | undefined, seed: number): number => { + if (arrayLength === undefined) { + return sampleFCGenerator(fc.integer({ min: DEFAULT_ARRAY_MIN, max: DEFAULT_ARRAY_MAX }), seed); + } + if (typeof arrayLength === 'number') { + return arrayLength; + } + const min = + arrayLength.min ?? + (arrayLength.exclusiveMin !== undefined ? Math.floor(arrayLength.exclusiveMin) + 1 : DEFAULT_ARRAY_MIN); + const max = + arrayLength.max ?? + (arrayLength.exclusiveMax !== undefined ? Math.ceil(arrayLength.exclusiveMax) - 1 : DEFAULT_ARRAY_MAX); + if (min > max) { + return DEFAULT_ARRAY_MIN; + } + return sampleFCGenerator(fc.integer({ min, max }), seed); +}; + +const extractConflicts = (results: Array>): RestrictionConflict[] => + results.flatMap((result) => (result.success ? [] : [result.data])); + +/** + * Returns a `RestrictionConflict` when `required: true` and `empty: true` are both active, or + * `undefined` when the combination is valid. The same conflict applies to all field value types. + */ +const buildRequiredEmptyConflict = (required: boolean, empty: boolean): RestrictionConflict | undefined => { + if (required && empty) { + return { + type: 'required-empty' as const, + values: [required, empty], + reason: 'required:true and empty:true cannot both be satisfied', + }; + } + return undefined; +}; + +type ResolvedNumericConstraints = { + codeList: number[] | undefined; + fallbackRange: RestrictionRange | undefined; + conflicts: RestrictionConflict[]; +}; + +type ResolvedStringConstraints = { + codeList: string[] | undefined; + regexPattern: string | undefined; + conflicts: RestrictionConflict[]; +}; + +/** + * Resolves numeric field constraints from `collected` restrictions into the values needed by + * `generateSingle`. Strips reference tags from codeLists, reduces multiple codeLists to their + * intersection and multiple ranges to their tightest overlap, then cross-filters the codeList + * against the merged range. Returns `undefined` for each output that has no active restriction. + * `conflicts` includes reducer failures and any cross-type conflict where no codeList value + * satisfies the range. When the cross-type conflict fires, `codeList` falls back to the + * unfiltered list so generation can still produce a best-effort value. + */ +const resolveNumericConstraints = (codeLists: number[][], ranges: RestrictionRange[]): ResolvedNumericConstraints => { + // ---- Ranges + const rangeResult = reduceRanges(ranges); + const validatedRange = rangeResult.success ? rangeResult.data : undefined; + const fallbackRange = (rangeResult.success ? rangeResult.data : ranges[0]) ?? undefined; + + // ---- Code lists + const codeListResult = reduceCodeLists(codeLists); + const unrefinedCodeList = (codeListResult.success ? codeListResult.data : codeLists[0]) ?? undefined; + const rangeFilteredList = + unrefinedCodeList !== undefined ? filterCodeListByRange(unrefinedCodeList, validatedRange) : undefined; + + const crossTypeConflict = rangeFilteredList !== undefined && rangeFilteredList.length === 0; + const conflicts = extractConflicts([codeListResult, rangeResult]); + if (crossTypeConflict) { + conflicts.push({ + type: 'codeList', + values: [unrefinedCodeList, validatedRange], + reason: 'No value in codeList satisfies the range restriction.', + }); + } + + return { + codeList: crossTypeConflict ? unrefinedCodeList : (rangeFilteredList ?? unrefinedCodeList), + fallbackRange, + conflicts, + }; +}; + +/** + * Resolves string field constraints from `collected` restrictions into the values needed by + * `generateSingle`. Strips reference tags from codeLists and regex entries, reduces multiple + * codeLists to their intersection and multiple regex patterns to a lookahead conjunction, then + * cross-filters the codeList against the merged regex. Returns `undefined` for each output that + * has no active restriction. `conflicts` includes reducer failures and any cross-type conflict + * where no codeList value satisfies the regex. When the cross-type conflict fires, `codeList` + * falls back to the unfiltered list so generation can still produce a best-effort value. + */ +const resolveStringConstraints = ( + codeLists: string[][], + regex: CollectedRestrictions['regex'], +): ResolvedStringConstraints => { + // ---- Regex + const concreteRegex = regex.filter((entry) => (typeof entry === 'string' ? !isReferenceTag(entry) : true)); + const regexResult = reduceRegex(concreteRegex); + const mergedRegex = regexResult.success ? regexResult.data : undefined; + const regexPattern = mergedRegex; + + // ---- Code lists + const codeListResult = reduceCodeLists(codeLists); + const unrefinedCodeList = (codeListResult.success ? codeListResult.data : codeLists[0]) ?? undefined; + const regexFilteredList = + unrefinedCodeList !== undefined ? filterCodeListByRegex(unrefinedCodeList, mergedRegex) : undefined; + + const crossTypeConflict = regexFilteredList !== undefined && regexFilteredList.length === 0; + const conflicts = extractConflicts([codeListResult, regexResult]); + if (crossTypeConflict) { + conflicts.push({ + type: 'codeList', + values: [unrefinedCodeList, mergedRegex], + reason: 'No value in codeList satisfies the regex restriction.', + }); + } + + return { + codeList: crossTypeConflict ? unrefinedCodeList : (regexFilteredList ?? unrefinedCodeList), + regexPattern, + conflicts, + }; +}; + +/** + * Dispatches to `generateSingle` once for scalar fields, or multiple times for array fields. + * + * When `isArray` is `true`, the array length is resolved from `arrayLength` (exact count, range + * bounds, or default 1–3). Each element is generated with a unique derived seed (`seed + index + 1`) + * so that elements within the same array are distinct while the full array remains reproducible. + * + * If any element generation returns a failure, all element conflicts are collected and the function + * returns a single failure result whose value is the array of best-effort element values. + */ +const wrapArrayIfNeeded = ( + generateSingle: (elementSeed: number) => FieldGeneratorResult, + isArray: boolean | undefined, + seed: number, + arrayLength: number | RestrictionRange | undefined, +): FieldGeneratorResult => { + if (!isArray) { + return generateSingle(seed); + } + + const count = resolveArrayLength(arrayLength, seed); + const results = Array.from({ length: count }, (_, index) => generateSingle(seed + index + 1)); + + const allConflicts = results.flatMap((result) => (result.success ? [] : result.data.conflicts)); + + // Type Assertion Justification: + // Each element is TElement (boolean | number | string). The array is homogeneous at runtime + // because each generator passes a concrete type (e.g. boolean, number, string), making the + // resulting TElement[] a valid boolean[] | number[] | string[]. TypeScript cannot prove this + // from the generic bound alone, so we assert to DataRecordValue here. + const values = results.map((result) => (result.success ? result.data : result.data.value)) as DataRecordValue; + + if (allConflicts.length > 0) { + return failWith('Array element generation encountered conflicting restrictions.', { + value: values, + conflicts: allConflicts, + }); + } + return success(values); +}; + +/* ************************** * + * Boolean Generator * + * ************************** */ + +/** + * Generates a valid value for a `SchemaBooleanField`. + * + * Returns `FieldGeneratorResult`. On success, `result.data` is a `boolean`, or a `boolean[]` when + * `field.isArray` is `true`. Array length is controlled by `options.arrayLength`; defaults to 1–3. + * + * Active restrictions are resolved from `field.restrictions` using the provided `record` to evaluate + * any conditional branches. `required` and `empty` restrictions do not constrain the generated value, + * but `required: true` combined with `empty: true` is a conflict that produces a failure result. + * See `docs/resolving-restrictions.md` for full details. + */ +export const generateBooleanValue: FieldGenerator = ( + field, + options = {}, +): FieldGeneratorResult => { + const { seed = randomSeed(), record = {}, arrayLength, emptyRate, excludeValues } = options; + const collected = collectRestrictions(field.restrictions, record); + const required = reduceRequired(collected.required); + const empty = reduceEmpty(collected.empty); + + const resolvedEmptyRate = Math.min(1, Math.max(0, emptyRate ?? DEFAULT_EMPTY_RATE)); + if (!required && shouldGenerateEmpty(seed, resolvedEmptyRate)) { + return success(undefined); + } + + const requiredEmptyConflict = buildRequiredEmptyConflict(required, empty); + + // Generates one value for a scalar field or one element of an array field; called by wrapArrayIfNeeded. + const generateSingle = (elementSeed: number): FieldGeneratorResult => { + const value = sampleFCGenerator(fc.boolean(), elementSeed, excludeValues); + if (requiredEmptyConflict !== undefined) { + return failWith('Field has conflicting required:true and empty:true restrictions.', { + value, + conflicts: [requiredEmptyConflict], + }); + } + return success(value); + }; + + return wrapArrayIfNeeded(generateSingle, field.isArray, seed, arrayLength); +}; + +/* ************************** * + * Numeric Generator (shared) * + * ************************** */ + +const INTEGER_FALLBACK_MIN = Number.MIN_SAFE_INTEGER; +const INTEGER_FALLBACK_MAX = Number.MAX_SAFE_INTEGER; + +const NUMBER_FALLBACK_MIN = -1_000_000; +const NUMBER_FALLBACK_MAX = 1_000_000; + +const integerFromRange = (range: RestrictionRange | undefined, seed: number): number => { + const min = range?.min ?? (range?.exclusiveMin !== undefined ? range.exclusiveMin + 1 : INTEGER_FALLBACK_MIN); + const max = range?.max ?? (range?.exclusiveMax !== undefined ? range.exclusiveMax - 1 : INTEGER_FALLBACK_MAX); + return sampleFCGenerator(fc.integer({ min, max }), seed); +}; + +const numberFromRange = (range: RestrictionRange | undefined, seed: number): number => { + const min = Math.fround(range?.min ?? range?.exclusiveMin ?? NUMBER_FALLBACK_MIN); + const max = Math.fround(range?.max ?? range?.exclusiveMax ?? NUMBER_FALLBACK_MAX); + const minExcluded = range?.exclusiveMin !== undefined; + const maxExcluded = range?.exclusiveMax !== undefined; + return sampleFCGenerator(fc.float({ min, max, minExcluded, maxExcluded, noNaN: true }), seed); +}; + +const generateNumericValue = ( + field: SchemaIntegerField | SchemaNumberField, + options: FieldGeneratorOptions, + fromRange: (range: RestrictionRange | undefined, seed: number) => number, +): FieldGeneratorResult => { + const { seed = randomSeed(), record = {}, arrayLength, emptyRate, excludeValues } = options; + const collected = collectRestrictions(field.restrictions, record); + const required = reduceRequired(collected.required); + const empty = reduceEmpty(collected.empty); + + const resolvedEmptyRate = Math.min(1, Math.max(0, emptyRate ?? DEFAULT_EMPTY_RATE)); + if (!required && shouldGenerateEmpty(seed, resolvedEmptyRate)) { + return success(undefined); + } + // Filter each code list to only numeric values. We expect it to only contain reference tags and numbers, so this will clear unused reference tags. + const numericCodeLists = collected.codeList + .map((list) => list.filter((entry): entry is number => typeof entry === 'number')) + .filter((list) => list.length > 0); + const { + codeList, + fallbackRange, + conflicts: constraintConflicts, + } = resolveNumericConstraints(numericCodeLists, collected.range); + const requiredEmptyConflict = buildRequiredEmptyConflict(required, empty); + const conflicts = + requiredEmptyConflict !== undefined ? [...constraintConflicts, requiredEmptyConflict] : constraintConflicts; + + // Generates one value for a scalar field or one element of an array field; called by wrapArrayIfNeeded. + const generateSingle = (elementSeed: number): FieldGeneratorResult => { + if (codeList !== undefined && codeList.length > 0) { + const value = sampleFCGenerator(fc.constantFrom(...codeList), elementSeed, excludeValues); + return conflicts.length > 0 ? + failWith('Conflicting restrictions; value generated from merged codeList.', { + value, + conflicts, + }) + : success(value); + } + + const value = fromRange(fallbackRange, elementSeed); + return conflicts.length > 0 ? + failWith('Conflicting restrictions; value generated from fallback range.', { + value, + conflicts, + }) + : success(value); + }; + + return wrapArrayIfNeeded(generateSingle, field.isArray, seed, arrayLength); +}; + +/* ************************** * + * Integer Generator * + * ************************** */ + +/** + * Generates a valid value for a `SchemaIntegerField`. + * + * Returns `FieldGeneratorResult`. On success, `result.data` is an integer `number`, or a `number[]` + * of integers when `field.isArray` is `true`. Array length is controlled by `options.arrayLength`; + * defaults to 1–3. + * + * Active restrictions are resolved from `field.restrictions` using the provided `record` to evaluate + * any conditional branches, then merged across all active `codeList` and `range` entries before + * generation. Contradictory restrictions produce a failure result with a best-effort fallback value. + * See `docs/resolving-restrictions.md` for full details. + */ +export const generateIntegerValue: FieldGenerator = ( + field, + options = {}, +): FieldGeneratorResult => generateNumericValue(field, options, integerFromRange); + +/* ************************** * + * Number Generator * + * ************************** */ + +/** + * Generates a valid value for a `SchemaNumberField`. + * + * Returns `FieldGeneratorResult`. On success, `result.data` is a `number` (may be floating-point), + * or a `number[]` when `field.isArray` is `true`. Array length is controlled by `options.arrayLength`; + * defaults to 1–3. + * + * Active restrictions are resolved from `field.restrictions` using the provided `record` to evaluate + * any conditional branches, then merged across all active `codeList` and `range` entries before + * generation. Contradictory restrictions produce a failure result with a best-effort fallback value. + * See `docs/resolving-restrictions.md` for full details. + */ +export const generateNumberValue: FieldGenerator = ( + field, + options = {}, +): FieldGeneratorResult => generateNumericValue(field, options, numberFromRange); + +/* ************************** * + * String Generator * + * ************************** */ + +/** + * Generates a valid value for a `SchemaStringField`. + * + * Returns `FieldGeneratorResult`. On success, `result.data` is a `string`, or a `string[]` when + * `field.isArray` is `true`. Array length is controlled by `options.arrayLength`; defaults to 1–3. + * + * Active restrictions are resolved from `field.restrictions` using the provided `record` to evaluate + * any conditional branches, then merged across all active `codeList` and `regex` entries before + * generation. Contradictory restrictions produce a failure result with a best-effort fallback value. + * See `docs/resolving-restrictions.md` for full details. + */ +export const generateStringValue: FieldGenerator = ( + field, + options = {}, +): FieldGeneratorResult => { + const { seed = randomSeed(), record = {}, arrayLength, emptyRate, excludeValues } = options; + const collected = collectRestrictions(field.restrictions, record); + const required = reduceRequired(collected.required); + const empty = reduceEmpty(collected.empty); + + const resolvedEmptyRate = Math.min(1, Math.max(0, emptyRate ?? DEFAULT_EMPTY_RATE)); + if (!required && shouldGenerateEmpty(seed, resolvedEmptyRate)) { + return success(undefined); + } + // Filter string lists to only include string values, and remove reference tags. We only expect string values but the types are permissive to support numeric code lists, this filters out that edge case. + const stringCodeLists = collected.codeList + .map((list) => list.filter((entry): entry is string => typeof entry === 'string' && !isReferenceTag(entry))) + .filter((list) => list.length > 0); + const { + codeList, + regexPattern, + conflicts: constraintConflicts, + } = resolveStringConstraints(stringCodeLists, collected.regex); + const requiredEmptyConflict = buildRequiredEmptyConflict(required, empty); + const conflicts = + requiredEmptyConflict !== undefined ? [...constraintConflicts, requiredEmptyConflict] : constraintConflicts; + + // Generates one value for a scalar field or one element of an array field; called by wrapArrayIfNeeded. + const generateSingle = (elementSeed: number): FieldGeneratorResult => { + if (codeList !== undefined && codeList.length > 0) { + const value = sampleFCGenerator(fc.constantFrom(...codeList), elementSeed, excludeValues); + return conflicts.length > 0 ? + failWith('Conflicting restrictions; value generated from merged codeList.', { + value, + conflicts, + }) + : success(value); + } + + if (regexPattern !== undefined) { + const value = sampleFCGenerator(fc.stringMatching(new RegExp(regexPattern)), elementSeed, excludeValues); + return conflicts.length > 0 ? + failWith('Conflicting restrictions; value generated from regex.', { + value, + conflicts, + }) + : success(value); + } + + const value = sampleFCGenerator(fc.string({ minLength: 1, maxLength: 20 }), elementSeed, excludeValues); + return conflicts.length > 0 ? + failWith('Conflicting restrictions; value generated without restrictions.', { + value, + conflicts, + }) + : success(value); + }; + + return wrapArrayIfNeeded(generateSingle, field.isArray, seed, arrayLength); +}; diff --git a/packages/data-generator/src/dataGeneration/fields/resolveRestrictions.ts b/packages/data-generator/src/dataGeneration/fields/resolveRestrictions.ts new file mode 100644 index 00000000..1ef658a3 --- /dev/null +++ b/packages/data-generator/src/dataGeneration/fields/resolveRestrictions.ts @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2026 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { + isConditionalRestriction, + type ConditionalRestriction, + type DataRecord, + type RestrictionRange, + type RestrictionRegex, +} from '@overture-stack/lectern-dictionary'; +import { testConditionalRestriction } from '@overture-stack/lectern-validation'; + +/* ************************** * + * Collected Restrictions * + * ************************** */ + +/** + * All active (non-conditional) restriction values collected by type, after resolving all conditional + * branches. Each array contains one entry per active restriction object that specified that key. + */ +export type CollectedRestrictions = { + codeList: (string | number)[][]; + empty: boolean[]; + range: RestrictionRange[]; + regex: RestrictionRegex[]; + required: boolean[]; +}; + +type AnyRestrictionObject = { + codeList?: unknown; + empty?: boolean; + range?: RestrictionRange; + regex?: RestrictionRegex; + required?: boolean; +}; + +const emptyCollected = (): CollectedRestrictions => ({ + codeList: [], + empty: [], + range: [], + regex: [], + required: [], +}); + +const mergeCollected = (target: CollectedRestrictions, source: CollectedRestrictions): CollectedRestrictions => ({ + codeList: [...target.codeList, ...source.codeList], + empty: [...target.empty, ...source.empty], + range: [...target.range, ...source.range], + regex: [...target.regex, ...source.regex], + required: [...target.required, ...source.required], +}); + +const collectFromPlainRestriction = (restriction: AnyRestrictionObject): CollectedRestrictions => { + const collected = emptyCollected(); + if (restriction.codeList !== undefined) { + collected.codeList.push(restriction.codeList as (string | number)[]); + } + if (restriction.empty !== undefined) { + collected.empty.push(restriction.empty); + } + if (restriction.range !== undefined) { + collected.range.push(restriction.range); + } + if (restriction.regex !== undefined) { + collected.regex.push(restriction.regex); + } + if (restriction.required !== undefined) { + collected.required.push(restriction.required); + } + return collected; +}; + +/** + * Recursively traverse a field's restrictions, evaluating all conditional branches against the + * partial record, and collect all active non-conditional restriction values grouped by type. + * + * Each entry in the returned arrays represents one active restriction object that specified that key. + * Missing fields in `record` are treated as `undefined` when evaluating conditions. The returned + * `CollectedRestrictions` is then passed to the per-type reducers in `restrictionReducers.ts` to + * produce a single merged value for each restriction type. + */ +export const collectRestrictions = ( + restrictions: + | TRestrictions + | ConditionalRestriction + | (TRestrictions | ConditionalRestriction)[] + | undefined, + record: DataRecord, +): CollectedRestrictions => { + if (restrictions === undefined) { + return emptyCollected(); + } + + const entries = Array.isArray(restrictions) ? restrictions : [restrictions]; + let collected = emptyCollected(); + + for (const entry of entries) { + if (isConditionalRestriction(entry)) { + const conditionPasses = testConditionalRestriction(entry.if, undefined, record); + const branchCollected = collectRestrictions(conditionPasses ? entry.then : entry.else, record); + collected = mergeCollected(collected, branchCollected); + } else { + const plain = collectFromPlainRestriction(entry); + collected = mergeCollected(collected, plain); + } + } + + return collected; +}; diff --git a/packages/data-generator/src/dataGeneration/fields/restrictionReducers.ts b/packages/data-generator/src/dataGeneration/fields/restrictionReducers.ts new file mode 100644 index 00000000..76666c6a --- /dev/null +++ b/packages/data-generator/src/dataGeneration/fields/restrictionReducers.ts @@ -0,0 +1,265 @@ +/* + * Copyright (c) 2026 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { + failWith, + success, + type RestrictionRange, + type RestrictionRegex, + type Result, +} from '@overture-stack/lectern-dictionary'; + +/** + * The conflict data returned in the failure case of a restriction reducer. Describes which + * restriction values could not be reconciled and why. + */ +export type RestrictionConflict = { + type: 'codeList' | 'range' | 'required-empty'; + values: unknown[]; + reason: string; +}; + +/** + * Result type for restriction reducers. On success, carries the single merged restriction value. + * On failure, carries a `RestrictionConflict` describing what could not be reconciled. + */ +export type RestrictionReducerResult = Result; + +/* ************************** * + * Boolean reducers * + * ************************** */ + +/** + * Reduces a list of `required` restriction values to a single boolean. Multiple `true` values are + * treated as equivalent - `required: true` if any entry is true, `false` otherwise. + * An empty list returns `false`. + * + * This reducer cannot produce a conflict on its own; the `required`/`empty` conflict is detected + * at the field level after both are reduced independently. + */ +export const reduceRequired = (values: boolean[]): boolean => + values.length > 0 ? values.some((value) => value) : false; + +/** + * Reduces a list of `empty` restriction values to a single boolean. Multiple `true` values are + * treated as equivalent - `empty: true` if any entry is true, `false` otherwise. + * An empty list returns `false`. + * + * This reducer cannot produce a conflict on its own; the `required`/`empty` conflict is detected + * at the field level after both are reduced independently. + */ +export const reduceEmpty = (values: boolean[]): boolean => (values.length > 0 ? values.some((value) => value) : false); + +/* ************************** * + * CodeList reducer * + * ************************** */ + +/** + * Reduces a list of codeLists to the single intersection of all lists. + * + * - Zero lists: returns `success(undefined)` - no restriction applies. + * - One list: returns `success(list)` unchanged. + * - Multiple lists: returns `success(intersection)` when the intersection is non-empty. + * Returns a failure with `type: 'codeList'` when the intersection is empty, meaning no value + * can satisfy all lists simultaneously. The failure carries a `data` field (the conflict) but no + * fallback value - the caller is responsible for choosing a fallback from the input lists. + */ +export const reduceCodeLists = (lists: T[][]): RestrictionReducerResult => { + if (lists.length === 0) { + return success(undefined); + } + if (lists.length === 1) { + return success(lists[0]); + } + + const [first, ...rest] = lists; + const restSets = rest.map((list) => new Set(list)); + const intersection = (first ?? []).filter((value) => restSets.every((set) => set.has(value))); + + if (intersection.length === 0) { + return failWith('codeLists have no common values.', { + type: 'codeList', + values: lists, + reason: `codeLists have no common values: ${lists.map((list) => JSON.stringify(list)).join(', ')}`, + }); + } + + return success(intersection); +}; + +/* ************************** * + * Range reducer * + * ************************** */ + +/** + * Reduces a list of range restrictions to the tightest overlapping subrange. + * + * - Zero ranges: returns `success(undefined)` - no restriction applies. + * - One range: returns `success(range)` unchanged. + * - Multiple ranges: returns `success(merged)` where `merged` is the intersection of all ranges. + * The lower bound is the highest lower bound across all ranges, and the upper bound is the lowest + * upper bound. Exclusive bounds are preferred over inclusive when they are equal. + * Returns a failure with `type: 'range'` when the resulting range is empty (lower bound exceeds + * upper bound, or they are equal with at least one exclusive side). + */ +export const reduceRanges = (ranges: RestrictionRange[]): RestrictionReducerResult => { + if (ranges.length === 0) { + return success(undefined); + } + if (ranges.length === 1) { + return success(ranges[0]); + } + + let low: number | undefined; + let high: number | undefined; + let lowExclusive = false; + let highExclusive = false; + + for (const range of ranges) { + const rangeLow = range.min ?? range.exclusiveMin; + const rangeHigh = range.max ?? range.exclusiveMax; + const rangeLowExclusive = range.exclusiveMin !== undefined; + const rangeHighExclusive = range.exclusiveMax !== undefined; + + if (rangeLow !== undefined) { + if (low === undefined || rangeLow > low || (rangeLow === low && rangeLowExclusive && !lowExclusive)) { + low = rangeLow; + lowExclusive = rangeLowExclusive; + } + } + if (rangeHigh !== undefined) { + if (high === undefined || rangeHigh < high || (rangeHigh === high && rangeHighExclusive && !highExclusive)) { + high = rangeHigh; + highExclusive = rangeHighExclusive; + } + } + } + + // Empty implies no possible values in the range, when lower bound is greater than upper bound. + const isEmpty = + low !== undefined && high !== undefined && (low > high || (low === high && (lowExclusive || highExclusive))); + + if (isEmpty) { + return failWith('Ranges have no overlapping values.', { + type: 'range', + values: ranges, + reason: `Ranges have no overlapping values: ${ranges.map((range) => JSON.stringify(range)).join(', ')}`, + }); + } + + const merged: RestrictionRange = {}; + if (low !== undefined) { + if (lowExclusive) { + merged.exclusiveMin = low; + } else { + merged.min = low; + } + } + if (high !== undefined) { + if (highExclusive) { + merged.exclusiveMax = high; + } else { + merged.max = high; + } + } + + return success(merged); +}; + +/* ************************** * + * Cross-type filters * + * ************************** */ + +/** + * Returns `true` if `value` falls within all bounds specified by `range`. Handles inclusive (`min`, + * `max`) and exclusive (`exclusiveMin`, `exclusiveMax`) bounds independently. + */ +export const satisfiesRange = (value: number, range: RestrictionRange): boolean => { + if (range.min !== undefined && value < range.min) { + return false; + } + if (range.max !== undefined && value > range.max) { + return false; + } + if (range.exclusiveMin !== undefined && value <= range.exclusiveMin) { + return false; + } + if (range.exclusiveMax !== undefined && value >= range.exclusiveMax) { + return false; + } + return true; +}; + +/** + * Filters a numeric code list to values that satisfy `range`. Returns the full list unchanged when + * `range` is `undefined`. Returns an empty array when no values satisfy the range - the caller is + * responsible for treating this as a conflict. + */ +export const filterCodeListByRange = (codeList: number[], range: RestrictionRange | undefined): number[] => { + if (range === undefined) { + return codeList; + } + return codeList.filter((entry) => satisfiesRange(entry, range)); +}; + +/** + * Filters a string code list to values that match `regex`. Returns the full list unchanged when + * `regex` is `undefined`. Returns an empty array when no values match - the caller is responsible + * for treating this as a conflict. + */ +export const filterCodeListByRegex = (codeList: string[], regex: string | undefined): string[] => { + if (regex === undefined) { + return codeList; + } + const compiled = new RegExp(regex); + return codeList.filter((entry) => compiled.test(entry)); +}; + +/* ************************** * + * Regex reducer * + * ************************** */ + +/** + * Reduces a list of regex restrictions to a single combined pattern using lookahead conjunction. + * + * - Zero patterns: returns `success(undefined)` - no restriction applies. + * - One pattern: returns `success(pattern)` unchanged. + * - Multiple patterns: wraps each in a non-capturing lookahead `(?=pattern)` and concatenates them + * into a single string. The result matches strings that satisfy all patterns simultaneously. + * + * This reducer always succeeds - syntactic combination is always possible. However, the combined + * pattern may be semantically impossible to satisfy (e.g. `/^a/` AND `/^b/`). Detection of such + * impossibility is deferred to generation time, where fast-check will throw if it cannot produce + * a conforming string. + * + * Each input may be a single pattern string or an array of pattern strings. Arrays are flattened + * before combination. + */ +export const reduceRegex = (patterns: RestrictionRegex[]): RestrictionReducerResult => { + const allPatterns = patterns.flatMap((entry) => (Array.isArray(entry) ? entry : [entry])); + + if (allPatterns.length === 0) { + return success(undefined); + } + if (allPatterns.length === 1) { + return success(allPatterns[0]); + } + + return success(allPatterns.map((pattern) => `(?=${pattern})`).join('')); +}; diff --git a/packages/data-generator/src/dataGeneration/records/fieldDependencies.ts b/packages/data-generator/src/dataGeneration/records/fieldDependencies.ts new file mode 100644 index 00000000..6589d89a --- /dev/null +++ b/packages/data-generator/src/dataGeneration/records/fieldDependencies.ts @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2026 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import { isConditionalRestriction, type ConditionalRestriction, type Schema } from '@overture-stack/lectern-dictionary'; + +/** + * Maps each field name to the set of field names it references in its conditional restrictions. + * Fields with no conditional restrictions map to an empty `Set`. + */ +export type FieldDependencyMap = Map>; + +/** + * An ordered list of generation tiers. Fields in tier `n` depend only on fields in tiers `0..n-1`. + * All fields within the same tier have no ordering constraint between them. + * When a dependency cycle is detected, all fields involved in the cycle are placed in the same tier. + */ +export type FieldGenerationOrder = string[][]; + +/** + * Recursively walks a restriction tree and adds all field names referenced in conditional `if` + * clauses to `accumulator`. Both `then` and `else` branches are traversed regardless of runtime + * evaluation - this is a static analysis pass. + */ +const collectDependencyFieldNames = ( + restriction: + | TRestrictions + | ConditionalRestriction + | (TRestrictions | ConditionalRestriction)[] + | undefined, + accumulator: Set, +): void => { + if (restriction === undefined) { + return; + } + + const entries = Array.isArray(restriction) ? restriction : [restriction]; + + for (const entry of entries) { + if (isConditionalRestriction(entry)) { + for (const condition of entry.if.conditions) { + for (const fieldName of condition.fields) { + accumulator.add(fieldName); + } + } + collectDependencyFieldNames(entry.then, accumulator); + collectDependencyFieldNames(entry.else, accumulator); + } + } +}; + +/** + * Builds a static dependency map for all fields in `schema`. Each field maps to the set of other + * field names it references in conditional restrictions. Self-references and references to fields + * not defined in the schema are excluded. + */ +export const extractFieldDependencies = (schema: Schema): FieldDependencyMap => { + const schemaFieldNames = new Set(schema.fields.map((field) => field.name)); + const dependencyMap: FieldDependencyMap = new Map(); + + for (const field of schema.fields) { + const rawDependencies = new Set(); + collectDependencyFieldNames(field.restrictions, rawDependencies); + const dependencies = new Set( + [...rawDependencies].filter((dependency) => dependency !== field.name && schemaFieldNames.has(dependency)), + ); + dependencyMap.set(field.name, dependencies); + } + + return dependencyMap; +}; + +/** + * Determines the order in which fields should be generated using Kahn's topological sort algorithm. + * Returns a list of tiers: fields in tier `n` may only reference fields from earlier tiers. + * Fields within the same tier have no ordering constraint between them. + * + * If a dependency cycle is detected (fields that cannot be resolved due to circular references), + * all remaining cyclic fields are placed together in a final tier. + */ +export const resolveGenerationOrder = (schema: Schema): FieldGenerationOrder => { + const dependencyMap = extractFieldDependencies(schema); + const schemaFieldNames = schema.fields.map((field) => field.name); + + const inDegree = new Map(); + const dependents = new Map>(); + + for (const fieldName of schemaFieldNames) { + inDegree.set(fieldName, 0); + dependents.set(fieldName, new Set()); + } + + for (const fieldName of schemaFieldNames) { + const dependencies = dependencyMap.get(fieldName) ?? new Set(); + for (const dependency of dependencies) { + inDegree.set(fieldName, (inDegree.get(fieldName) ?? 0) + 1); + const dependentSet = dependents.get(dependency); + if (dependentSet !== undefined) { + dependentSet.add(fieldName); + } + } + } + + // Seed the ready queue with all zero-in-degree fields, preserving schema definition order. + const readyQueue: string[] = schemaFieldNames.filter((fieldName) => (inDegree.get(fieldName) ?? 0) === 0); + const processed = new Set(); + const order: FieldGenerationOrder = []; + + while (processed.size < schemaFieldNames.length) { + if (readyQueue.length === 0) { + // Cycle detected - only the fields still in the graph (inDegree > 0) are cyclic. + // Independent fields with inDegree === 0 were already drained into readyQueue before + // this branch, so any field remaining here genuinely participates in a cycle. + const cyclic = schemaFieldNames.filter((fieldName) => !processed.has(fieldName)); + order.push(cyclic); + break; + } + + // Drain the full ready queue into one tier (all have inDegree 0, no ordering constraint). + const currentTier = readyQueue.splice(0); + order.push(currentTier); + + for (const fieldName of currentTier) { + processed.add(fieldName); + for (const dependent of dependents.get(fieldName) ?? new Set()) { + const newDegree = (inDegree.get(dependent) ?? 1) - 1; + inDegree.set(dependent, newDegree); + if (newDegree === 0) { + readyQueue.push(dependent); + } + } + } + } + + return order; +}; diff --git a/packages/data-generator/src/dataGeneration/records/recordGenerator.ts b/packages/data-generator/src/dataGeneration/records/recordGenerator.ts new file mode 100644 index 00000000..d0f3b8b4 --- /dev/null +++ b/packages/data-generator/src/dataGeneration/records/recordGenerator.ts @@ -0,0 +1,189 @@ +/* + * Copyright (c) 2026 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import type { DataRecord, DataRecordValue, Schema, SchemaField } from '@overture-stack/lectern-dictionary'; +import { knuthHash } from '../../common/hash'; +import { + generateBooleanValue, + generateIntegerValue, + generateNumberValue, + generateStringValue, +} from '../fields/fieldGenerators'; +import { resolveGenerationOrder } from './fieldDependencies'; + +/** + * Supplies the set of valid parent rows for each FK relationship when generating child records. + * + * The map is keyed by the **parent schema name** (matching `ForeignKeyRestriction.schema`). + * Each value is an array of partial `DataRecord` objects - one entry per available parent row. + * + * Each partial record need only contain the fields named in the FK mappings' `foreign` side for + * the relevant FK rule. It does not need to be a complete record from the parent schema; any fields + * not referenced by FK mappings on this child schema are ignored. + * + * Example: if the child schema has a FK to `"donor"` with mapping `{ local: "donor_id", foreign: "id" }`, + * the pool entry for `"donor"` must include at least `{ id: someValue }` for each available parent row. + * + * For composite FK rules (multiple mappings in a single `ForeignKeyRestriction`), all mapped local + * fields are assigned from the **same** selected parent row, preserving relational consistency. + * + * When a parent schema name has no entry in this map, FK fields referencing that schema are + * generated normally - field-level restrictions apply and no FK constraint is enforced. + */ +export type ForeignKeyPool = Map; + +/** + * Options for `generateRecord`. + * + * `overrides` provides values for specific fields by name; those fields are not generated. + * `seed` controls the RNG so that the same schema and seed always produce the same record. + * `foreignKeyPool` supplies available parent rows for FK-constrained fields. + * `emptyRate` is the probability (0–1) that any non-required field is left empty (`undefined`). + * Passed through to each field generator unchanged; see `FieldGeneratorOptions.emptyRate`. + * + * `fieldExclusions` maps field names to sets of values that the generator must not produce for + * that field. Passed through to each field generator as `excludeValues`; see + * `FieldGeneratorOptions.excludeValues`. Used by the schema generator to enforce `unique` + * constraints across records. + */ +export type RecordGeneratorOptions = { + overrides?: DataRecord; + seed?: number; + foreignKeyPool?: ForeignKeyPool; + emptyRate?: number; + fieldExclusions?: Record>; +}; + +const seededIndexInRange = (seed: number, length: number): number => knuthHash(seed) % length; + +/** + * Resolves FK-derived field values from `foreignKeyPool` for all FK rules on `schema`. + * + * For each `ForeignKeyRestriction`, a single parent row is selected at random from the pool. + * All local fields in that rule's mappings are assigned from that same row, preserving composite + * FK consistency. FK row selection uses seeds offset beyond the per-field seed range + * (`seed + schema.fields.length + fkIndex`) so they never collide with field generation seeds. + * + * Returns a partial `DataRecord` containing only the FK-derived field values. Fields whose parent + * schema has no pool entry are omitted - they will be generated normally. + */ +const resolveForeignKeyOverrides = (schema: Schema, pool: ForeignKeyPool, seed: number | undefined): DataRecord => { + const fkOverrides: DataRecord = {}; + const foreignKeyRules = schema.restrictions?.foreignKey ?? []; + + foreignKeyRules.forEach((fkRestriction, fkIndex) => { + const parentRows = pool.get(fkRestriction.schema); + if (parentRows === undefined || parentRows.length === 0) { + return; + } + + const rowSeed = seed !== undefined ? seed + schema.fields.length + fkIndex + 1 : undefined; + const rowIndex = + rowSeed !== undefined ? + seededIndexInRange(rowSeed, parentRows.length) + : Math.floor(Math.random() * parentRows.length); + + const selectedRow = parentRows[rowIndex] ?? parentRows[0]; + if (selectedRow === undefined) { + return; + } + + for (const mapping of fkRestriction.mappings) { + if (Object.hasOwn(selectedRow, mapping.foreign)) { + fkOverrides[mapping.local] = selectedRow[mapping.foreign]; + } + } + }); + + return fkOverrides; +}; + +/** + * Generates a `DataRecord` with values for every field in `schema`. Each field's value is produced + * by the appropriate field generator for its `valueType`, respecting all active restrictions. + * + * If `options.seed` is provided, the RNG is seeded before generation so the same schema and seed + * always produce the same record. Per-field seeds are derived by offsetting the record seed by the + * field's definition-order index in `schema.fields`, ensuring stability even when generation order + * differs from definition order. + * + * If `options.overrides` is provided, fields with an override value use that value directly and are + * not generated. Explicit overrides take priority over `foreignKeyPool` values. + * + * If `options.foreignKeyPool` is provided, fields governed by a FK restriction on the schema are + * assigned values from a randomly selected parent row rather than generated freely. All local fields + * within a single FK rule are drawn from the same parent row to preserve composite FK consistency. + * See `ForeignKeyPool` for the expected pool structure. + * + * Fields are generated in dependency order: a field whose conditional restrictions reference other + * fields is always generated after those fields, so the partial record passed into later generators + * reflects the correct values when evaluating conditional branches. + */ +export const generateRecord = (schema: Schema, options?: RecordGeneratorOptions): DataRecord => { + const { seed, overrides = {}, foreignKeyPool, emptyRate, fieldExclusions } = options ?? {}; + const record: DataRecord = {}; + + const fkOverrides = foreignKeyPool !== undefined ? resolveForeignKeyOverrides(schema, foreignKeyPool, seed) : {}; + const effectiveOverrides: DataRecord = { ...fkOverrides, ...overrides }; + + const fieldByName = new Map(schema.fields.map((field) => [field.name, field])); + const fieldIndexByName = new Map(schema.fields.map((field, fieldIndex) => [field.name, fieldIndex])); + + const generationOrder = resolveGenerationOrder(schema); + + for (const tier of generationOrder) { + for (const fieldName of tier) { + if (Object.hasOwn(effectiveOverrides, fieldName)) { + record[fieldName] = effectiveOverrides[fieldName]; + continue; + } + + const field = fieldByName.get(fieldName); + if (field === undefined) { + continue; + } + + const definitionIndex = fieldIndexByName.get(fieldName) ?? 0; + const fieldSeed = seed !== undefined ? seed + definitionIndex + 1 : undefined; + const excludeValues = fieldExclusions?.[fieldName]; + const fieldOptions = { seed: fieldSeed, record, emptyRate, excludeValues }; + + let result; + switch (field.valueType) { + case 'boolean': + result = generateBooleanValue(field, fieldOptions); + break; + case 'integer': + result = generateIntegerValue(field, fieldOptions); + break; + case 'number': + result = generateNumberValue(field, fieldOptions); + break; + case 'string': + result = generateStringValue(field, fieldOptions); + break; + } + + const value: DataRecordValue = result.success ? result.data : result.data.value; + record[fieldName] = value; + } + } + + return record; +}; diff --git a/packages/data-generator/src/dataGeneration/records/schemaGenerator.ts b/packages/data-generator/src/dataGeneration/records/schemaGenerator.ts new file mode 100644 index 00000000..a49bb95c --- /dev/null +++ b/packages/data-generator/src/dataGeneration/records/schemaGenerator.ts @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2026 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import type { DataRecord, DataRecordValue, Schema } from '@overture-stack/lectern-dictionary'; +import { knuthHash } from '../../common/hash'; +import { type ForeignKeyPool, generateRecord } from './recordGenerator'; + +type UniqueFieldTracker = Map>; + +const serializeKeyTuple = (schema: Schema, record: DataRecord): string => + JSON.stringify((schema.restrictions?.uniqueKey ?? []).map((fieldName) => record[fieldName])); + +/** + * Options for `generateSchemaRecords`. + * + * `initialUniqueValues` pre-populates the uniqueness trackers to avoid collisions with records + * already written elsewhere (e.g. when appending to an existing file). + * `fields` maps field names to arrays of pre-seen values for `unique` fields. + * `keys` is an array of pre-seen serialized `uniqueKey` tuples (`JSON.stringify(keyFieldValues)`). + * + * All other options behave identically to `RecordGeneratorOptions`. + */ +export type SchemaGeneratorOptions = { + count: number; + seed?: number; + foreignKeyPool?: ForeignKeyPool; + emptyRate?: number; + initialUniqueValues?: { + fields?: Record; + keys?: string[]; + }; +}; + +const MAX_UNIQUE_KEY_RETRIES = 10; + +// Each (recordSeed, retryCount) pair produces a distinct seed independent of the main sequence. +// XOR with a large odd constant multiple of retryCount before hashing so each retry count maps to +// a different pre-hash value. 2246822519 is a large odd 32-bit prime chosen for bit dispersion. +const deriveRetrySeed = (recordSeed: number, retryCount: number): number => + knuthHash((recordSeed ^ (retryCount * 2246822519)) >>> 0); + +/** + * Synchronous generator that yields `options.count` `DataRecord` values for `schema`. + * + * Enforces `unique` field constraints by excluding already-seen values from each field generator. + * Enforces `uniqueKey` constraints by retrying generation (up to 10 times) when a composite key + * tuple collides. After exhausting retries the record is yielded as-is. + */ +export function* generateSchemaRecords(schema: Schema, options?: SchemaGeneratorOptions): Generator { + const { count = 0, seed, foreignKeyPool, emptyRate, initialUniqueValues } = options ?? {}; + + const uniqueFields = schema.fields.filter((field) => field.unique === true); + const uniqueKeyFields = schema.restrictions?.uniqueKey ?? []; + + const uniqueFieldTracker: UniqueFieldTracker = new Map( + uniqueFields.map((field) => { + const initialValues = initialUniqueValues?.fields?.[field.name] ?? []; + return [field.name, new Set(initialValues)]; + }), + ); + + const uniqueKeyTracker = new Set(initialUniqueValues?.keys ?? []); + + for (let recordIndex = 0; recordIndex < count; recordIndex++) { + const fieldExclusions: Record> = {}; + for (const [fieldName, seenValues] of uniqueFieldTracker) { + fieldExclusions[fieldName] = seenValues; + } + + const recordSeed = seed !== undefined ? seed + recordIndex + 1 : undefined; + let record = generateRecord(schema, { seed: recordSeed, foreignKeyPool, emptyRate, fieldExclusions }); + + if (uniqueKeyFields.length > 0) { + let retryCount = 0; + while (uniqueKeyTracker.has(serializeKeyTuple(schema, record)) && retryCount < MAX_UNIQUE_KEY_RETRIES) { + retryCount++; + const retrySeed = recordSeed !== undefined ? deriveRetrySeed(recordSeed, retryCount) : undefined; + record = generateRecord(schema, { seed: retrySeed, foreignKeyPool, emptyRate, fieldExclusions }); + } + uniqueKeyTracker.add(serializeKeyTuple(schema, record)); + } + + for (const [fieldName, seenValues] of uniqueFieldTracker) { + seenValues.add(record[fieldName]); + } + + yield record; + } +} diff --git a/packages/data-generator/src/index.ts b/packages/data-generator/src/index.ts new file mode 100644 index 00000000..2f268fac --- /dev/null +++ b/packages/data-generator/src/index.ts @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +export { + type DataFileHandle, + type WriteRecordError, + openDataFile, + openTsvFile, + openCsvFile, + writeRecord, + closeDataFile, +} from './dataFile/dataFileWriter'; +export { generateSchemaFile, generateDictionaryFiles, type GenerateFileError } from './dataFile/dataFileGenerator'; +export { + FieldGenerator, + FieldGeneratorFailureData, + FieldGeneratorOptions, + FieldGeneratorResult, + generateBooleanValue, + generateIntegerValue, + generateNumberValue, + generateStringValue, +} from './dataGeneration/fields/fieldGenerators'; +export { + extractFieldDependencies, + resolveGenerationOrder, + type FieldDependencyMap, + type FieldGenerationOrder, +} from './dataGeneration/records/fieldDependencies'; +export { + generateRecord, + type ForeignKeyPool, + type RecordGeneratorOptions, +} from './dataGeneration/records/recordGenerator'; +export { generateSchemaRecords, type SchemaGeneratorOptions } from './dataGeneration/records/schemaGenerator'; +export { + generateDictionaryRecords, + type DictionaryGeneratorOptions, + type DictionaryRecord, + type DictionarySchemaCount, +} from './dataGeneration/dictionary/dictionaryGenerator'; +export { collectRestrictions, CollectedRestrictions } from './dataGeneration/fields/resolveRestrictions'; diff --git a/packages/data-generator/test/dataFileGenerator.spec.ts b/packages/data-generator/test/dataFileGenerator.spec.ts new file mode 100644 index 00000000..231971cc --- /dev/null +++ b/packages/data-generator/test/dataFileGenerator.spec.ts @@ -0,0 +1,188 @@ +/* + * Copyright (c) 2026 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import assert from 'node:assert'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, it } from 'mocha'; +import type { Dictionary, Schema } from '@overture-stack/lectern-dictionary'; +import { generateDictionaryFiles, generateSchemaFile } from '../src/dataFile/dataFileGenerator'; + +const SEED = 42; +const NO_EMPTY = { emptyRate: 0 } as const; + +const donorSchema: Schema = { + name: 'donor', + fields: [ + { name: 'id', valueType: 'string', unique: true, restrictions: undefined }, + { name: 'program', valueType: 'string', restrictions: { codeList: ['P1', 'P2'] } }, + ], +}; + +const sampleSchema: Schema = { + name: 'sample', + fields: [ + { name: 'sample_id', valueType: 'string', unique: true, restrictions: undefined }, + { name: 'donor_id', valueType: 'string', restrictions: undefined }, + ], + restrictions: { + foreignKey: [{ schema: 'donor', mappings: [{ local: 'donor_id', foreign: 'id' }] }], + }, +}; + +const dictionary: Dictionary = { + name: 'test-dictionary', + version: '1.0', + schemas: [donorSchema, sampleSchema], +}; + +let tmpDir: string; + +beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'data-generator-test-')); +}); + +afterEach(() => { + fs.rmSync(tmpDir, { recursive: true, force: true }); +}); + +describe('generateSchemaFile', () => { + it('creates a file named . in the output directory', async () => { + await generateSchemaFile(donorSchema, tmpDir, 'tsv', { count: 3, seed: SEED, ...NO_EMPTY }); + assert.ok(fs.existsSync(path.join(tmpDir, 'donor.tsv'))); + }); + + it('writes the correct number of data rows plus one header row', async () => { + await generateSchemaFile(donorSchema, tmpDir, 'tsv', { count: 3, seed: SEED, ...NO_EMPTY }); + const lines = fs.readFileSync(path.join(tmpDir, 'donor.tsv'), 'utf8').trim().split('\n'); + assert.strictEqual(lines.length, 4); // 1 header + 3 data + }); + + it('first row is the header with schema field names', async () => { + await generateSchemaFile(donorSchema, tmpDir, 'tsv', { count: 1, seed: SEED, ...NO_EMPTY }); + const lines = fs.readFileSync(path.join(tmpDir, 'donor.tsv'), 'utf8').split('\n'); + assert.strictEqual(lines[0], 'id\tprogram'); + }); + + it('uses comma delimiter for csv format', async () => { + await generateSchemaFile(donorSchema, tmpDir, 'csv', { count: 1, seed: SEED, ...NO_EMPTY }); + const lines = fs.readFileSync(path.join(tmpDir, 'donor.csv'), 'utf8').split('\n'); + assert.strictEqual(lines[0], 'id,program'); + }); + + it('returns failure when the output directory does not exist', async () => { + const result = await generateSchemaFile(donorSchema, path.join(tmpDir, 'nonexistent'), 'tsv', { + count: 1, + seed: SEED, + }); + assert.strictEqual(result.success, false); + assert.strictEqual(result.data.error, 'DIRECTORY_NOT_FOUND'); + }); + + it('returns failure when the output file already exists', async () => { + fs.writeFileSync(path.join(tmpDir, 'donor.tsv'), 'existing content'); + const result = await generateSchemaFile(donorSchema, tmpDir, 'tsv', { count: 1, seed: SEED }); + assert.strictEqual(result.success, false); + assert.strictEqual(result.data.error, 'FILE_ALREADY_EXISTS'); + }); + + it('does not modify an existing file on failure', async () => { + const existingPath = path.join(tmpDir, 'donor.tsv'); + fs.writeFileSync(existingPath, 'existing content'); + await generateSchemaFile(donorSchema, tmpDir, 'tsv', { count: 1, seed: SEED }); + assert.strictEqual(fs.readFileSync(existingPath, 'utf8'), 'existing content'); + }); + + it('file is readable (stream closed) after successful generation', async () => { + await generateSchemaFile(donorSchema, tmpDir, 'tsv', { count: 2, seed: SEED, ...NO_EMPTY }); + const filePath = path.join(tmpDir, 'donor.tsv'); + // Verify content is fully flushed by reading it back immediately after the call returns. + const content = fs.readFileSync(filePath, 'utf8'); + const lines = content.trim().split('\n'); + assert.strictEqual(lines.length, 3); // header + 2 data rows, all flushed + }); +}); + +describe('generateDictionaryFiles', () => { + it('creates one file per schema with a non-zero count', async () => { + await generateDictionaryFiles(dictionary, tmpDir, 'tsv', { + counts: { donor: 3, sample: 5 }, + seed: SEED, + ...NO_EMPTY, + }); + assert.ok(fs.existsSync(path.join(tmpDir, 'donor.tsv'))); + assert.ok(fs.existsSync(path.join(tmpDir, 'sample.tsv'))); + }); + + it('does not create a file for schemas with count 0', async () => { + await generateDictionaryFiles(dictionary, tmpDir, 'tsv', { + counts: { donor: 3, sample: 0 }, + seed: SEED, + ...NO_EMPTY, + }); + assert.ok(!fs.existsSync(path.join(tmpDir, 'sample.tsv'))); + }); + + it('each file contains the correct number of data rows', async () => { + await generateDictionaryFiles(dictionary, tmpDir, 'tsv', { + counts: { donor: 2, sample: 4 }, + seed: SEED, + ...NO_EMPTY, + }); + const donorLines = fs.readFileSync(path.join(tmpDir, 'donor.tsv'), 'utf8').trim().split('\n'); + const sampleLines = fs.readFileSync(path.join(tmpDir, 'sample.tsv'), 'utf8').trim().split('\n'); + assert.strictEqual(donorLines.length, 3); // 1 header + 2 data + assert.strictEqual(sampleLines.length, 5); // 1 header + 4 data + }); + + it('returns failure when the output directory does not exist', async () => { + const result = await generateDictionaryFiles(dictionary, path.join(tmpDir, 'nonexistent'), 'tsv', { + counts: { donor: 1 }, + seed: SEED, + }); + assert.strictEqual(result.success, false); + assert.strictEqual(result.data.error, 'DIRECTORY_NOT_FOUND'); + }); + + it('all files are readable (streams closed) after successful generation', async () => { + await generateDictionaryFiles(dictionary, tmpDir, 'tsv', { + counts: { donor: 2, sample: 3 }, + seed: SEED, + ...NO_EMPTY, + }); + // Verify both files are fully flushed by reading them back immediately after generation. + const donorContent = fs.readFileSync(path.join(tmpDir, 'donor.tsv'), 'utf8'); + const sampleContent = fs.readFileSync(path.join(tmpDir, 'sample.tsv'), 'utf8'); + assert.strictEqual(donorContent.trim().split('\n').length, 3); // header + 2 + assert.strictEqual(sampleContent.trim().split('\n').length, 4); // header + 3 + }); + + it('returns failure before writing any files when any expected output file already exists', async () => { + fs.writeFileSync(path.join(tmpDir, 'donor.tsv'), 'existing content'); + const result = await generateDictionaryFiles(dictionary, tmpDir, 'tsv', { + counts: { donor: 2, sample: 3 }, + seed: SEED, + }); + assert.strictEqual(result.success, false); + assert.strictEqual(result.data.error, 'FILE_ALREADY_EXISTS'); + // sample.tsv must not have been created since we fail before writing + assert.ok(!fs.existsSync(path.join(tmpDir, 'sample.tsv'))); + }); +}); diff --git a/packages/data-generator/test/dataFileWriter.spec.ts b/packages/data-generator/test/dataFileWriter.spec.ts new file mode 100644 index 00000000..b3b44984 --- /dev/null +++ b/packages/data-generator/test/dataFileWriter.spec.ts @@ -0,0 +1,266 @@ +/* + * Copyright (c) 2026 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import assert from 'node:assert'; +import fs from 'node:fs'; +import fsp from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { after, afterEach, before, describe, it } from 'mocha'; +import type { Schema } from '@overture-stack/lectern-dictionary'; +import { DEFAULT_DELIMITER } from '@overture-stack/lectern-dictionary'; +import { closeDataFile, openCsvFile, openDataFile, openTsvFile, writeRecord } from '../src/dataFile/dataFileWriter'; + +const schema: Schema = { + name: 'test', + fields: [ + { name: 'id', valueType: 'string', restrictions: undefined }, + { name: 'count', valueType: 'integer', restrictions: undefined }, + { name: 'active', valueType: 'boolean', restrictions: undefined }, + ], +}; + +const schemaWithArray: Schema = { + name: 'array_test', + fields: [ + { name: 'id', valueType: 'string', restrictions: undefined }, + { name: 'tags', valueType: 'string', isArray: true, delimiter: ';', restrictions: undefined }, + ], +}; + +let tempDir: string; + +before(async () => { + tempDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'data-file-writer-test-')); +}); + +after(async () => { + await fsp.rm(tempDir, { recursive: true, force: true }); +}); + +const tempFile = (name: string): string => path.join(tempDir, name); + +const readLines = async (filePath: string): Promise => { + const content = await fsp.readFile(filePath, 'utf8'); + return content.split('\n').filter((line) => line.length > 0); +}; + +describe('dataFileWriter', () => { + describe('openTsvFile', () => { + it('writes a header row with tab-separated field names', async () => { + const filePath = tempFile('tsv-header.tsv'); + const handle = await openTsvFile(schema, filePath); + await closeDataFile(handle); + + const lines = await readLines(filePath); + assert.strictEqual(lines[0], 'id\tcount\tactive'); + }); + + it('header field order matches schema.fields order', async () => { + const filePath = tempFile('tsv-order.tsv'); + const handle = await openTsvFile(schema, filePath); + await closeDataFile(handle); + + const lines = await readLines(filePath); + const headers = lines[0]?.split('\t') ?? []; + assert.deepStrictEqual(headers, ['id', 'count', 'active']); + }); + }); + + describe('openCsvFile', () => { + it('writes a header row with comma-separated field names', async () => { + const filePath = tempFile('csv-header.csv'); + const handle = await openCsvFile(schema, filePath); + await closeDataFile(handle); + + const lines = await readLines(filePath); + assert.strictEqual(lines[0], 'id,count,active'); + }); + }); + + describe('openDataFile', () => { + it('tsv format uses tab as column delimiter', async () => { + const filePath = tempFile('format-tsv.tsv'); + const handle = await openDataFile(schema, filePath, 'tsv'); + await writeRecord(handle, { id: 'a', count: 1, active: true }); + await closeDataFile(handle); + + const lines = await readLines(filePath); + assert.strictEqual(lines[1], 'a\t1\ttrue'); + }); + + it('csv format uses comma as column delimiter', async () => { + const filePath = tempFile('format-csv.csv'); + const handle = await openDataFile(schema, filePath, 'csv'); + await writeRecord(handle, { id: 'a', count: 1, active: true }); + await closeDataFile(handle); + + const lines = await readLines(filePath); + assert.strictEqual(lines[1], 'a,1,true'); + }); + }); + + describe('writeRecord', () => { + it('returns a success result when the write succeeds', async () => { + const filePath = tempFile('write-success.tsv'); + const handle = await openTsvFile(schema, filePath); + const result = await writeRecord(handle, { id: 'a', count: 1, active: true }); + await closeDataFile(handle); + assert.strictEqual(result.success, true); + }); + + it('appends rows in the order writeRecord was called', async () => { + const filePath = tempFile('order.tsv'); + const handle = await openTsvFile(schema, filePath); + await writeRecord(handle, { id: 'first', count: 1, active: true }); + await writeRecord(handle, { id: 'second', count: 2, active: false }); + await writeRecord(handle, { id: 'third', count: 3, active: true }); + await closeDataFile(handle); + + const lines = await readLines(filePath); + assert.strictEqual(lines[1], 'first\t1\ttrue'); + assert.strictEqual(lines[2], 'second\t2\tfalse'); + assert.strictEqual(lines[3], 'third\t3\ttrue'); + }); + + it('column order follows schema.fields regardless of record key order', async () => { + const filePath = tempFile('col-order.tsv'); + const handle = await openTsvFile(schema, filePath); + // Record keys in reverse order + await writeRecord(handle, { active: true, count: 7, id: 'z' }); + await closeDataFile(handle); + + const lines = await readLines(filePath); + // Columns must still be id, count, active + assert.strictEqual(lines[1], 'z\t7\ttrue'); + }); + + it('field missing from record serializes as empty string', async () => { + const filePath = tempFile('missing-field.tsv'); + const handle = await openTsvFile(schema, filePath); + await writeRecord(handle, { id: 'x' }); // count and active omitted + await closeDataFile(handle); + + const lines = await readLines(filePath); + assert.strictEqual(lines[1], 'x\t\t'); + }); + + it('undefined field value serializes as empty string', async () => { + const filePath = tempFile('undefined-field.tsv'); + const handle = await openTsvFile(schema, filePath); + await writeRecord(handle, { id: 'x', count: undefined, active: undefined }); + await closeDataFile(handle); + + const lines = await readLines(filePath); + assert.strictEqual(lines[1], 'x\t\t'); + }); + + it('extra fields in record not present in schema are ignored', async () => { + const filePath = tempFile('extra-fields.tsv'); + const handle = await openTsvFile(schema, filePath); + await writeRecord(handle, { id: 'x', count: 1, active: true, extra: 'ignored' }); + await closeDataFile(handle); + + const lines = await readLines(filePath); + const columns = lines[1]?.split('\t') ?? []; + assert.strictEqual(columns.length, 3); + }); + + it('array field uses field.delimiter not column delimiter', async () => { + const filePath = tempFile('array-delimiter.tsv'); + const handle = await openTsvFile(schemaWithArray, filePath); + await writeRecord(handle, { id: 'x', tags: ['a', 'b', 'c'] }); + await closeDataFile(handle); + + const lines = await readLines(filePath); + // tags column should use ';' (field.delimiter), not '\t' (column delimiter) + assert.strictEqual(lines[1], 'x\ta;b;c'); + }); + + it('array field with no field.delimiter uses DEFAULT_DELIMITER', async () => { + const schemaNoDelimiter: Schema = { + name: 'no_delim', + fields: [ + { name: 'id', valueType: 'string', restrictions: undefined }, + { name: 'tags', valueType: 'string', isArray: true, restrictions: undefined }, + ], + }; + const filePath = tempFile('array-default-delimiter.tsv'); + const handle = await openTsvFile(schemaNoDelimiter, filePath); + await writeRecord(handle, { id: 'x', tags: ['a', 'b'] }); + await closeDataFile(handle); + + const lines = await readLines(filePath); + assert.strictEqual(lines[1], `x\ta${DEFAULT_DELIMITER}b`); + }); + }); + + describe('closeDataFile', () => { + it('closes the underlying stream', async () => { + const filePath = tempFile('close.tsv'); + const handle = await openTsvFile(schema, filePath); + await closeDataFile(handle); + + // File should be fully written and readable + const exists = fs.existsSync(filePath); + assert.ok(exists); + }); + + it('calling closeDataFile twice does not throw', async () => { + const filePath = tempFile('double-close.tsv'); + const handle = await openTsvFile(schema, filePath); + await closeDataFile(handle); + await assert.doesNotReject(() => closeDataFile(handle)); + }); + + it('writeRecord after close returns a STREAM_CLOSED failure result', async () => { + const filePath = tempFile('write-after-close.tsv'); + const handle = await openTsvFile(schema, filePath); + await closeDataFile(handle); + const result = await writeRecord(handle, { id: 'x', count: 1, active: true }); + assert.strictEqual(result.success, false); + assert.strictEqual(result.data.error, 'STREAM_CLOSED'); + }); + }); + + it('produces identical file content for the same records written in the same order', async () => { + const records = [ + { id: 'a', count: 1, active: true }, + { id: 'b', count: 2, active: false }, + ]; + + const filePathA = tempFile('deterministic-a.tsv'); + const handleA = await openTsvFile(schema, filePathA); + for (const record of records) { + await writeRecord(handleA, record); + } + await closeDataFile(handleA); + + const filePathB = tempFile('deterministic-b.tsv'); + const handleB = await openTsvFile(schema, filePathB); + for (const record of records) { + await writeRecord(handleB, record); + } + await closeDataFile(handleB); + + const contentA = await fsp.readFile(filePathA, 'utf8'); + const contentB = await fsp.readFile(filePathB, 'utf8'); + assert.strictEqual(contentA, contentB); + }); +}); diff --git a/packages/data-generator/test/dictionaryGenerator.spec.ts b/packages/data-generator/test/dictionaryGenerator.spec.ts new file mode 100644 index 00000000..f540608b --- /dev/null +++ b/packages/data-generator/test/dictionaryGenerator.spec.ts @@ -0,0 +1,272 @@ +/* + * Copyright (c) 2026 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import assert from 'node:assert'; +import { describe, it } from 'mocha'; +import type { Dictionary } from '@overture-stack/lectern-dictionary'; +import { generateDictionaryRecords } from '../src/dataGeneration/dictionary/dictionaryGenerator'; + +const SEED = 42; +const NO_EMPTY = { emptyRate: 0 } as const; + +const donorSchema = { + name: 'donor', + fields: [ + { name: 'id', valueType: 'string' as const, unique: true, restrictions: undefined }, + { name: 'program', valueType: 'string' as const, restrictions: { codeList: ['P1', 'P2', 'P3'] } }, + ], +}; + +const sampleSchema = { + name: 'sample', + fields: [ + { name: 'sample_id', valueType: 'string' as const, unique: true, restrictions: undefined }, + { name: 'donor_id', valueType: 'string' as const, restrictions: undefined }, + { name: 'type', valueType: 'string' as const, restrictions: { codeList: ['T1', 'T2'] } }, + ], + restrictions: { + foreignKey: [{ schema: 'donor', mappings: [{ local: 'donor_id', foreign: 'id' }] }], + }, +}; + +const standaloneSchema = { + name: 'project', + fields: [{ name: 'code', valueType: 'string' as const, restrictions: { codeList: ['A', 'B', 'C'] } }], +}; + +const dictionary: Dictionary = { + name: 'test-dictionary', + version: '1.0', + schemas: [donorSchema, sampleSchema, standaloneSchema], +}; + +const collectBySchema = ( + generator: Generator<{ schemaName: string; record: Record }>, +): Record[]> => { + const result: Record[]> = {}; + for (const { schemaName, record } of generator) { + (result[schemaName] ??= []).push(record); + } + return result; +}; + +describe('generateDictionaryRecords', () => { + it('yields one tagged record at a time', () => { + const generator = generateDictionaryRecords(dictionary, { counts: { donor: 2 }, seed: SEED, ...NO_EMPTY }); + const first = generator.next(); + assert.strictEqual(first.done, false); + assert.ok(typeof first.value?.schemaName === 'string'); + assert.ok(typeof first.value?.record === 'object'); + }); + + it('generates the correct number of records per schema', () => { + const records = collectBySchema( + generateDictionaryRecords(dictionary, { counts: { donor: 3, sample: 5, project: 2 }, seed: SEED, ...NO_EMPTY }), + ); + assert.strictEqual(records['donor']?.length, 3); + assert.strictEqual(records['sample']?.length, 5); + assert.strictEqual(records['project']?.length, 2); + }); + + it('schemas with count 0 are not included in the output', () => { + const records = collectBySchema( + generateDictionaryRecords(dictionary, { counts: { donor: 3, sample: 0, project: 2 }, seed: SEED, ...NO_EMPTY }), + ); + assert.ok(!Object.hasOwn(records, 'sample'), 'sample should not appear in output'); + }); + + it('schemas absent from counts are not included in the output', () => { + const records = collectBySchema( + generateDictionaryRecords(dictionary, { counts: { donor: 3 }, seed: SEED, ...NO_EMPTY }), + ); + assert.ok(!Object.hasOwn(records, 'sample'), 'sample should not appear in output'); + assert.ok(!Object.hasOwn(records, 'project'), 'project should not appear in output'); + }); + + it('parent schema records are yielded before child schema records', () => { + const yielded: string[] = []; + for (const { schemaName } of generateDictionaryRecords(dictionary, { + counts: { donor: 2, sample: 3 }, + seed: SEED, + ...NO_EMPTY, + })) { + yielded.push(schemaName); + } + const lastDonorIndex = yielded.lastIndexOf('donor'); + const firstSampleIndex = yielded.indexOf('sample'); + assert.ok(lastDonorIndex < firstSampleIndex, 'all donor records must be yielded before any sample records'); + }); + + it('child FK fields reference values that appear in parent records', () => { + const records = collectBySchema( + generateDictionaryRecords(dictionary, { counts: { donor: 4, sample: 10 }, seed: SEED, ...NO_EMPTY }), + ); + const donorIds = new Set(records['donor']?.map((record) => record['id'])); + for (const record of records['sample'] ?? []) { + assert.ok( + donorIds.has(record['donor_id']), + `sample donor_id '${String(record['donor_id'])}' not found in donor ids`, + ); + } + }); + + it('schema with no FK relationships generates independently', () => { + const records = collectBySchema( + generateDictionaryRecords(dictionary, { counts: { project: 5 }, seed: SEED, ...NO_EMPTY }), + ); + assert.strictEqual(records['project']?.length, 5); + for (const record of records['project'] ?? []) { + assert.ok(['A', 'B', 'C'].includes(record['code'] as string), `unexpected code: ${String(record['code'])}`); + } + }); + + describe('multi-level hierarchy', () => { + // grandparent → parent → child: 3-tier FK chain. + // child.parent_id must reference a parent record, and parent.grandparent_id must reference grandparent. + const grandparentSchema = { + name: 'grandparent', + fields: [{ name: 'gp_id', valueType: 'string' as const, unique: true, restrictions: undefined }], + }; + + const parentSchema = { + name: 'parent', + fields: [ + { name: 'p_id', valueType: 'string' as const, unique: true, restrictions: undefined }, + { name: 'grandparent_id', valueType: 'string' as const, restrictions: undefined }, + ], + restrictions: { + foreignKey: [{ schema: 'grandparent', mappings: [{ local: 'grandparent_id', foreign: 'gp_id' }] }], + }, + }; + + const childSchema = { + name: 'child', + fields: [ + { name: 'c_id', valueType: 'string' as const, unique: true, restrictions: undefined }, + { name: 'parent_id', valueType: 'string' as const, restrictions: undefined }, + ], + restrictions: { + foreignKey: [{ schema: 'parent', mappings: [{ local: 'parent_id', foreign: 'p_id' }] }], + }, + }; + + const threeLayerDictionary: Dictionary = { + name: 'three-layer', + version: '1.0', + schemas: [grandparentSchema, parentSchema, childSchema], + }; + + it('grandparent records are yielded before parent and child', () => { + const yielded: string[] = []; + for (const { schemaName } of generateDictionaryRecords(threeLayerDictionary, { + counts: { grandparent: 2, parent: 4, child: 8 }, + seed: SEED, + ...NO_EMPTY, + })) { + yielded.push(schemaName); + } + const lastGrandparent = yielded.lastIndexOf('grandparent'); + const firstParent = yielded.indexOf('parent'); + const lastParent = yielded.lastIndexOf('parent'); + const firstChild = yielded.indexOf('child'); + assert.ok(lastGrandparent < firstParent, 'all grandparent records must come before any parent records'); + assert.ok(lastParent < firstChild, 'all parent records must come before any child records'); + }); + + it('FK integrity holds across all three levels', () => { + const records = collectBySchema( + generateDictionaryRecords(threeLayerDictionary, { + counts: { grandparent: 2, parent: 4, child: 8 }, + seed: SEED, + ...NO_EMPTY, + }), + ); + const gpIds = new Set(records['grandparent']?.map((record) => record['gp_id'])); + const parentIds = new Set(records['parent']?.map((record) => record['p_id'])); + for (const record of records['parent'] ?? []) { + assert.ok( + gpIds.has(record['grandparent_id']), + `parent.grandparent_id '${String(record['grandparent_id'])}' not in grandparent ids`, + ); + } + for (const record of records['child'] ?? []) { + assert.ok( + parentIds.has(record['parent_id']), + `child.parent_id '${String(record['parent_id'])}' not in parent ids`, + ); + } + }); + }); + + describe('cyclic FK dependencies', () => { + // Schema A has a FK to B, and B has a FK to A — a cycle. + // The generator must not hang or throw; it should place both schemas in one tier + // and generate records for both (FK constraints in the cycle will not be enforced + // since neither can be a parent to the other). + const schemaA = { + name: 'cycle_a', + fields: [ + { name: 'a_id', valueType: 'string' as const, unique: true, restrictions: undefined }, + { name: 'b_ref', valueType: 'string' as const, restrictions: undefined }, + ], + restrictions: { + foreignKey: [{ schema: 'cycle_b', mappings: [{ local: 'b_ref', foreign: 'b_id' }] }], + }, + }; + + const schemaB = { + name: 'cycle_b', + fields: [ + { name: 'b_id', valueType: 'string' as const, unique: true, restrictions: undefined }, + { name: 'a_ref', valueType: 'string' as const, restrictions: undefined }, + ], + restrictions: { + foreignKey: [{ schema: 'cycle_a', mappings: [{ local: 'a_ref', foreign: 'a_id' }] }], + }, + }; + + const cyclicDictionary: Dictionary = { + name: 'cyclic', + version: '1.0', + schemas: [schemaA, schemaB], + }; + + it('generates the requested number of records for both schemas without hanging', () => { + const records = collectBySchema( + generateDictionaryRecords(cyclicDictionary, { counts: { cycle_a: 3, cycle_b: 3 }, seed: SEED, ...NO_EMPTY }), + ); + assert.strictEqual(records['cycle_a']?.length, 3, 'expected 3 cycle_a records'); + assert.strictEqual(records['cycle_b']?.length, 3, 'expected 3 cycle_b records'); + }); + }); + + it('produces identical output for the same seed', () => { + const options = { counts: { donor: 3, sample: 5 }, seed: SEED, ...NO_EMPTY }; + const first = [...generateDictionaryRecords(dictionary, options)]; + const second = [...generateDictionaryRecords(dictionary, options)]; + assert.deepStrictEqual(first, second); + }); + + it('produces different output for different seeds', () => { + const counts = { donor: 3, sample: 5 }; + const first = [...generateDictionaryRecords(dictionary, { counts, seed: 1, ...NO_EMPTY })]; + const second = [...generateDictionaryRecords(dictionary, { counts, seed: 99, ...NO_EMPTY })]; + assert.notDeepStrictEqual(first, second); + }); +}); diff --git a/packages/data-generator/test/fieldDependencies.spec.ts b/packages/data-generator/test/fieldDependencies.spec.ts new file mode 100644 index 00000000..4c5dfd05 --- /dev/null +++ b/packages/data-generator/test/fieldDependencies.spec.ts @@ -0,0 +1,380 @@ +/* + * Copyright (c) 2026 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import assert from 'node:assert'; +import { describe, it } from 'mocha'; +import type { Schema } from '@overture-stack/lectern-dictionary'; +import { extractFieldDependencies, resolveGenerationOrder } from '../src/dataGeneration/records/fieldDependencies'; + +describe('extractFieldDependencies', () => { + it('returns empty sets for all fields when no conditional restrictions are present', () => { + const schema: Schema = { + name: 'test', + fields: [ + { name: 'alpha', valueType: 'string', restrictions: undefined }, + { name: 'beta', valueType: 'string', restrictions: undefined }, + { name: 'gamma', valueType: 'string', restrictions: undefined }, + ], + }; + + const dependencyMap = extractFieldDependencies(schema); + assert.deepStrictEqual(dependencyMap.get('alpha'), new Set()); + assert.deepStrictEqual(dependencyMap.get('beta'), new Set()); + assert.deepStrictEqual(dependencyMap.get('gamma'), new Set()); + }); + + it('captures a simple linear dependency: beta depends on alpha', () => { + const schema: Schema = { + name: 'test', + fields: [ + { name: 'alpha', valueType: 'string', restrictions: undefined }, + { + name: 'beta', + valueType: 'string', + restrictions: [ + { + if: { conditions: [{ fields: ['alpha'], match: { value: 'x' } }] }, + then: { codeList: ['yes'] }, + else: { codeList: ['no'] }, + }, + ], + }, + ], + }; + + const dependencyMap = extractFieldDependencies(schema); + assert.deepStrictEqual(dependencyMap.get('alpha'), new Set()); + assert.deepStrictEqual(dependencyMap.get('beta'), new Set(['alpha'])); + }); + + it('captures field names from nested conditional branches', () => { + // 'outer' conditionally references 'first'; its then-branch itself conditionally references 'inner'. + const schema: Schema = { + name: 'test', + fields: [ + { name: 'first', valueType: 'string', restrictions: undefined }, + { name: 'inner', valueType: 'string', restrictions: undefined }, + { + name: 'outer', + valueType: 'string', + restrictions: [ + { + if: { conditions: [{ fields: ['first'], match: { value: 'a' } }] }, + then: [ + { + if: { conditions: [{ fields: ['inner'], match: { value: 'b' } }] }, + then: { codeList: ['deep'] }, + }, + ], + }, + ], + }, + ], + }; + + const dependencyMap = extractFieldDependencies(schema); + assert.deepStrictEqual(dependencyMap.get('outer'), new Set(['first', 'inner'])); + }); + + it('excludes references to fields not defined in the schema', () => { + const schema: Schema = { + name: 'test', + fields: [ + { + name: 'alpha', + valueType: 'string', + restrictions: [ + { + if: { conditions: [{ fields: ['unknown'], match: { value: 'x' } }] }, + then: { codeList: ['yes'] }, + }, + ], + }, + ], + }; + + const dependencyMap = extractFieldDependencies(schema); + assert.deepStrictEqual(dependencyMap.get('alpha'), new Set()); + }); + + it('excludes self-references', () => { + const schema: Schema = { + name: 'test', + fields: [ + { + name: 'alpha', + valueType: 'string', + restrictions: [ + { + if: { conditions: [{ fields: ['alpha'], match: { value: 'x' } }] }, + then: { codeList: ['yes'] }, + }, + ], + }, + ], + }; + + const dependencyMap = extractFieldDependencies(schema); + assert.deepStrictEqual(dependencyMap.get('alpha'), new Set()); + }); +}); + +describe('resolveGenerationOrder', () => { + it('flattens fields with no dependencies into a single tier', () => { + const schema: Schema = { + name: 'test', + fields: [ + { name: 'alpha', valueType: 'string', restrictions: undefined }, + { name: 'beta', valueType: 'string', restrictions: undefined }, + { name: 'gamma', valueType: 'string', restrictions: undefined }, + ], + }; + + const order = resolveGenerationOrder(schema); + assert.strictEqual(order.length, 1); + assert.deepStrictEqual(new Set(order[0]), new Set(['alpha', 'beta', 'gamma'])); + }); + + it('produces two tiers for a linear dependency: alpha first, then beta', () => { + const schema: Schema = { + name: 'test', + fields: [ + { name: 'alpha', valueType: 'string', restrictions: undefined }, + { + name: 'beta', + valueType: 'string', + restrictions: [ + { + if: { conditions: [{ fields: ['alpha'], match: { value: 'x' } }] }, + then: { codeList: ['yes'] }, + }, + ], + }, + ], + }; + + const order = resolveGenerationOrder(schema); + assert.strictEqual(order.length, 2); + assert.deepStrictEqual(order[0], ['alpha']); + assert.deepStrictEqual(order[1], ['beta']); + }); + + it('produces three tiers for a chain: alpha → beta → gamma', () => { + const schema: Schema = { + name: 'test', + fields: [ + { name: 'alpha', valueType: 'string', restrictions: undefined }, + { + name: 'beta', + valueType: 'string', + restrictions: [ + { + if: { conditions: [{ fields: ['alpha'], match: { value: 'x' } }] }, + then: { codeList: ['b'] }, + }, + ], + }, + { + name: 'gamma', + valueType: 'string', + restrictions: [ + { + if: { conditions: [{ fields: ['beta'], match: { value: 'b' } }] }, + then: { codeList: ['c'] }, + }, + ], + }, + ], + }; + + const order = resolveGenerationOrder(schema); + assert.strictEqual(order.length, 3); + assert.deepStrictEqual(order[0], ['alpha']); + assert.deepStrictEqual(order[1], ['beta']); + assert.deepStrictEqual(order[2], ['gamma']); + }); + + it('produces two tiers for a fan-in: alpha and beta in the same tier, gamma after', () => { + // gamma depends on both alpha and beta, which are independent of each other. + const schema: Schema = { + name: 'test', + fields: [ + { name: 'alpha', valueType: 'string', restrictions: undefined }, + { name: 'beta', valueType: 'string', restrictions: undefined }, + { + name: 'gamma', + valueType: 'string', + restrictions: [ + { + if: { conditions: [{ fields: ['alpha', 'beta'], match: { value: 'x' } }] }, + then: { codeList: ['c'] }, + }, + ], + }, + ], + }; + + const order = resolveGenerationOrder(schema); + assert.strictEqual(order.length, 2); + assert.deepStrictEqual(new Set(order[0]), new Set(['alpha', 'beta'])); + assert.deepStrictEqual(order[1], ['gamma']); + }); + + it('places both fields in one tier when there is a two-field cycle', () => { + const schema: Schema = { + name: 'test', + fields: [ + { + name: 'alpha', + valueType: 'string', + restrictions: [ + { + if: { conditions: [{ fields: ['beta'], match: { value: 'b' } }] }, + then: { codeList: ['a'] }, + }, + ], + }, + { + name: 'beta', + valueType: 'string', + restrictions: [ + { + if: { conditions: [{ fields: ['alpha'], match: { value: 'a' } }] }, + then: { codeList: ['b'] }, + }, + ], + }, + ], + }; + + const order = resolveGenerationOrder(schema); + assert.strictEqual(order.length, 1); + assert.deepStrictEqual(new Set(order[0]), new Set(['alpha', 'beta'])); + }); + + it('places all three fields in one tier when there is a three-field cycle', () => { + const schema: Schema = { + name: 'test', + fields: [ + { + name: 'alpha', + valueType: 'string', + restrictions: [ + { + if: { conditions: [{ fields: ['gamma'], match: { value: 'c' } }] }, + then: { codeList: ['a'] }, + }, + ], + }, + { + name: 'beta', + valueType: 'string', + restrictions: [ + { + if: { conditions: [{ fields: ['alpha'], match: { value: 'a' } }] }, + then: { codeList: ['b'] }, + }, + ], + }, + { + name: 'gamma', + valueType: 'string', + restrictions: [ + { + if: { conditions: [{ fields: ['beta'], match: { value: 'b' } }] }, + then: { codeList: ['c'] }, + }, + ], + }, + ], + }; + + const order = resolveGenerationOrder(schema); + assert.strictEqual(order.length, 1); + assert.deepStrictEqual(new Set(order[0]), new Set(['alpha', 'beta', 'gamma'])); + }); + + it('places only cyclic fields in the cycle tier, leaving independent fields in their own tier', () => { + // alpha and beta form a cycle. gamma has no dependencies and must appear in its own tier + // before the cycle tier - it must not be lumped together with the cyclic fields. + const schema: Schema = { + name: 'test', + fields: [ + { + name: 'alpha', + valueType: 'string', + restrictions: [ + { + if: { conditions: [{ fields: ['beta'], match: { value: 'b' } }] }, + then: { codeList: ['a'] }, + }, + ], + }, + { + name: 'beta', + valueType: 'string', + restrictions: [ + { + if: { conditions: [{ fields: ['alpha'], match: { value: 'a' } }] }, + then: { codeList: ['b'] }, + }, + ], + }, + { name: 'gamma', valueType: 'string', restrictions: undefined }, + ], + }; + + const order = resolveGenerationOrder(schema); + // gamma has no dependencies - it must be in an earlier tier than the cyclic alpha+beta pair. + const gammaPosition = order.findIndex((tier) => tier.includes('gamma')); + const alphaPosition = order.findIndex((tier) => tier.includes('alpha')); + const betaPosition = order.findIndex((tier) => tier.includes('beta')); + assert.ok(gammaPosition < alphaPosition, 'gamma must be in an earlier tier than alpha'); + assert.strictEqual(alphaPosition, betaPosition, 'alpha and beta must be in the same (cycle) tier'); + // gamma must be alone in its tier - not lumped with the cyclic fields. + assert.ok(order[gammaPosition]?.length === 1, 'gamma tier should contain only gamma'); + }); + + it('correctly orders fields when the dependent appears before its dependency in schema.fields', () => { + // beta (index 0) depends on alpha (index 1). Generation order must be alpha before beta + // even though beta comes first in schema.fields. + const schema: Schema = { + name: 'test', + fields: [ + { + name: 'beta', + valueType: 'string', + restrictions: [ + { + if: { conditions: [{ fields: ['alpha'], match: { value: 'x' } }] }, + then: { codeList: ['yes'] }, + else: { codeList: ['no'] }, + }, + ], + }, + { name: 'alpha', valueType: 'string', restrictions: { codeList: ['x', 'y'] } }, + ], + }; + + const order = resolveGenerationOrder(schema); + const alphaPosition = order.findIndex((tier) => tier.includes('alpha')); + const betaPosition = order.findIndex((tier) => tier.includes('beta')); + assert.ok(alphaPosition < betaPosition, 'alpha must be in an earlier tier than beta'); + }); +}); diff --git a/packages/data-generator/test/fieldGenerators.spec.ts b/packages/data-generator/test/fieldGenerators.spec.ts new file mode 100644 index 00000000..75747cd1 --- /dev/null +++ b/packages/data-generator/test/fieldGenerators.spec.ts @@ -0,0 +1,657 @@ +/* + * Copyright (c) 2026 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import assert from 'node:assert'; +import { + generateBooleanValue, + generateIntegerValue, + generateNumberValue, + generateStringValue, +} from '../src/dataGeneration/fields/fieldGenerators'; + +const SEED = 42; +// Tests that assert on concrete value types must opt out of the default empty rate. +const NO_EMPTY = { emptyRate: 0 } as const; + +describe('generateBooleanValue', () => { + const baseField = { name: 'active', valueType: 'boolean' } as const; + + it('returns a boolean', () => { + const result = generateBooleanValue(baseField, { seed: SEED, ...NO_EMPTY }); + assert.ok(result.success); + assert.strictEqual(typeof result.data, 'boolean'); + }); + + it('returns an array of booleans when isArray is true', () => { + const result = generateBooleanValue({ ...baseField, isArray: true }, { seed: SEED, ...NO_EMPTY }); + assert.ok(result.success); + assert.ok(Array.isArray(result.data)); + for (const element of result.data as boolean[]) { + assert.strictEqual(typeof element, 'boolean'); + } + }); + + it('returns the same value for the same seed', () => { + const first = generateBooleanValue(baseField, { seed: SEED, ...NO_EMPTY }); + const second = generateBooleanValue(baseField, { seed: SEED, ...NO_EMPTY }); + assert.deepStrictEqual(first, second); + }); +}); + +describe('generateIntegerValue', () => { + const baseField = { name: 'count', valueType: 'integer' } as const; + + it('returns an integer', () => { + const result = generateIntegerValue(baseField, { seed: SEED, ...NO_EMPTY }); + assert.ok(result.success); + assert.ok(typeof result.data === 'number' && Number.isInteger(result.data)); + }); + + it('returns an array of integers when isArray is true', () => { + const result = generateIntegerValue({ ...baseField, isArray: true }, { seed: SEED, ...NO_EMPTY }); + assert.ok(result.success); + assert.ok(Array.isArray(result.data)); + for (const element of result.data as number[]) { + assert.ok(typeof element === 'number' && Number.isInteger(element)); + } + }); + + it('returns an array of the specified length when arrayLength is a number', () => { + const result = generateIntegerValue({ ...baseField, isArray: true }, { seed: SEED, ...NO_EMPTY, arrayLength: 5 }); + assert.ok(result.success); + assert.ok(Array.isArray(result.data)); + assert.strictEqual((result.data as number[]).length, 5); + }); + + it('returns an array whose length falls within a range when arrayLength is a RestrictionRange', () => { + for (let seed = 0; seed < 10; seed++) { + const result = generateIntegerValue( + { ...baseField, isArray: true }, + { seed, ...NO_EMPTY, arrayLength: { min: 4, max: 6 } }, + ); + assert.ok(result.success); + const length = (result.data as number[]).length; + assert.ok(length >= 4 && length <= 6, `array length ${length} outside [4, 6]`); + } + }); + + it('derives integer array length correctly from fractional exclusiveMin', () => { + for (let seed = 0; seed < 10; seed++) { + const result = generateIntegerValue( + { ...baseField, isArray: true }, + { seed, ...NO_EMPTY, arrayLength: { exclusiveMin: 2.5, max: 6 } }, + ); + assert.ok(result.success); + const length = (result.data as number[]).length; + assert.ok(length >= 3 && length <= 6, `array length ${length} outside [3, 6]`); + } + }); + + it('derives integer array length correctly from fractional exclusiveMax', () => { + for (let seed = 0; seed < 10; seed++) { + const result = generateIntegerValue( + { ...baseField, isArray: true }, + { seed, ...NO_EMPTY, arrayLength: { min: 1, exclusiveMax: 4.7 } }, + ); + assert.ok(result.success); + const length = (result.data as number[]).length; + assert.ok(length >= 1 && length <= 4, `array length ${length} outside [1, 4]`); + } + }); + + it('returns a single-element array when exclusive integer bounds leave no valid length', () => { + // exclusiveMin: 1, exclusiveMax: 2 → min=2, max=1 after arithmetic — impossible integer range. + // The guard should return DEFAULT_ARRAY_MIN (1) instead of throwing. + for (let seed = 0; seed < 10; seed++) { + const result = generateIntegerValue( + { ...baseField, isArray: true }, + { seed, ...NO_EMPTY, arrayLength: { exclusiveMin: 1, exclusiveMax: 2 } }, + ); + assert.ok(result.success); + assert.ok(Array.isArray(result.data)); + assert.strictEqual((result.data as number[]).length, 1); + } + }); + + it('returns a value from codeList when codeList restriction is present', () => { + const codeList = [10, 20, 30]; + const field = { ...baseField, restrictions: { codeList } }; + for (let seed = 0; seed < 20; seed++) { + const result = generateIntegerValue(field, { seed, ...NO_EMPTY }); + assert.ok(result.success); + assert.ok(codeList.includes(result.data as number), `${result.data} not in codeList`); + } + }); + + it('returns a value within range when range restriction is present', () => { + const field = { ...baseField, restrictions: { range: { min: 5, max: 10 } } }; + for (let seed = 0; seed < 20; seed++) { + const result = generateIntegerValue(field, { seed, ...NO_EMPTY }); + assert.ok(result.success); + const value = result.data as number; + assert.ok(value >= 5 && value <= 10, `${value} outside [5, 10]`); + } + }); + + it('respects exclusiveMin and exclusiveMax', () => { + const field = { ...baseField, restrictions: { range: { exclusiveMin: 0, exclusiveMax: 5 } } }; + for (let seed = 0; seed < 20; seed++) { + const result = generateIntegerValue(field, { seed, ...NO_EMPTY }); + assert.ok(result.success); + const value = result.data as number; + assert.ok(value >= 1 && value <= 4, `${value} outside (0, 5)`); + } + }); + + it('applies the then branch of a conditional restriction when the condition passes', () => { + const field = { + ...baseField, + restrictions: { + if: { conditions: [{ fields: ['status'], match: { value: 'active' } }] }, + then: { range: { min: 100, max: 200 } }, + else: { range: { min: 0, max: 10 } }, + }, + }; + const activeResult = generateIntegerValue(field, { seed: SEED, ...NO_EMPTY, record: { status: 'active' } }); + assert.ok(activeResult.success); + const valueWhenActive = activeResult.data as number; + assert.ok(valueWhenActive >= 100 && valueWhenActive <= 200, `${valueWhenActive} not in [100, 200]`); + + const inactiveResult = generateIntegerValue(field, { seed: SEED, ...NO_EMPTY, record: { status: 'inactive' } }); + assert.ok(inactiveResult.success); + const valueWhenInactive = inactiveResult.data as number; + assert.ok(valueWhenInactive >= 0 && valueWhenInactive <= 10, `${valueWhenInactive} not in [0, 10]`); + }); + + it('treats missing condition fields as undefined, taking the else branch', () => { + const field = { + ...baseField, + restrictions: { + if: { conditions: [{ fields: ['status'], match: { exists: true } }] }, + then: { range: { min: 100, max: 200 } }, + else: { range: { min: 0, max: 10 } }, + }, + }; + const result = generateIntegerValue(field, { seed: SEED, ...NO_EMPTY, record: {} }); + assert.ok(result.success); + const value = result.data as number; + assert.ok(value >= 0 && value <= 10, `${value} not in else range [0, 10]`); + }); + + it('takes the then branch when case:any and at least one condition passes', () => { + const field = { + ...baseField, + restrictions: { + if: { + case: 'any' as const, + conditions: [ + { fields: ['a'], match: { value: 'yes' } }, + { fields: ['b'], match: { value: 'yes' } }, + ], + }, + then: { range: { min: 100, max: 200 } }, + else: { range: { min: 0, max: 10 } }, + }, + }; + const result = generateIntegerValue(field, { seed: SEED, ...NO_EMPTY, record: { a: 'no', b: 'yes' } }); + assert.ok(result.success); + const value = result.data as number; + assert.ok(value >= 100 && value <= 200, `${value} not in then range [100, 200]`); + }); + + it('takes the then branch when case:none and no condition passes', () => { + const field = { + ...baseField, + restrictions: { + if: { + case: 'none' as const, + conditions: [ + { fields: ['a'], match: { value: 'yes' } }, + { fields: ['b'], match: { value: 'yes' } }, + ], + }, + then: { range: { min: 100, max: 200 } }, + else: { range: { min: 0, max: 10 } }, + }, + }; + const result = generateIntegerValue(field, { seed: SEED, ...NO_EMPTY, record: { a: 'no', b: 'no' } }); + assert.ok(result.success); + const value = result.data as number; + assert.ok(value >= 100 && value <= 200, `${value} not in then range [100, 200]`); + }); + + it('returns a value within the intersection when two compatible ranges are present', () => { + const field = { + ...baseField, + restrictions: [{ range: { min: 0, max: 20 } }, { range: { min: 10, max: 30 } }], + }; + for (let seed = 0; seed < 20; seed++) { + const result = generateIntegerValue(field, { seed, ...NO_EMPTY }); + assert.ok(result.success); + const value = result.data as number; + assert.ok(value >= 10 && value <= 20, `${value} outside intersection [10, 20]`); + } + }); + + it('returns success with a fallback value when multiple ranges conflict', () => { + const field = { + ...baseField, + restrictions: [{ range: { min: 0, max: 5 } }, { range: { min: 10, max: 20 } }], + }; + const result = generateIntegerValue(field, { seed: SEED, ...NO_EMPTY }); + assert.ok(!result.success, 'expected failure due to conflicting ranges'); + assert.strictEqual(result.data.conflicts[0]?.type, 'range'); + assert.ok(typeof result.data.value === 'number', 'fallback value should still be a number'); + }); + + it('returns a value satisfying both codeList and range when they are compatible', () => { + const field = { + ...baseField, + restrictions: [{ codeList: [1, 5, 10, 50] }, { range: { min: 5, max: 15 } }], + }; + for (let seed = 0; seed < 20; seed++) { + const result = generateIntegerValue(field, { seed, ...NO_EMPTY }); + assert.ok(result.success); + const value = result.data as number; + assert.ok([5, 10].includes(value), `${value} not in intersection of codeList and range`); + } + }); + + it('returns failure when no codeList value satisfies the range', () => { + const field = { + ...baseField, + restrictions: [{ codeList: [1, 2, 3] }, { range: { min: 10, max: 20 } }], + }; + const result = generateIntegerValue(field, { seed: SEED, ...NO_EMPTY }); + assert.ok(!result.success, 'expected failure because no codeList value is in range'); + assert.ok(typeof result.data.value === 'number', 'fallback value should still be a number'); + assert.ok([1, 2, 3].includes(result.data.value as number), 'fallback value should come from the codeList'); + }); + + it('returns the same value for the same seed', () => { + const first = generateIntegerValue(baseField, { seed: SEED, ...NO_EMPTY }); + const second = generateIntegerValue(baseField, { seed: SEED, ...NO_EMPTY }); + assert.deepStrictEqual(first, second); + }); +}); + +describe('generateNumberValue', () => { + const baseField = { name: 'score', valueType: 'number' } as const; + + it('returns a number', () => { + const result = generateNumberValue(baseField, { seed: SEED, ...NO_EMPTY }); + assert.ok(result.success); + assert.strictEqual(typeof result.data, 'number'); + }); + + it('returns an array of numbers when isArray is true', () => { + const result = generateNumberValue({ ...baseField, isArray: true }, { seed: SEED, ...NO_EMPTY }); + assert.ok(result.success); + assert.ok(Array.isArray(result.data)); + for (const element of result.data as number[]) { + assert.strictEqual(typeof element, 'number'); + } + }); + + it('returns an array of the specified length when arrayLength is a number', () => { + const result = generateNumberValue({ ...baseField, isArray: true }, { seed: SEED, ...NO_EMPTY, arrayLength: 7 }); + assert.ok(result.success); + assert.strictEqual((result.data as number[]).length, 7); + }); + + it('returns an array whose length falls within a range when arrayLength is a RestrictionRange', () => { + for (let seed = 0; seed < 10; seed++) { + const result = generateNumberValue( + { ...baseField, isArray: true }, + { seed, ...NO_EMPTY, arrayLength: { min: 2, max: 4 } }, + ); + assert.ok(result.success); + const length = (result.data as number[]).length; + assert.ok(length >= 2 && length <= 4, `array length ${length} outside [2, 4]`); + } + }); + + it('returns a value from codeList when codeList restriction is present', () => { + const codeList = [1.1, 2.2, 3.3]; + const field = { ...baseField, restrictions: { codeList } }; + for (let seed = 0; seed < 20; seed++) { + const result = generateNumberValue(field, { seed, ...NO_EMPTY }); + assert.ok(result.success); + assert.ok(codeList.includes(result.data as number), `${result.data} not in codeList`); + } + }); + + it('returns a value within range when range restriction is present', () => { + const field = { ...baseField, restrictions: { range: { min: 0, max: 1 } } }; + for (let seed = 0; seed < 20; seed++) { + const result = generateNumberValue(field, { seed, ...NO_EMPTY }); + assert.ok(result.success); + const value = result.data as number; + assert.ok(value >= 0 && value <= 1, `${value} outside [0, 1]`); + } + }); + + it('respects exclusiveMin and exclusiveMax', () => { + const field = { ...baseField, restrictions: { range: { exclusiveMin: 0, exclusiveMax: 1 } } }; + for (let seed = 0; seed < 20; seed++) { + const result = generateNumberValue(field, { seed, ...NO_EMPTY }); + assert.ok(result.success); + const value = result.data as number; + assert.ok(value > 0 && value < 1, `${value} outside (0, 1)`); + } + }); + + it('returns a value within the intersection when two compatible ranges are present', () => { + const field = { + ...baseField, + restrictions: [{ range: { min: 0, max: 10 } }, { range: { min: 5, max: 20 } }], + }; + for (let seed = 0; seed < 20; seed++) { + const result = generateNumberValue(field, { seed, ...NO_EMPTY }); + assert.ok(result.success); + const value = result.data as number; + assert.ok(value >= 5 && value <= 10, `${value} outside intersection [5, 10]`); + } + }); + + it('returns failure with a fallback value when multiple ranges conflict', () => { + const field = { + ...baseField, + restrictions: [{ range: { min: 0, max: 5 } }, { range: { min: 10, max: 20 } }], + }; + const result = generateNumberValue(field, { seed: SEED, ...NO_EMPTY }); + assert.ok(!result.success, 'expected failure due to conflicting ranges'); + assert.strictEqual(result.data.conflicts[0]?.type, 'range'); + assert.ok(typeof result.data.value === 'number', 'fallback value should still be a number'); + }); + + it('applies conditional restriction branch based on record', () => { + const field = { + ...baseField, + restrictions: { + if: { conditions: [{ fields: ['category'], match: { value: 'high' } }] }, + then: { range: { min: 10, max: 20 } }, + else: { range: { min: 0, max: 5 } }, + }, + }; + const highResult = generateNumberValue(field, { seed: SEED, ...NO_EMPTY, record: { category: 'high' } }); + assert.ok(highResult.success); + const high = highResult.data as number; + assert.ok(high >= 10 && high <= 20, `${high} not in [10, 20]`); + + const lowResult = generateNumberValue(field, { seed: SEED, ...NO_EMPTY, record: { category: 'low' } }); + assert.ok(lowResult.success); + const low = lowResult.data as number; + assert.ok(low >= 0 && low <= 5, `${low} not in [0, 5]`); + }); + + it('returns success with a fallback value when multiple codeLists conflict', () => { + const field = { + ...baseField, + restrictions: [{ codeList: [1.1, 2.2] }, { codeList: [3.3, 4.4] }], + }; + const result = generateNumberValue(field, { seed: SEED, ...NO_EMPTY }); + assert.ok(!result.success, 'expected failure due to disjoint codeLists'); + assert.strictEqual(result.data.conflicts[0]?.type, 'codeList'); + assert.ok(typeof result.data.value === 'number', 'fallback value should still be a number'); + }); + + it('returns a value satisfying both codeList and range when they are compatible', () => { + const field = { + ...baseField, + restrictions: [{ codeList: [0.5, 1.5, 5.0, 10.0] }, { range: { min: 1, max: 6 } }], + }; + const validValues = [1.5, 5.0]; + for (let seed = 0; seed < 20; seed++) { + const result = generateNumberValue(field, { seed, ...NO_EMPTY }); + assert.ok(result.success); + const value = result.data as number; + assert.ok(validValues.includes(value), `${value} not in intersection of codeList and range`); + } + }); + + it('returns failure when no codeList value satisfies the range', () => { + const field = { + ...baseField, + restrictions: [{ codeList: [0.1, 0.5, 0.9] }, { range: { min: 5, max: 10 } }], + }; + const result = generateNumberValue(field, { seed: SEED, ...NO_EMPTY }); + assert.ok(!result.success, 'expected failure because no codeList value is in range'); + assert.ok(typeof result.data.value === 'number', 'fallback value should still be a number'); + assert.ok([0.1, 0.5, 0.9].includes(result.data.value as number), 'fallback value should come from the codeList'); + }); + + it('returns the same value for the same seed', () => { + const first = generateNumberValue(baseField, { seed: SEED, ...NO_EMPTY }); + const second = generateNumberValue(baseField, { seed: SEED, ...NO_EMPTY }); + assert.deepStrictEqual(first, second); + }); +}); + +describe('generateStringValue', () => { + const baseField = { name: 'label', valueType: 'string' } as const; + + it('returns a string', () => { + const result = generateStringValue(baseField, { seed: SEED, ...NO_EMPTY }); + assert.ok(result.success); + assert.strictEqual(typeof result.data, 'string'); + }); + + it('returns an array of strings when isArray is true', () => { + const result = generateStringValue({ ...baseField, isArray: true }, { seed: SEED, ...NO_EMPTY }); + assert.ok(result.success); + assert.ok(Array.isArray(result.data)); + for (const element of result.data as string[]) { + assert.strictEqual(typeof element, 'string'); + } + }); + + it('returns an array of the specified length when arrayLength is a number', () => { + const result = generateStringValue({ ...baseField, isArray: true }, { seed: SEED, ...NO_EMPTY, arrayLength: 4 }); + assert.ok(result.success); + assert.strictEqual((result.data as string[]).length, 4); + }); + + it('returns an array whose length falls within a range when arrayLength is a RestrictionRange', () => { + for (let seed = 0; seed < 10; seed++) { + const result = generateStringValue( + { ...baseField, isArray: true }, + { seed, ...NO_EMPTY, arrayLength: { min: 3, max: 5 } }, + ); + assert.ok(result.success); + const length = (result.data as string[]).length; + assert.ok(length >= 3 && length <= 5, `array length ${length} outside [3, 5]`); + } + }); + + it('returns a value from codeList when codeList restriction is present', () => { + const codeList = ['alpha', 'beta', 'gamma']; + const field = { ...baseField, restrictions: { codeList } }; + for (let seed = 0; seed < 20; seed++) { + const result = generateStringValue(field, { seed, ...NO_EMPTY }); + assert.ok(result.success); + assert.ok(codeList.includes(result.data as string), `"${result.data}" not in codeList`); + } + }); + + it('returns a string matching the regex restriction', () => { + const pattern = '^[A-Z]{2}\\d{4}$'; + const field = { ...baseField, restrictions: { regex: pattern } }; + const regex = new RegExp(pattern); + for (let seed = 0; seed < 20; seed++) { + const result = generateStringValue(field, { seed, ...NO_EMPTY }); + assert.ok(result.success); + assert.ok(regex.test(result.data as string), `"${result.data}" does not match ${pattern}`); + } + }); + + it('skips ReferenceTag entries in codeList and falls back to arbitrary string', () => { + const field = { ...baseField, restrictions: { codeList: ['#/references/codes'] } }; + const result = generateStringValue(field, { seed: SEED, ...NO_EMPTY }); + assert.ok(result.success); + assert.strictEqual(typeof result.data, 'string'); + }); + + it('applies the then codeList when condition passes', () => { + const thenList = ['yes', 'true']; + const elseList = ['no', 'false']; + const field = { + ...baseField, + restrictions: { + if: { conditions: [{ fields: ['enabled'], match: { value: true } }] }, + then: { codeList: thenList }, + else: { codeList: elseList }, + }, + }; + for (let seed = 0; seed < 10; seed++) { + const enabledResult = generateStringValue(field, { seed, ...NO_EMPTY, record: { enabled: true } }); + assert.ok(enabledResult.success); + assert.ok(thenList.includes(enabledResult.data as string), `"${enabledResult.data}" not in then codeList`); + + const disabledResult = generateStringValue(field, { seed, ...NO_EMPTY, record: { enabled: false } }); + assert.ok(disabledResult.success); + assert.ok(elseList.includes(disabledResult.data as string), `"${disabledResult.data}" not in else codeList`); + } + }); + + it('treats missing condition field as undefined, taking the else branch', () => { + const field = { + ...baseField, + restrictions: { + if: { conditions: [{ fields: ['type'], match: { exists: true } }] }, + then: { codeList: ['A', 'B'] }, + else: { codeList: ['X', 'Y'] }, + }, + }; + for (let seed = 0; seed < 10; seed++) { + const result = generateStringValue(field, { seed, ...NO_EMPTY, record: {} }); + assert.ok(result.success); + assert.ok(['X', 'Y'].includes(result.data as string), `"${result.data}" not in else codeList`); + } + }); + + it('returns success with a fallback value when codeLists from two conditional branches conflict', () => { + const field = { + ...baseField, + restrictions: [{ codeList: ['alpha', 'beta'] }, { codeList: ['gamma', 'delta'] }], + }; + const result = generateStringValue(field, { seed: SEED, ...NO_EMPTY }); + assert.ok(!result.success, 'expected failure due to disjoint codeLists'); + assert.strictEqual(result.data.conflicts[0]?.type, 'codeList'); + assert.ok(typeof result.data.value === 'string', 'fallback value should still be a string'); + }); + + it('returns a value satisfying both codeList and regex when they are compatible', () => { + const field = { + ...baseField, + restrictions: [{ codeList: ['abc', 'ABC', 'xyz', 'XYZ'] }, { regex: '^[A-Z]+$' }], + }; + const validValues = ['ABC', 'XYZ']; + for (let seed = 0; seed < 20; seed++) { + const result = generateStringValue(field, { seed, ...NO_EMPTY }); + assert.ok(result.success); + assert.ok( + validValues.includes(result.data as string), + `"${result.data}" not in intersection of codeList and regex`, + ); + } + }); + + it('returns failure when no codeList value satisfies the regex', () => { + const field = { + ...baseField, + restrictions: [{ codeList: ['abc', 'xyz'] }, { regex: '^[0-9]+$' }], + }; + const result = generateStringValue(field, { seed: SEED, ...NO_EMPTY }); + assert.ok(!result.success, 'expected failure because no codeList value matches the regex'); + assert.ok(typeof result.data.value === 'string', 'fallback value should still be a string'); + assert.ok(['abc', 'xyz'].includes(result.data.value as string), 'fallback value should come from the codeList'); + }); + + it('returns the same value for the same seed', () => { + const first = generateStringValue(baseField, { seed: SEED, ...NO_EMPTY }); + const second = generateStringValue(baseField, { seed: SEED, ...NO_EMPTY }); + assert.deepStrictEqual(first, second); + }); +}); + +describe('emptyRate', () => { + const boolField = { name: 'b', valueType: 'boolean' as const, restrictions: undefined }; + const intField = { name: 'i', valueType: 'integer' as const, restrictions: undefined }; + const numField = { name: 'n', valueType: 'number' as const, restrictions: undefined }; + const strField = { name: 's', valueType: 'string' as const, restrictions: undefined }; + const requiredStrField = { + name: 's', + valueType: 'string' as const, + restrictions: { required: true }, + }; + + it('returns undefined for every seed when emptyRate is 1', () => { + for (let seed = 0; seed < 20; seed++) { + assert.strictEqual(generateBooleanValue(boolField, { seed, emptyRate: 1 }).data, undefined); + assert.strictEqual(generateIntegerValue(intField, { seed, emptyRate: 1 }).data, undefined); + assert.strictEqual(generateNumberValue(numField, { seed, emptyRate: 1 }).data, undefined); + assert.strictEqual(generateStringValue(strField, { seed, emptyRate: 1 }).data, undefined); + } + }); + + it('never returns undefined when emptyRate is 0', () => { + for (let seed = 0; seed < 20; seed++) { + assert.notStrictEqual(generateBooleanValue(boolField, { seed, emptyRate: 0 }).data, undefined); + assert.notStrictEqual(generateIntegerValue(intField, { seed, emptyRate: 0 }).data, undefined); + assert.notStrictEqual(generateNumberValue(numField, { seed, emptyRate: 0 }).data, undefined); + assert.notStrictEqual(generateStringValue(strField, { seed, emptyRate: 0 }).data, undefined); + } + }); + + it('never returns undefined for a required field regardless of emptyRate', () => { + for (let seed = 0; seed < 20; seed++) { + const result = generateStringValue(requiredStrField, { seed, emptyRate: 1 }); + assert.notStrictEqual(result.data, undefined); + } + }); + + it('clamps emptyRate values outside [0, 1]', () => { + for (let seed = 0; seed < 20; seed++) { + assert.strictEqual( + generateStringValue(strField, { seed, emptyRate: 999 }).data, + undefined, + 'values > 1 should clamp to 1', + ); + assert.notStrictEqual( + generateStringValue(strField, { seed, emptyRate: -999 }).data, + undefined, + 'values < 0 should clamp to 0', + ); + } + }); + + it('produces undefined for approximately the expected fraction of seeds at the default rate', () => { + const results = Array.from({ length: 200 }, (_, seed) => generateStringValue(strField, { seed })); + const emptyCount = results.filter((result) => result.data === undefined).length; + // With default rate 0.25 and 200 samples, expect roughly 50 ± 30 empty values. + assert.ok(emptyCount > 20 && emptyCount < 80, `expected ~50 empty values, got ${emptyCount}`); + }); + + it('returns the same result for the same seed (empty check is reproducible)', () => { + const first = generateStringValue(strField, { seed: SEED }); + const second = generateStringValue(strField, { seed: SEED }); + assert.deepStrictEqual(first, second); + }); +}); diff --git a/packages/data-generator/test/recordGenerator.spec.ts b/packages/data-generator/test/recordGenerator.spec.ts new file mode 100644 index 00000000..8fb29f10 --- /dev/null +++ b/packages/data-generator/test/recordGenerator.spec.ts @@ -0,0 +1,302 @@ +/* + * Copyright (c) 2026 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import assert from 'node:assert'; +import { describe, it } from 'mocha'; +import type { Schema } from '@overture-stack/lectern-dictionary'; +import { generateRecord, type ForeignKeyPool } from '../src/dataGeneration/records/recordGenerator'; + +const SEED = 42; +const NO_EMPTY = { emptyRate: 0 } as const; + +const schema: Schema = { + name: 'test', + fields: [ + { name: 'boolField', valueType: 'boolean', restrictions: undefined }, + { name: 'intField', valueType: 'integer', restrictions: undefined }, + { name: 'numField', valueType: 'number', restrictions: undefined }, + { name: 'strField', valueType: 'string', restrictions: undefined }, + ], +}; + +const schemaWithRestrictions: Schema = { + name: 'restricted', + fields: [ + { + name: 'status', + valueType: 'string', + restrictions: { codeList: ['active', 'inactive', 'pending'] }, + }, + { + name: 'score', + valueType: 'integer', + restrictions: { range: { min: 0, max: 100 } }, + }, + { + name: 'rating', + valueType: 'number', + restrictions: { range: { min: 0, max: 5 } }, + }, + ], +}; + +const schemaWithConditional: Schema = { + name: 'conditional', + fields: [ + { + name: 'type', + valueType: 'string', + restrictions: { codeList: ['A', 'B'] }, + }, + { + name: 'label', + valueType: 'string', + restrictions: [ + { + if: { conditions: [{ fields: ['type'], match: { value: 'A' } }] }, + then: { codeList: ['alpha'] }, + else: { codeList: ['beta'] }, + }, + ], + }, + ], +}; + +describe('generateRecord', () => { + it('returns a record with a key for every field in the schema', () => { + const record = generateRecord(schema, { seed: SEED, ...NO_EMPTY }); + for (const field of schema.fields) { + assert.ok(Object.hasOwn(record, field.name), `missing field: ${field.name}`); + } + }); + + it('generates a boolean for boolean fields', () => { + const record = generateRecord(schema, { seed: SEED, ...NO_EMPTY }); + assert.strictEqual(typeof record['boolField'], 'boolean'); + }); + + it('generates an integer number for integer fields', () => { + const record = generateRecord(schema, { seed: SEED, ...NO_EMPTY }); + const value = record['intField']; + assert.strictEqual(typeof value, 'number'); + assert.ok(Number.isInteger(value)); + }); + + it('generates a number for number fields', () => { + const record = generateRecord(schema, { seed: SEED, ...NO_EMPTY }); + assert.strictEqual(typeof record['numField'], 'number'); + }); + + it('generates a string for string fields', () => { + const record = generateRecord(schema, { seed: SEED, ...NO_EMPTY }); + assert.strictEqual(typeof record['strField'], 'string'); + }); + + it('respects codeList restrictions', () => { + const record = generateRecord(schemaWithRestrictions, { seed: SEED, ...NO_EMPTY }); + assert.ok(['active', 'inactive', 'pending'].includes(record['status'] as string)); + }); + + it('respects range restrictions on integer fields', () => { + const record = generateRecord(schemaWithRestrictions, { seed: SEED, ...NO_EMPTY }); + const score = record['score'] as number; + assert.ok(score >= 0 && score <= 100); + }); + + it('respects range restrictions on number fields', () => { + const record = generateRecord(schemaWithRestrictions, { seed: SEED, ...NO_EMPTY }); + const rating = record['rating'] as number; + assert.ok(rating >= 0 && rating <= 5); + }); + + it('uses override values directly without generating', () => { + const record = generateRecord(schema, { seed: SEED, ...NO_EMPTY, overrides: { strField: 'forced' } }); + assert.strictEqual(record['strField'], 'forced'); + }); + + it('does not overwrite non-overridden fields', () => { + const record = generateRecord(schema, { seed: SEED, ...NO_EMPTY, overrides: { strField: 'forced' } }); + assert.strictEqual(typeof record['boolField'], 'boolean'); + assert.strictEqual(typeof record['intField'], 'number'); + }); + + it('produces identical records for the same seed', () => { + const first = generateRecord(schema, { seed: SEED, ...NO_EMPTY }); + const second = generateRecord(schema, { seed: SEED, ...NO_EMPTY }); + assert.deepStrictEqual(first, second); + }); + + it('produces different records for different seeds', () => { + const records = Array.from({ length: 10 }, (_, index) => generateRecord(schema, { seed: index, ...NO_EMPTY })); + const serialized = records.map((record) => JSON.stringify(record)); + const unique = new Set(serialized); + assert.ok(unique.size > 1, 'expected at least some records to differ across seeds'); + }); + + it('threads the partial record into later field generators for conditional restriction resolution', () => { + // label's codeList depends on the value of type, which is generated first. + // With a fixed seed, the same type value is always generated, so the conditional + // always resolves the same way. We run both branches by fixing the override. + const recordWithTypeA = generateRecord(schemaWithConditional, { + seed: SEED, + ...NO_EMPTY, + overrides: { type: 'A' }, + }); + assert.strictEqual(recordWithTypeA['label'], 'alpha'); + + const recordWithTypeB = generateRecord(schemaWithConditional, { + seed: SEED, + ...NO_EMPTY, + overrides: { type: 'B' }, + }); + assert.strictEqual(recordWithTypeB['label'], 'beta'); + }); + + it('works with a schema that has no restrictions', () => { + assert.doesNotThrow(() => generateRecord(schema)); + }); + + describe('foreignKeyPool', () => { + const childSchema: Schema = { + name: 'sample', + fields: [ + { name: 'donor_id', valueType: 'string', restrictions: undefined }, + { name: 'sample_type', valueType: 'string', restrictions: undefined }, + ], + restrictions: { + foreignKey: [ + { + schema: 'donor', + mappings: [{ local: 'donor_id', foreign: 'id' }], + }, + ], + }, + }; + + const compositeFkSchema: Schema = { + name: 'sample', + fields: [ + { name: 'donor_id', valueType: 'string', restrictions: undefined }, + { name: 'program_id', valueType: 'string', restrictions: undefined }, + { name: 'sample_type', valueType: 'string', restrictions: undefined }, + ], + restrictions: { + foreignKey: [ + { + schema: 'donor', + mappings: [ + { local: 'donor_id', foreign: 'id' }, + { local: 'program_id', foreign: 'program' }, + ], + }, + ], + }, + }; + + it('assigns the FK local field from the matching foreign field in the selected parent row', () => { + const pool: ForeignKeyPool = new Map([['donor', [{ id: 'D001' }]]]); + const record = generateRecord(childSchema, { seed: SEED, ...NO_EMPTY, foreignKeyPool: pool }); + assert.strictEqual(record['donor_id'], 'D001'); + }); + + it('assigns all local fields in a composite FK rule from the same selected parent row', () => { + const pool: ForeignKeyPool = new Map([ + [ + 'donor', + [ + { id: 'D001', program: 'PROG-A' }, + { id: 'D002', program: 'PROG-B' }, + ], + ], + ]); + const record = generateRecord(compositeFkSchema, { seed: SEED, ...NO_EMPTY, foreignKeyPool: pool }); + const donorId = record['donor_id']; + const programId = record['program_id']; + // Both fields must come from the same row. + assert.ok( + (donorId === 'D001' && programId === 'PROG-A') || (donorId === 'D002' && programId === 'PROG-B'), + `donor_id=${String(donorId)} and program_id=${String(programId)} do not belong to the same parent row`, + ); + }); + + it('generates the FK field normally when no pool entry exists for the parent schema', () => { + const pool: ForeignKeyPool = new Map(); + const record = generateRecord(childSchema, { seed: SEED, ...NO_EMPTY, foreignKeyPool: pool }); + assert.strictEqual(typeof record['donor_id'], 'string'); + }); + + it('produces identical records for the same seed when a pool is provided', () => { + const pool: ForeignKeyPool = new Map([['donor', [{ id: 'D001' }, { id: 'D002' }, { id: 'D003' }]]]); + const first = generateRecord(childSchema, { seed: SEED, ...NO_EMPTY, foreignKeyPool: pool }); + const second = generateRecord(childSchema, { seed: SEED, ...NO_EMPTY, foreignKeyPool: pool }); + assert.deepStrictEqual(first, second); + }); + + it('explicit overrides take priority over FK pool values', () => { + const pool: ForeignKeyPool = new Map([['donor', [{ id: 'D001' }]]]); + const record = generateRecord(childSchema, { + seed: SEED, + foreignKeyPool: pool, + overrides: { donor_id: 'OVERRIDE' }, + }); + assert.strictEqual(record['donor_id'], 'OVERRIDE'); + }); + }); + + it('correctly resolves conditional restrictions when the dependent field appears before its dependency in schema.fields', () => { + // 'label' (index 0) has a conditional restriction referencing 'type' (index 1). + // Without dependency ordering, 'type' would be generated after 'label', so the conditional + // would always evaluate against an incomplete record. With ordering, 'type' is generated first. + const schemaWithReversedOrder: Schema = { + name: 'reversed', + fields: [ + { + name: 'label', + valueType: 'string', + restrictions: [ + { + if: { conditions: [{ fields: ['type'], match: { value: 'A' } }] }, + then: { codeList: ['alpha'] }, + else: { codeList: ['beta'] }, + }, + ], + }, + { + name: 'type', + valueType: 'string', + restrictions: { codeList: ['A', 'B'] }, + }, + ], + }; + + const recordWithTypeA = generateRecord(schemaWithReversedOrder, { + seed: SEED, + ...NO_EMPTY, + overrides: { type: 'A' }, + }); + assert.strictEqual(recordWithTypeA['label'], 'alpha'); + + const recordWithTypeB = generateRecord(schemaWithReversedOrder, { + seed: SEED, + ...NO_EMPTY, + overrides: { type: 'B' }, + }); + assert.strictEqual(recordWithTypeB['label'], 'beta'); + }); +}); diff --git a/packages/data-generator/test/resolveRestrictions.spec.ts b/packages/data-generator/test/resolveRestrictions.spec.ts new file mode 100644 index 00000000..b6ee4950 --- /dev/null +++ b/packages/data-generator/test/resolveRestrictions.spec.ts @@ -0,0 +1,248 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import assert from 'node:assert'; +import { collectRestrictions } from '../src/dataGeneration/fields/resolveRestrictions'; + +type StringRestrictions = { + codeList?: (string | number)[]; + required?: boolean; +}; + +describe('collectRestrictions — plain restrictions', () => { + it('returns empty collections when restrictions is undefined', () => { + const result = collectRestrictions(undefined, {}); + assert.deepStrictEqual(result.codeList, []); + assert.deepStrictEqual(result.required, []); + }); + + it('collects a single plain restriction object', () => { + const result = collectRestrictions({ required: true }, {}); + assert.deepStrictEqual(result.required, [true]); + }); + + it('collects multiple plain restriction objects from an array', () => { + const result = collectRestrictions([{ codeList: ['a', 'b'] }, { required: true }], {}); + assert.deepStrictEqual(result.codeList, [['a', 'b']]); + assert.deepStrictEqual(result.required, [true]); + }); +}); + +describe('collectRestrictions — conditional restrictions (default case: all)', () => { + it('takes the then branch when the condition passes', () => { + const result = collectRestrictions( + { + if: { conditions: [{ fields: ['status'], match: { value: 'active' } }] }, + then: { required: true }, + else: { required: false }, + }, + { status: 'active' }, + ); + assert.deepStrictEqual(result.required, [true]); + }); + + it('takes the else branch when the condition fails', () => { + const result = collectRestrictions( + { + if: { conditions: [{ fields: ['status'], match: { value: 'active' } }] }, + then: { required: true }, + else: { required: false }, + }, + { status: 'inactive' }, + ); + assert.deepStrictEqual(result.required, [false]); + }); + + it('treats missing condition field as undefined, failing an exists:true check', () => { + const result = collectRestrictions( + { + if: { conditions: [{ fields: ['status'], match: { exists: true } }] }, + then: { required: true }, + else: { required: false }, + }, + {}, + ); + assert.deepStrictEqual(result.required, [false]); + }); +}); + +describe('collectRestrictions — condition case: any', () => { + it('passes when at least one of multiple condition fields matches', () => { + // case:'any' on conditions: passes if any condition is true + const result = collectRestrictions( + { + if: { + case: 'any', + conditions: [ + { fields: ['a'], match: { value: 'yes' } }, + { fields: ['b'], match: { value: 'yes' } }, + ], + }, + then: { required: true }, + else: { required: false }, + }, + { a: 'no', b: 'yes' }, + ); + assert.deepStrictEqual(result.required, [true]); + }); + + it('fails when none of the conditions match under case: any', () => { + const result = collectRestrictions( + { + if: { + case: 'any', + conditions: [ + { fields: ['a'], match: { value: 'yes' } }, + { fields: ['b'], match: { value: 'yes' } }, + ], + }, + then: { required: true }, + else: { required: false }, + }, + { a: 'no', b: 'no' }, + ); + assert.deepStrictEqual(result.required, [false]); + }); +}); + +describe('collectRestrictions — condition case: none', () => { + it('passes when none of the conditions match', () => { + const result = collectRestrictions( + { + if: { + case: 'none', + conditions: [ + { fields: ['a'], match: { value: 'yes' } }, + { fields: ['b'], match: { value: 'yes' } }, + ], + }, + then: { required: true }, + else: { required: false }, + }, + { a: 'no', b: 'no' }, + ); + assert.deepStrictEqual(result.required, [true]); + }); + + it('fails when any condition matches under case: none', () => { + const result = collectRestrictions( + { + if: { + case: 'none', + conditions: [ + { fields: ['a'], match: { value: 'yes' } }, + { fields: ['b'], match: { value: 'yes' } }, + ], + }, + then: { required: true }, + else: { required: false }, + }, + { a: 'yes', b: 'no' }, + ); + assert.deepStrictEqual(result.required, [false]); + }); +}); + +describe('collectRestrictions — condition.case across multiple fields', () => { + it('case:any — passes when at least one named field satisfies the match', () => { + // condition.case governs how results across multiple fields are combined. + // With case:'any', the condition passes if any of the listed fields matches. + const result = collectRestrictions( + { + if: { + conditions: [{ fields: ['a', 'b'], match: { value: 'yes' }, case: 'any' }], + }, + then: { required: true }, + else: { required: false }, + }, + { a: 'no', b: 'yes' }, + ); + assert.deepStrictEqual(result.required, [true]); + }); + + it('case:any — fails when no named field satisfies the match', () => { + const result = collectRestrictions( + { + if: { + conditions: [{ fields: ['a', 'b'], match: { value: 'yes' }, case: 'any' }], + }, + then: { required: true }, + else: { required: false }, + }, + { a: 'no', b: 'no' }, + ); + assert.deepStrictEqual(result.required, [false]); + }); + + it('case:none — passes when no named field satisfies the match', () => { + const result = collectRestrictions( + { + if: { + conditions: [{ fields: ['a', 'b'], match: { value: 'yes' }, case: 'none' }], + }, + then: { required: true }, + else: { required: false }, + }, + { a: 'no', b: 'no' }, + ); + assert.deepStrictEqual(result.required, [true]); + }); + + it('case:none — fails when any named field satisfies the match', () => { + const result = collectRestrictions( + { + if: { + conditions: [{ fields: ['a', 'b'], match: { value: 'yes' }, case: 'none' }], + }, + then: { required: true }, + else: { required: false }, + }, + { a: 'yes', b: 'no' }, + ); + assert.deepStrictEqual(result.required, [false]); + }); + + it('case:all (default) — passes only when all named fields satisfy the match', () => { + const result = collectRestrictions( + { + if: { + conditions: [{ fields: ['a', 'b'], match: { value: 'yes' } }], + }, + then: { required: true }, + else: { required: false }, + }, + { a: 'yes', b: 'yes' }, + ); + assert.deepStrictEqual(result.required, [true]); + }); + + it('case:all (default) — fails when any named field does not satisfy the match', () => { + const result = collectRestrictions( + { + if: { + conditions: [{ fields: ['a', 'b'], match: { value: 'yes' } }], + }, + then: { required: true }, + else: { required: false }, + }, + { a: 'yes', b: 'no' }, + ); + assert.deepStrictEqual(result.required, [false]); + }); +}); diff --git a/packages/data-generator/test/restrictionReducers.spec.ts b/packages/data-generator/test/restrictionReducers.spec.ts new file mode 100644 index 00000000..732c6cf3 --- /dev/null +++ b/packages/data-generator/test/restrictionReducers.spec.ts @@ -0,0 +1,228 @@ +/* + * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import assert from 'node:assert'; +import { + reduceCodeLists, + reduceEmpty, + reduceRanges, + reduceRegex, + reduceRequired, +} from '../src/dataGeneration/fields/restrictionReducers'; + +describe('reduceRequired', () => { + it('returns false for an empty list', () => { + assert.strictEqual(reduceRequired([]), false); + }); + + it('returns true when any value is true', () => { + assert.strictEqual(reduceRequired([false, true, false]), true); + }); + + it('returns false when all values are false', () => { + assert.strictEqual(reduceRequired([false, false]), false); + }); + + it('returns true for a single true value', () => { + assert.strictEqual(reduceRequired([true]), true); + }); +}); + +describe('reduceEmpty', () => { + it('returns false for an empty list', () => { + assert.strictEqual(reduceEmpty([]), false); + }); + + it('returns true when any value is true', () => { + assert.strictEqual(reduceEmpty([false, true]), true); + }); + + it('returns false when all values are false', () => { + assert.strictEqual(reduceEmpty([false, false]), false); + }); +}); + +describe('reduceCodeLists', () => { + it('returns success(undefined) for zero lists', () => { + const result = reduceCodeLists([]); + assert.ok(result.success); + assert.strictEqual(result.data, undefined); + }); + + it('returns success with the single list unchanged', () => { + const list = ['a', 'b', 'c']; + const result = reduceCodeLists([list]); + assert.ok(result.success); + assert.deepStrictEqual(result.data, list); + }); + + it('returns the intersection of two overlapping lists', () => { + const result = reduceCodeLists([ + ['a', 'b', 'c'], + ['b', 'c', 'd'], + ]); + assert.ok(result.success); + assert.deepStrictEqual(result.data, ['b', 'c']); + }); + + it('returns the intersection of three overlapping lists', () => { + const result = reduceCodeLists([ + ['a', 'b', 'c', 'd'], + ['b', 'c', 'd', 'e'], + ['c', 'd', 'e', 'f'], + ]); + assert.ok(result.success); + assert.deepStrictEqual(result.data, ['c', 'd']); + }); + + it('returns failure when two lists have no common values', () => { + const result = reduceCodeLists([ + ['a', 'b'], + ['c', 'd'], + ]); + assert.ok(!result.success); + assert.strictEqual(result.data.type, 'codeList'); + }); + + it('returns failure when the intersection becomes empty across three lists', () => { + const result = reduceCodeLists([ + ['a', 'b'], + ['a', 'c'], + ['b', 'c'], + ]); + assert.ok(!result.success); + assert.strictEqual(result.data.type, 'codeList'); + }); + + it('works with numeric code lists', () => { + const result = reduceCodeLists([ + [1, 2, 3], + [2, 3, 4], + ]); + assert.ok(result.success); + assert.deepStrictEqual(result.data, [2, 3]); + }); +}); + +describe('reduceRanges', () => { + it('returns success(undefined) for zero ranges', () => { + const result = reduceRanges([]); + assert.ok(result.success); + assert.strictEqual(result.data, undefined); + }); + + it('returns success with the single range unchanged', () => { + const range = { min: 0, max: 10 }; + const result = reduceRanges([range]); + assert.ok(result.success); + assert.deepStrictEqual(result.data, range); + }); + + it('returns the overlapping subrange of two inclusive ranges', () => { + const result = reduceRanges([ + { min: 0, max: 10 }, + { min: 5, max: 15 }, + ]); + assert.ok(result.success); + assert.deepStrictEqual(result.data, { min: 5, max: 10 }); + }); + + it('returns the tightest subrange across three ranges', () => { + const result = reduceRanges([ + { min: 0, max: 20 }, + { min: 5, max: 15 }, + { min: 8, max: 12 }, + ]); + assert.ok(result.success); + assert.deepStrictEqual(result.data, { min: 8, max: 12 }); + }); + + it('prefers exclusive bound when inclusive and exclusive bounds are equal', () => { + const result = reduceRanges([{ min: 5 }, { exclusiveMin: 5 }]); + assert.ok(result.success); + assert.deepStrictEqual(result.data, { exclusiveMin: 5 }); + }); + + it('returns failure when two ranges do not overlap', () => { + const result = reduceRanges([ + { min: 0, max: 5 }, + { min: 10, max: 20 }, + ]); + assert.ok(!result.success); + assert.strictEqual(result.data.type, 'range'); + }); + + it('returns failure when bounds are equal but both exclusive sides', () => { + const result = reduceRanges([{ max: 5 }, { exclusiveMin: 5 }]); + assert.ok(!result.success); + assert.strictEqual(result.data.type, 'range'); + }); + + it('returns failure when lower bound exceeds upper bound', () => { + const result = reduceRanges([{ min: 10 }, { max: 5 }]); + assert.ok(!result.success); + assert.strictEqual(result.data.type, 'range'); + }); + + it('handles ranges with only a lower bound', () => { + const result = reduceRanges([{ min: 0 }, { min: 5 }]); + assert.ok(result.success); + assert.deepStrictEqual(result.data, { min: 5 }); + }); + + it('handles ranges with only an upper bound', () => { + const result = reduceRanges([{ max: 10 }, { max: 5 }]); + assert.ok(result.success); + assert.deepStrictEqual(result.data, { max: 5 }); + }); +}); + +describe('reduceRegex', () => { + it('returns success(undefined) for zero patterns', () => { + const result = reduceRegex([]); + assert.ok(result.success); + assert.strictEqual(result.data, undefined); + }); + + it('returns the single pattern unchanged', () => { + const pattern = '^[A-Z]+$'; + const result = reduceRegex([pattern]); + assert.ok(result.success); + assert.strictEqual(result.data, pattern); + }); + + it('combines two patterns using lookaheads', () => { + const result = reduceRegex(['^[A-Z]', '\\d{4}$']); + assert.ok(result.success); + assert.strictEqual(result.data, '(?=^[A-Z])(?=\\d{4}$)'); + }); + + it('flattens array patterns before combining', () => { + const result = reduceRegex([['^[A-Z]', '\\d$'], '^[A-Z]\\d']); + assert.ok(result.success); + assert.strictEqual(result.data, '(?=^[A-Z])(?=\\d$)(?=^[A-Z]\\d)'); + }); + + it('always succeeds even when patterns are semantically incompatible', () => { + // ^a and ^b can never both match, but the reducer still succeeds + const result = reduceRegex(['^a', '^b']); + assert.ok(result.success); + assert.ok(typeof result.data === 'string'); + }); +}); diff --git a/packages/data-generator/test/schemaGenerator.spec.ts b/packages/data-generator/test/schemaGenerator.spec.ts new file mode 100644 index 00000000..ec6f7026 --- /dev/null +++ b/packages/data-generator/test/schemaGenerator.spec.ts @@ -0,0 +1,223 @@ +/* + * Copyright (c) 2026 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of + * the GNU Affero General Public License v3.0. You should have received a copy of the + * GNU Affero General Public License along with this program. + * If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import assert from 'node:assert'; +import { describe, it } from 'mocha'; +import type { Schema } from '@overture-stack/lectern-dictionary'; +import { generateSchemaRecords } from '../src/dataGeneration/records/schemaGenerator'; +import type { ForeignKeyPool } from '../src/dataGeneration/records/recordGenerator'; + +const SEED = 42; +const NO_EMPTY = { emptyRate: 0 } as const; + +const schema: Schema = { + name: 'test', + fields: [ + { name: 'id', valueType: 'string', restrictions: { codeList: ['A', 'B', 'C', 'D', 'E'] } }, + { name: 'label', valueType: 'string', restrictions: undefined }, + ], +}; + +const schemaWithUnique: Schema = { + name: 'unique_test', + fields: [ + { + name: 'code', + valueType: 'string', + unique: true, + restrictions: { codeList: ['X1', 'X2', 'X3', 'X4', 'X5'] }, + }, + { name: 'name', valueType: 'string', restrictions: undefined }, + ], +}; + +const schemaWithUniqueKey: Schema = { + name: 'uniquekey_test', + fields: [ + { name: 'program', valueType: 'string', restrictions: { codeList: ['P1', 'P2', 'P3'] } }, + { name: 'donor', valueType: 'string', restrictions: { codeList: ['D1', 'D2', 'D3'] } }, + { name: 'label', valueType: 'string', restrictions: undefined }, + ], + restrictions: { + uniqueKey: ['program', 'donor'], + }, +}; + +const schemaWithFk: Schema = { + name: 'child', + fields: [ + { name: 'donor_id', valueType: 'string', restrictions: undefined }, + { name: 'value', valueType: 'string', restrictions: undefined }, + ], + restrictions: { + foreignKey: [{ schema: 'donor', mappings: [{ local: 'donor_id', foreign: 'id' }] }], + }, +}; + +describe('generateSchemaRecords', () => { + it('yields exactly count records', () => { + const records = [...generateSchemaRecords(schema, { count: 5, seed: SEED, ...NO_EMPTY })]; + assert.strictEqual(records.length, 5); + }); + + it('yields zero records when count is 0', () => { + const records = [...generateSchemaRecords(schema, { count: 0, seed: SEED })]; + assert.strictEqual(records.length, 0); + }); + + it('yields zero records when options are omitted', () => { + const records = [...generateSchemaRecords(schema)]; + assert.strictEqual(records.length, 0); + }); + + it('produces identical sequences for the same seed', () => { + const first = [...generateSchemaRecords(schema, { count: 5, seed: SEED, ...NO_EMPTY })]; + const second = [...generateSchemaRecords(schema, { count: 5, seed: SEED, ...NO_EMPTY })]; + assert.deepStrictEqual(first, second); + }); + + it('produces different sequences for different seeds', () => { + const first = [...generateSchemaRecords(schema, { count: 5, seed: 1, ...NO_EMPTY })]; + const second = [...generateSchemaRecords(schema, { count: 5, seed: 99, ...NO_EMPTY })]; + assert.notDeepStrictEqual(first, second); + }); + + it('each yielded record has all schema fields', () => { + const records = [...generateSchemaRecords(schema, { count: 3, seed: SEED, ...NO_EMPTY })]; + for (const record of records) { + for (const field of schema.fields) { + assert.ok(Object.hasOwn(record, field.name), `missing field: ${field.name}`); + } + } + }); + + describe('unique field enforcement', () => { + it('unique field values are distinct across all yielded records', () => { + const count = 5; + const records = [...generateSchemaRecords(schemaWithUnique, { count, seed: SEED, ...NO_EMPTY })]; + const codeValues = records.map((record) => record['code']); + const unique = new Set(codeValues); + assert.strictEqual(unique.size, count, `expected ${count} distinct code values, got ${unique.size}`); + }); + + it('initialUniqueValues.fields pre-populates exclusion for unique fields', () => { + // Pre-seed 4 of the 5 codeList values, leaving only 'X5' available. + const records = [ + ...generateSchemaRecords(schemaWithUnique, { + count: 1, + seed: SEED, + ...NO_EMPTY, + initialUniqueValues: { fields: { code: ['X1', 'X2', 'X3', 'X4'] } }, + }), + ]; + assert.strictEqual(records[0]?.['code'], 'X5'); + }); + }); + + describe('uniqueKey enforcement', () => { + it('uniqueKey tuples are distinct across all yielded records', () => { + const count = 9; // 3 programs × 3 donors = 9 unique combinations + const records = [...generateSchemaRecords(schemaWithUniqueKey, { count, seed: SEED, ...NO_EMPTY })]; + const tuples = records.map((record) => JSON.stringify([record['program'], record['donor']])); + const uniqueTuples = new Set(tuples); + assert.strictEqual(uniqueTuples.size, count, `expected ${count} distinct key tuples, got ${uniqueTuples.size}`); + }); + + it('colliding positions retry to a different value and all output keys avoid the pre-seeded set', () => { + // Use a 3×3 codeList space → 9 possible key combinations. + // Generate 4 records (baseline). Pre-seed those 4 keys, then request 5 more with the + // same seed. The 5 remaining combinations are all available, so every colliding position + // can retry to an unused key. Assert: all 5 output keys are distinct and none appear in + // the pre-seeded set. + const medKeySchema: Schema = { + name: 'med_key', + fields: [ + { name: 'a', valueType: 'string', restrictions: { codeList: ['A1', 'A2', 'A3'] } }, + { name: 'b', valueType: 'string', restrictions: { codeList: ['B1', 'B2', 'B3'] } }, + ], + restrictions: { uniqueKey: ['a', 'b'] }, + }; + + const keyOf = (record: Record): string => JSON.stringify([record['a'], record['b']]); + + const baseline = [...generateSchemaRecords(medKeySchema, { count: 4, seed: SEED, ...NO_EMPTY })]; + const preSeenKeys = baseline.map(keyOf); + + const withPreSeen = [ + ...generateSchemaRecords(medKeySchema, { + count: 5, + seed: SEED, + ...NO_EMPTY, + initialUniqueValues: { keys: preSeenKeys }, + }), + ]; + + // All 5 output keys must be distinct — retry mechanism found unused combinations. + const outputKeys = withPreSeen.map(keyOf); + assert.strictEqual(new Set(outputKeys).size, 5, 'all 5 output keys must be distinct after retries'); + + // None of the output keys may be in the pre-seeded set. + for (const key of outputKeys) { + assert.ok(!preSeenKeys.includes(key), `output key ${key} was in the pre-seeded set`); + } + }); + + it('initialUniqueValues.keys pre-populates the uniqueKey tracker', () => { + // Generate without initial keys, then use those as pre-seen — the second run must + // avoid those exact tuples since it shares the same seed. + const firstRun = [...generateSchemaRecords(schemaWithUniqueKey, { count: 3, seed: SEED, ...NO_EMPTY })]; + const preSeenKeys = firstRun.map((record) => JSON.stringify([record['program'], record['donor']])); + + const secondRun = [ + ...generateSchemaRecords(schemaWithUniqueKey, { + count: 3, + seed: SEED, + ...NO_EMPTY, + initialUniqueValues: { keys: preSeenKeys }, + }), + ]; + + for (const record of secondRun) { + const key = JSON.stringify([record['program'], record['donor']]); + assert.ok(!preSeenKeys.includes(key), `generated key ${key} was in the pre-seen set`); + } + }); + }); + + describe('foreignKeyPool', () => { + it('FK-constrained field values come from the pool', () => { + const pool: ForeignKeyPool = new Map([['donor', [{ id: 'D001' }, { id: 'D002' }]]]); + const records = [ + ...generateSchemaRecords(schemaWithFk, { count: 5, seed: SEED, ...NO_EMPTY, foreignKeyPool: pool }), + ]; + const validIds = new Set(['D001', 'D002']); + for (const record of records) { + assert.ok(validIds.has(record['donor_id'] as string), `unexpected donor_id: ${String(record['donor_id'])}`); + } + }); + }); + + it('records are yielded lazily — generator does not pre-compute all records', () => { + const generator = generateSchemaRecords(schema, { count: 1000, seed: SEED, ...NO_EMPTY }); + // Taking only the first record should not trigger generation of all 1000. + const firstResult = generator.next(); + assert.strictEqual(firstResult.done, false); + assert.ok(firstResult.value !== undefined); + }); +}); diff --git a/packages/data-generator/tsconfig.build.json b/packages/data-generator/tsconfig.build.json new file mode 100644 index 00000000..df0d74a2 --- /dev/null +++ b/packages/data-generator/tsconfig.build.json @@ -0,0 +1,4 @@ +{ + "extends": "./tsconfig.json", + "exclude": ["test/**/*.ts"] +} diff --git a/packages/data-generator/tsconfig.json b/packages/data-generator/tsconfig.json new file mode 100644 index 00000000..080de94d --- /dev/null +++ b/packages/data-generator/tsconfig.json @@ -0,0 +1,21 @@ +{ + "compilerOptions": { + "target": "ESNext", + "lib": ["ESNext"], + "module": "CommonJS", + "moduleResolution": "node", + "resolveJsonModule": true, + "strict": true, + "noImplicitAny": true, + "noUnusedParameters": true, + "noUncheckedIndexedAccess": true, + "esModuleInterop": true, + "declaration": true, + "sourceMap": true, + "inlineSources": true, + "outDir": "dist/", + "skipLibCheck": true, + "types": ["node"] + }, + "include": ["./src/**/*.ts", "./test/**/*.ts"] +} diff --git a/packages/dictionary/src/metaSchema/restrictionsSchemas.ts b/packages/dictionary/src/metaSchema/restrictionsSchemas.ts index 1497cc40..78e30d61 100644 --- a/packages/dictionary/src/metaSchema/restrictionsSchemas.ts +++ b/packages/dictionary/src/metaSchema/restrictionsSchemas.ts @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * Copyright (c) 2026 The Ontario Institute for Cancer Research. All rights reserved * * This program and the accompanying materials are made available under the terms of * the GNU Affero General Public License v3.0. You should have received a copy of the @@ -210,3 +210,11 @@ export type ConditionalRestriction = { | ConditionalRestriction | (TRestrictionObject | ConditionalRestriction)[]; }; + +export const isConditionalRestriction = ( + value: T | ConditionalRestriction, +): value is ConditionalRestriction => + typeof value === 'object' && + value !== null && + 'if' in value && + ConditionalRestrictionTest.safeParse(value.if).success; diff --git a/packages/validation/docs/iterative-validation/feature-design.md b/packages/validation/docs/iterative-validation/feature-design.md index 3c23f378..9483c430 100644 --- a/packages/validation/docs/iterative-validation/feature-design.md +++ b/packages/validation/docs/iterative-validation/feature-design.md @@ -19,10 +19,10 @@ The lectern `validation` package currently requires the entire dataset to be loa This proposal introduces four new stateful validator objects that accept records one at a time, maintain lightweight internal indices, and produce a complete error report once all records have been submitted: -1. **CrossRecordValidator** — stateful; tracks `unique` and `uniqueKey` violations across submitted records. Does not validate individual records. -2. **CrossSchemaValidator** — stateful; tracks `foreignKey` violations across schemas. Does not validate individual records or uniqueness. -3. **SchemaValidator** — combines per-record validation (`validateRecord`) with cross-record validation (`CrossRecordValidator`). -4. **DictionaryValidator** — combines per-record validation, cross-record validation, and cross-schema validation (`CrossSchemaValidator`). +1. **CrossRecordValidator** - stateful; tracks `unique` and `uniqueKey` violations across submitted records. Does not validate individual records. +2. **CrossSchemaValidator** - stateful; tracks `foreignKey` violations across schemas. Does not validate individual records or uniqueness. +3. **SchemaValidator** - combines per-record validation (`validateRecord`) with cross-record validation (`CrossRecordValidator`). +4. **DictionaryValidator** - combines per-record validation, cross-record validation, and cross-schema validation (`CrossSchemaValidator`). All four are exported so developers can use the lower-level components directly. @@ -34,7 +34,7 @@ All four are exported so developers can use the lower-level components directly. The validation functions in `@overture-stack/lectern-validation` require the entire dataset to be provided as an argument at the time of the function call. For schema-level and dictionary-level validation, this means all records across all related schemas must be fully loaded into memory before any validation can begin. -The consequence is that datasets above a certain size — determined by available application memory — cannot be validated at all. This is not a performance concern but a hard functional limit. +The consequence is that datasets above a certain size - determined by available application memory - cannot be validated at all. This is not a performance concern but a hard functional limit. ### Evidence @@ -58,10 +58,10 @@ Schema-level validation builds a hash map across all records to detect `unique` - **DictionaryValidator**: orchestrates per-record validation, `CrossRecordValidator`, and `CrossSchemaValidator`; routes submitted records by schema name. - All four components exported from the package so consumers can compose them independently. - Clear lifecycle contract for all stateful objects: construction, record submission, error retrieval, and report retrieval. -- Per-record validation errors are returned directly in the `Result` of `submit()` and are never stored internally — the caller handles or discards them. +- Per-record validation errors are returned directly in the `Result` of `submit()` and are never stored internally - the caller handles or discards them. - Cross-record (`unique`/`uniqueKey`) and cross-schema (`foreignKey`) violations are stored in the internal index and exposed via `errors()`, a generator that yields detailed error objects one at a time so the caller can handle and discard each without accumulating them all in memory. - `report()` returns aggregate stats only: record counts, record-level error counts, and violation counts broken down by constraint type and field. -- Internal memory is bounded to the size of the cross-record index (the `DataSetHashMap`) and the cross-schema reference sets — not to error volume or record count. +- Internal memory is bounded to the size of the cross-record index (the `DataSetHashMap`) and the cross-schema reference sets - not to error volume or record count. ### Non-Goals @@ -86,8 +86,8 @@ const crossRecordValidator = createCrossRecordValidator(schema); // submit() returns Result // success: record was accepted and added to the index -// failure DUPLICATE_ID: the id was already seen — record is ignored -// failure LOCKED: errors() generator is active — record is ignored. No records are accepted until generator is exhausted +// failure DUPLICATE_ID: the id was already seen - record is ignored +// failure LOCKED: errors() generator is active - record is ignored. No records are accepted until generator is exhausted const result = crossRecordValidator.submit({ id, data }); // single record const result = crossRecordValidator.submit(entries); // Array<{ id: string; data: DataRecord }> @@ -101,15 +101,15 @@ const report = crossRecordValidator.report(); // invalid({ details }) when any unique or uniqueKey violations occurred. // errors() returns a generator of detailed violation error objects. -// Calling errors() locks the validator — submit() returns failure LOCKED until the generator is exhausted. +// Calling errors() locks the validator - submit() returns failure LOCKED until the generator is exhausted. const errorGenerator = crossRecordValidator.errors(); for (const error of errorGenerator) { handleError(error); // CrossRecordValidationError } -// generator exhausted — validator is now unlocked, submit() accepts records again +// generator exhausted - validator is now unlocked, submit() accepts records again ``` -Internally maintains a `DataSetHashMap` (`Map`) per unique/uniqueKey rule — keys are field-value hashes, values are arrays of caller-supplied record IDs — built incrementally as records are submitted. Duplicate IDs are not processed and `submit()` returns `failure` with the duplicate ID. `report()` walks the completed map and counts violations per field (for `unique`) and total violations (for `uniqueKey`). Returns `valid()` when all counts are zero. `errors()` walks the same map and yields a detailed error object per violation; the validator is locked for the duration. +Internally maintains a `DataSetHashMap` (`Map`) per unique/uniqueKey rule - keys are field-value hashes, values are arrays of caller-supplied record IDs - built incrementally as records are submitted. Duplicate IDs are not processed and `submit()` returns `failure` with the duplicate ID. `report()` walks the completed map and counts violations per field (for `unique`) and total violations (for `uniqueKey`). Returns `valid()` when all counts are zero. `errors()` walks the same map and yields a detailed error object per violation; the validator is locked for the duration. #### CrossSchemaValidator @@ -120,9 +120,9 @@ const crossSchemaValidator = createCrossSchemaValidator(dictionary); // submit() returns Result // success: record was accepted and its FK-relevant field values added to the reference set -// failure DUPLICATE_ID: the id was already seen for this schema — record is ignored -// failure UNKNOWN_SCHEMA: schemaName is not in the dictionary — record is ignored -// failure LOCKED: errors() generator is active — record is ignored. No records are accepted until generator is exhausted +// failure DUPLICATE_ID: the id was already seen for this schema - record is ignored +// failure UNKNOWN_SCHEMA: schemaName is not in the dictionary - record is ignored +// failure LOCKED: errors() generator is active - record is ignored. No records are accepted until generator is exhausted const result = crossSchemaValidator.submit(schemaName, { id, data }); // single record const result = crossSchemaValidator.submit(schemaName, entries); // Array<{ id: string; data: DataRecord }> @@ -135,12 +135,12 @@ const report = crossSchemaValidator.report(); // invalid({ details }) when any foreignKey violations occurred. // errors() returns a generator of detailed FK violation error objects. -// Calling errors() locks the validator — submit() returns failure LOCKED until the generator is exhausted. +// Calling errors() locks the validator - submit() returns failure LOCKED until the generator is exhausted. const errorGenerator = crossSchemaValidator.errors(); for (const error of errorGenerator) { handleError(error); // CrossSchemaValidationError } -// generator exhausted — validator is now unlocked, submit() accepts records again +// generator exhausted - validator is now unlocked, submit() accepts records again ``` Internally maintains a `Map` (`Map>>`) built incrementally as records are submitted. Unknown schema names and duplicate IDs are both rejected at `submit()` time via `failure`. `report()` runs `testForeignKeyRestriction` for each submitted record against the completed reference map and accumulates violation counts per FK mapping, grouped by local schema. `errors()` yields a detailed error object per FK violation; the validator is locked for the duration. @@ -155,9 +155,9 @@ Combines per-record validation with cross-record validation. The primary interfa const schemaValidator = createSchemaValidator(schema); // submit() returns Result, { reason: 'DUPLICATE_ID' | 'LOCKED' }> -// success: record(s) accepted and validated — data is an array of per-record errors (empty if all records are valid) -// failure DUPLICATE_ID: a submitted id was already seen — no records from this call are processed -// failure LOCKED: errors() generator is active — record is ignored. No records are accepted until generator is exhausted +// success: record(s) accepted and validated - data is an array of per-record errors (empty if all records are valid) +// failure DUPLICATE_ID: a submitted id was already seen - no records from this call are processed +// failure LOCKED: errors() generator is active - record is ignored. No records are accepted until generator is exhausted const result = schemaValidator.submit({ id, data }); // single record const result = schemaValidator.submit(entries); // Array<{ id: string; data: DataRecord }> if (result.success) { @@ -166,7 +166,7 @@ if (result.success) { } } -// report() returns summary counts only — counts are maintained as running totals. +// report() returns summary counts only - counts are maintained as running totals. const report = schemaValidator.report(); // TestResult<{ // recordCount: number, @@ -180,12 +180,12 @@ const report = schemaValidator.report(); // invalid({ details }) when any record-level or cross-record violations occurred. // errors() returns a generator of detailed cross-record violation error objects. -// Calling errors() locks the validator — submit() returns failure LOCKED until the generator is exhausted. +// Calling errors() locks the validator - submit() returns failure LOCKED until the generator is exhausted. const errorGenerator = schemaValidator.errors(); for (const error of errorGenerator) { handleError(error); // CrossRecordValidationError } -// generator exhausted — validator is now unlocked, submit() accepts records again +// generator exhausted - validator is now unlocked, submit() accepts records again ``` Per-record errors are returned directly in the `submit()` result and never stored internally. Cross-record violations are stored in the internal index and exposed via `errors()`, which yields detailed error objects one at a time. `report()` returns a `TestResult` wrapping aggregate stats: `valid()` when no violations occurred at any level, `invalid({ details })` with counts otherwise. @@ -198,10 +198,10 @@ Combines per-record validation, cross-record validation, and cross-schema valida const dictionaryValidator = createDictionaryValidator(dictionary); // submit() returns Result, { reason: 'DUPLICATE_ID' | 'UNKNOWN_SCHEMA' | 'LOCKED' }> -// success: record(s) accepted and validated — data is an array of per-record errors (empty if all records are valid) -// failure DUPLICATE_ID: a submitted id was already seen for this schema — no records from this call are processed -// failure UNKNOWN_SCHEMA: schemaName is not in the dictionary — no records from this call are processed -// failure LOCKED: errors() generator is active — record is ignored. No records are accepted until generator is exhausted +// success: record(s) accepted and validated - data is an array of per-record errors (empty if all records are valid) +// failure DUPLICATE_ID: a submitted id was already seen for this schema - no records from this call are processed +// failure UNKNOWN_SCHEMA: schemaName is not in the dictionary - no records from this call are processed +// failure LOCKED: errors() generator is active - record is ignored. No records are accepted until generator is exhausted const result = dictionaryValidator.submit(schemaName, { id, data }); // single record const result = dictionaryValidator.submit(schemaName, entries); // Array<{ id: string; data: DataRecord }> if (result.success) { @@ -210,7 +210,7 @@ if (result.success) { } } -// report() returns summary counts only — counts are maintained as running totals. +// report() returns summary counts only - counts are maintained as running totals. const report = dictionaryValidator.report(); // TestResult<{ // unknownSchemaCount: number, // count of submit() calls for unrecognized schema names @@ -228,15 +228,15 @@ const report = dictionaryValidator.report(); // invalid({ details }) when any violations or unknown schema submissions occurred. // errors() returns a generator of detailed cross-record and cross-schema violation error objects. -// Calling errors() locks the validator — submit() returns failure LOCKED until the generator is exhausted. +// Calling errors() locks the validator - submit() returns failure LOCKED until the generator is exhausted. const errorGenerator = dictionaryValidator.errors(); for (const error of errorGenerator) { handleError(error); // CrossRecordValidationError | CrossSchemaValidationError } -// generator exhausted — validator is now unlocked, submit() accepts records again +// generator exhausted - validator is now unlocked, submit() accepts records again ``` -Internally holds one `CrossRecordValidator` per schema and one shared `CrossSchemaValidator`. On `submit()`, `validateRecord` runs immediately and errors are returned directly in the result — never stored. The record is also passed to the appropriate `CrossRecordValidator` and `CrossSchemaValidator`. `report()` returns aggregate counts only. `errors()` delegates to the internal `CrossRecordValidator` instances and `CrossSchemaValidator`, yielding their detailed error objects in sequence; the validator is locked for the duration. +Internally holds one `CrossRecordValidator` per schema and one shared `CrossSchemaValidator`. On `submit()`, `validateRecord` runs immediately and errors are returned directly in the result - never stored. The record is also passed to the appropriate `CrossRecordValidator` and `CrossSchemaValidator`. `report()` returns aggregate counts only. `errors()` delegates to the internal `CrossRecordValidator` instances and `CrossSchemaValidator`, yielding their detailed error objects in sequence; the validator is locked for the duration. --- @@ -255,13 +255,13 @@ All four validator objects have two states: **open** (accepting submissions) and #### Problem -`SchemaValidationRecordErrorUnique` and `SchemaValidationRecordErrorUniqueKey` both include `matchingRecords: number[]` — an array of indices into the record array passed to the batch validator. In streaming mode there is no record array, so positional indices have no meaning. +`SchemaValidationRecordErrorUnique` and `SchemaValidationRecordErrorUniqueKey` both include `matchingRecords: number[]` - an array of indices into the record array passed to the batch validator. In streaming mode there is no record array, so positional indices have no meaning. #### Solution: Caller-Provided Record IDs -`submit()` accepts entries of the shape `{ id: string; data: DataRecord }`. The `id` is a caller-supplied string that uniquely identifies the record within the submission — typically a file row number, a line offset, or any stable external identifier the caller maintains. +`submit()` accepts entries of the shape `{ id: string; data: DataRecord }`. The `id` is a caller-supplied string that uniquely identifies the record within the submission - typically a file row number, a line offset, or any stable external identifier the caller maintains. -The `DataSetHashMap` value type changes from `number[]` to `string[]`, storing caller-supplied IDs instead of positional indices. `matchingRecords: number[]` becomes `matchingRecords: string[]` in the unique error types — a breaking change to the public types, handled in the Error Type Migration Guide. +The `DataSetHashMap` value type changes from `number[]` to `string[]`, storing caller-supplied IDs instead of positional indices. `matchingRecords: number[]` becomes `matchingRecords: string[]` in the unique error types - a breaking change to the public types, handled in the Error Type Migration Guide. Per-record errors returned from `submit()` include the `id` so the caller can correlate errors back to the originating record. @@ -283,7 +283,7 @@ On `report()`: On `errors()`: 1. The validator transitions to the **locked** state. -2. The `DataSetHashMap` is walked; for each violation, a detailed error object is constructed and yielded. Error objects are not stored — each is yielded and then eligible for garbage collection once the caller advances the generator. +2. The `DataSetHashMap` is walked; for each violation, a detailed error object is constructed and yielded. Error objects are not stored - each is yielded and then eligible for garbage collection once the caller advances the generator. 3. When the generator is exhausted, the validator transitions back to **open**. `DictionaryValidator` follows the same flow per schema, with the additional step that each submitted record's FK-referenced field values are added to the `CrossSchemaValidator`'s `SchemaDataReference` map before the record is discarded. FK violations are computed at `report()` and `errors()` time. `DictionaryValidator.errors()` yields cross-record violations (from each per-schema `CrossRecordValidator`) followed by cross-schema violations (from the shared `CrossSchemaValidator`). @@ -318,7 +318,7 @@ This is an approach that a submission service like Lyric would be interested in, | # | Risk / Question | Owner | Resolution | |---|---|---|---| -| 1 | **FK testing requires re-examining submitted records at `report()` time.** `testForeignKeyRestriction` tests a record against a reference map. But records and reference data arrive interleaved during streaming — a record in schema B may arrive before all schema A records have been submitted, so the reference map is incomplete at submission time. FK violations can only be tested once the full reference map is built (i.e. at `report()`). This means either: (a) the `CrossSchemaValidator` holds a copy of every submitted record to replay them at `report()` time, reintroducing memory pressure proportional to record count; or (b) the caller is required to submit records in dependency order (all foreign schema records before all referencing schema records), allowing FK testing at submit time. Option (b) shifts burden to the caller and makes the API fragile. Option (a) is correct but undermines the memory goal for FK-heavy workloads. A third option: store only the FK-relevant field values per record (not the full record), which reduces overhead to the size of the referenced fields only. | | | +| 1 | **FK testing requires re-examining submitted records at `report()` time.** `testForeignKeyRestriction` tests a record against a reference map. But records and reference data arrive interleaved during streaming - a record in schema B may arrive before all schema A records have been submitted, so the reference map is incomplete at submission time. FK violations can only be tested once the full reference map is built (i.e. at `report()`). This means either: (a) the `CrossSchemaValidator` holds a copy of every submitted record to replay them at `report()` time, reintroducing memory pressure proportional to record count; or (b) the caller is required to submit records in dependency order (all foreign schema records before all referencing schema records), allowing FK testing at submit time. Option (b) shifts burden to the caller and makes the API fragile. Option (a) is correct but undermines the memory goal for FK-heavy workloads. A third option: store only the FK-relevant field values per record (not the full record), which reduces overhead to the size of the referenced fields only. | | | | 2 | **`matchingRecords` field semantics in streaming unique errors.** Resolved in section 4.3: `matchingRecords` becomes `string[]` of caller-supplied record IDs. Breaking change to public types; see Error Type Migration Guide. | | Resolved | --- @@ -377,7 +377,7 @@ This is an approach that a submission service like Lyric would be interested in, ### Correctness Invariants - + --- diff --git a/packages/validation/docs/iterative-validation/issue-recreation-plan.md b/packages/validation/docs/iterative-validation/issue-recreation-plan.md index 8a8d5257..d8fc2fa7 100644 --- a/packages/validation/docs/iterative-validation/issue-recreation-plan.md +++ b/packages/validation/docs/iterative-validation/issue-recreation-plan.md @@ -6,8 +6,8 @@ Two things must be shown clearly: -1. **The failure exists** — validation crashes or becomes inoperable at some record count -2. **Memory scales with input** — memory usage grows proportionally to record count, not arbitrarily +1. **The failure exists** - validation crashes or becomes inoperable at some record count +2. **Memory scales with input** - memory usage grows proportionally to record count, not arbitrarily --- @@ -15,8 +15,8 @@ Two things must be shown clearly: The test application runs inside a Docker container. Node.js runs inside the container; the host machine's total RAM is not a controlled variable. Memory limits are enforced at two levels: -- **Container memory limit** — set via `docker run --memory=` (or the equivalent in a Compose file). This is the hard ceiling enforced by the container runtime. If the process exceeds this, the container is OOM-killed by the OS. -- **Node.js heap limit** — set via `--max-old-space-size=` passed to the node process inside the container. This causes Node to throw a JavaScript heap out of memory error before hitting the container limit, producing a more informative error than a hard kill. The Node heap limit should be set below the container memory limit to ensure a clean, observable failure. +- **Container memory limit** - set via `docker run --memory=` (or the equivalent in a Compose file). This is the hard ceiling enforced by the container runtime. If the process exceeds this, the container is OOM-killed by the OS. +- **Node.js heap limit** - set via `--max-old-space-size=` passed to the node process inside the container. This causes Node to throw a JavaScript heap out of memory error before hitting the container limit, producing a more informative error than a hard kill. The Node heap limit should be set below the container memory limit to ensure a clean, observable failure. **Controlled variables to document:** - Docker image and Node.js version @@ -33,14 +33,14 @@ The test application runs inside a Docker container. Node.js runs inside the con **1. Test Dictionaries** Three dictionaries are needed, one per performance test case (see Test Cases below). Each dictionary should have: -- A wide field count per schema (20–30 fields) — this maximises per-record object overhead +- A wide field count per schema (20–30 fields) - this maximises per-record object overhead - A mix of field types (string, integer, boolean, arrays) **2. Record Generator** A TypeScript generator function (async generator) that produces `DataRecord` objects one at a time and yields them directly. It does not write to disk. The generator: - Accepts a record count and the target schema as arguments -- Produces valid records that pass all constraints — the goal is to stress memory, not trigger early exits on validation errors +- Produces valid records that pass all constraints - the goal is to stress memory, not trigger early exits on validation errors - Manages unique field values internally (incrementing counter, etc.) when the schema has a `unique` constraint - Manages foreign key values internally (drawing from the parent schema's emitted key set) when the schema has a `foreignKey` constraint @@ -66,7 +66,7 @@ A `Dockerfile` and `docker-compose.yml` (or equivalent) that: Three performance test cases, each exercising a distinct memory path in the current batch validator: **Case 1: No unique or foreign key constraints** -One schema with no `unique`, `uniqueKey`, or `foreignKey` restrictions. Field and record validation only. Establishes the memory baseline — this path holds no cross-record state and should scale with constant overhead per record. +One schema with no `unique`, `uniqueKey`, or `foreignKey` restrictions. Field and record validation only. Establishes the memory baseline - this path holds no cross-record state and should scale with constant overhead per record. **Case 2: Schema with a unique constraint** One schema with at least one field marked `unique`. The current batch validator builds a `DataSetHashMap` (`Map`) over the full record array before checking anything. This case demonstrates that the map grows linearly with record count and eventually exhausts heap. diff --git a/packages/validation/src/validateField/conditions/index.ts b/packages/validation/src/validateField/conditions/index.ts index c49a91d4..cc452cf1 100644 --- a/packages/validation/src/validateField/conditions/index.ts +++ b/packages/validation/src/validateField/conditions/index.ts @@ -23,3 +23,4 @@ export * from './testMatchExists'; export * from './testMatchRange'; export * from './testMatchRegex'; export * from './testMatchValue'; +export * from './testConditionalRestriction'; diff --git a/packages/validation/src/validateField/index.ts b/packages/validation/src/validateField/index.ts index 53056c48..e6637fc9 100644 --- a/packages/validation/src/validateField/index.ts +++ b/packages/validation/src/validateField/index.ts @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024 The Ontario Institute for Cancer Research. All rights reserved + * Copyright (c) 2026 The Ontario Institute for Cancer Research. All rights reserved * * This program and the accompanying materials are made available under the terms of * the GNU Affero General Public License v3.0. You should have received a copy of the @@ -17,6 +17,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +export * from './conditions'; export * from './restrictions'; export * from './FieldRestrictionRule'; export * from './FieldRestrictionTest'; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6916edcf..40faaa2a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -31,16 +31,16 @@ importers: version: 7.1.2(chai@4.5.0) chai-http: specifier: ^4.4.0 - version: 4.4.0 + version: 4.4.0(supports-color@8.1.1) mocha: specifier: ^10.7.0 version: 10.7.0 nx: specifier: ^16.10.0 - version: 16.10.0 + version: 16.10.0(debug@4.4.1(supports-color@8.1.1)) nyc: specifier: ^15.1.0 - version: 15.1.0 + version: 15.1.0(supports-color@8.1.1) prettier: specifier: ^3.3.3 version: 3.3.3 @@ -67,7 +67,7 @@ importers: version: 8.17.1 axios: specifier: ^1.7.2 - version: 1.9.0 + version: 1.9.0(debug@4.4.1(supports-color@5.5.0)) cors: specifier: ^2.8.5 version: 2.8.5 @@ -79,7 +79,7 @@ importers: version: 1.5.1 express: specifier: ^5.1.0 - version: 5.1.0 + version: 5.1.0(supports-color@5.5.0) immer: specifier: ^10.1.1 version: 10.1.1 @@ -97,16 +97,16 @@ importers: version: 0.4.17 mongoose: specifier: ^7.8.0 - version: 7.8.0 + version: 7.8.0(supports-color@5.5.0) ms: specifier: ^2.1.3 version: 2.1.3 node-vault: specifier: ^0.9.22 - version: 0.9.22 + version: 0.9.22(supports-color@5.5.0) swagger-ui-express: specifier: ^5.0.1 - version: 5.0.1(express@5.1.0) + version: 5.0.1(express@5.1.0(supports-color@5.5.0)) winston: specifier: ^3.13.1 version: 3.13.1 @@ -119,7 +119,7 @@ importers: version: 4.3.16 '@types/chai-http': specifier: ^4.2.4 - version: 4.2.4 + version: 4.2.4(supports-color@5.5.0) '@types/cors': specifier: ^2.8.18 version: 2.8.18 @@ -167,7 +167,7 @@ importers: version: 3.3.3 testcontainers: specifier: ^1.3.1 - version: 1.3.1 + version: 1.3.1(supports-color@5.5.0) typescript: specifier: ^5.5.4 version: 5.8.3 @@ -185,7 +185,7 @@ importers: version: link:../validation axios: specifier: ^1.7.2 - version: 1.9.0 + version: 1.9.0(debug@4.4.1(supports-color@8.1.1)) cd: specifier: ^0.3.3 version: 0.3.3 @@ -236,6 +236,19 @@ importers: specifier: ^5.5.4 version: 5.8.3 + packages/data-generator: + dependencies: + '@overture-stack/lectern-dictionary': + specifier: workspace:^ + version: link:../dictionary + '@overture-stack/lectern-validation': + specifier: workspace:^ + version: link:../validation + devDependencies: + fast-check: + specifier: ^4.9.0 + version: 4.9.0 + packages/dictionary: dependencies: immer: @@ -256,10 +269,10 @@ importers: dependencies: '@emotion/react': specifier: ^11.14.0 - version: 11.14.0(@types/react@19.1.5)(react@19.1.0) + version: 11.14.0(@types/react@19.1.5)(react@19.1.0)(supports-color@8.1.1) '@emotion/styled': specifier: ^11.14.0 - version: 11.14.0(@emotion/react@11.14.0(@types/react@19.1.5)(react@19.1.0))(@types/react@19.1.5)(react@19.1.0) + version: 11.14.0(@emotion/react@11.14.0(@types/react@19.1.5)(react@19.1.0)(supports-color@8.1.1))(@types/react@19.1.5)(react@19.1.0)(supports-color@8.1.1) '@overture-stack/lectern-client': specifier: workspace:* version: link:../client @@ -296,34 +309,34 @@ importers: devDependencies: '@chromatic-com/storybook': specifier: ^3.2.6 - version: 3.2.6(react@19.1.0)(storybook@8.6.14(prettier@3.3.3)) + version: 3.2.6(react@19.1.0)(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1)) '@storybook/addon-essentials': specifier: ^8.6.14 - version: 8.6.14(@types/react@19.1.5)(storybook@8.6.14(prettier@3.3.3)) + version: 8.6.14(@types/react@19.1.5)(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1)) '@storybook/blocks': specifier: ^8.6.14 - version: 8.6.14(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(storybook@8.6.14(prettier@3.3.3)) + version: 8.6.14(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1)) '@storybook/experimental-addon-test': specifier: ^8.6.14 - version: 8.6.14(@vitest/browser@3.1.4)(@vitest/runner@3.1.4)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(storybook@8.6.14(prettier@3.3.3))(vitest@3.1.4) + version: 8.6.14(@vitest/browser@3.1.4)(@vitest/runner@3.1.4)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1))(vitest@3.1.4) '@storybook/icons': specifier: ^1.4.0 version: 1.4.0(react-dom@19.1.0(react@19.1.0))(react@19.1.0) '@storybook/manager-api': specifier: ^8.6.14 - version: 8.6.14(storybook@8.6.14(prettier@3.3.3)) + version: 8.6.14(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1)) '@storybook/react': specifier: ^8.6.14 - version: 8.6.14(@storybook/test@8.6.14(storybook@8.6.14(prettier@3.3.3)))(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(storybook@8.6.14(prettier@3.3.3))(typescript@5.8.3) + version: 8.6.14(@storybook/test@8.6.14(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1)))(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1))(typescript@5.8.3) '@storybook/react-vite': specifier: ^8.6.14 - version: 8.6.14(@storybook/test@8.6.14(storybook@8.6.14(prettier@3.3.3)))(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(rollup@4.41.1)(storybook@8.6.14(prettier@3.3.3))(typescript@5.8.3)(vite@6.3.5(@types/node@22.0.0)(yaml@2.8.0)) + version: 8.6.14(@storybook/test@8.6.14(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1)))(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(rollup@4.41.1)(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1))(supports-color@8.1.1)(typescript@5.8.3)(vite@6.3.5(@types/node@20.14.13)(yaml@2.8.0)) '@storybook/test': specifier: ^8.6.14 - version: 8.6.14(storybook@8.6.14(prettier@3.3.3)) + version: 8.6.14(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1)) '@storybook/theming': specifier: ^8.6.14 - version: 8.6.14(storybook@8.6.14(prettier@3.3.3)) + version: 8.6.14(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1)) '@types/lodash': specifier: ^4.17.7 version: 4.17.7 @@ -338,13 +351,13 @@ importers: version: 3.16.3 '@vitejs/plugin-react': specifier: ^4.5.0 - version: 4.5.0(vite@6.3.5(@types/node@22.0.0)(yaml@2.8.0)) + version: 4.5.0(supports-color@8.1.1)(vite@6.3.5(@types/node@20.14.13)(yaml@2.8.0)) '@vitest/browser': specifier: ^3.1.4 - version: 3.1.4(playwright@1.52.0)(vite@6.3.5(@types/node@22.0.0)(yaml@2.8.0))(vitest@3.1.4) + version: 3.1.4(playwright@1.52.0)(vite@6.3.5(@types/node@20.14.13)(yaml@2.8.0))(vitest@3.1.4) '@vitest/coverage-v8': specifier: ^3.1.4 - version: 3.1.4(@vitest/browser@3.1.4)(vitest@3.1.4) + version: 3.1.4(@vitest/browser@3.1.4)(supports-color@8.1.1)(vitest@3.1.4) immer: specifier: ^10.1.1 version: 10.1.1 @@ -359,7 +372,7 @@ importers: version: 15.8.1 storybook: specifier: ^8.6.14 - version: 8.6.14(prettier@3.3.3) + version: 8.6.14(prettier@3.3.3)(supports-color@8.1.1) tsconfig-paths: specifier: ^4.2.0 version: 4.2.0 @@ -368,13 +381,13 @@ importers: version: 5.8.3 vite: specifier: ^6.3.5 - version: 6.3.5(@types/node@22.0.0)(yaml@2.8.0) + version: 6.3.5(@types/node@20.14.13)(yaml@2.8.0) vite-tsconfig-paths: specifier: ^5.1.4 - version: 5.1.4(typescript@5.8.3)(vite@6.3.5(@types/node@22.0.0)(yaml@2.8.0)) + version: 5.1.4(supports-color@8.1.1)(typescript@5.8.3)(vite@6.3.5(@types/node@20.14.13)(yaml@2.8.0)) vitest: specifier: ^3.1.4 - version: 3.1.4(@types/node@22.0.0)(@vitest/browser@3.1.4)(yaml@2.8.0) + version: 3.1.4(@types/node@20.14.13)(@vitest/browser@3.1.4)(supports-color@8.1.1)(yaml@2.8.0) packages/validation: dependencies: @@ -821,24 +834,28 @@ packages: engines: {node: '>= 10'} cpu: [arm64] os: [linux] + libc: [glibc] '@nx/nx-linux-arm64-musl@16.10.0': resolution: {integrity: sha512-uO6Gg+irqpVcCKMcEPIQcTFZ+tDI02AZkqkP7koQAjniLEappd8DnUBSQdcn53T086pHpdc264X/ZEpXFfrKWQ==} engines: {node: '>= 10'} cpu: [arm64] os: [linux] + libc: [musl] '@nx/nx-linux-x64-gnu@16.10.0': resolution: {integrity: sha512-134PW/u/arNFAQKpqMJniC7irbChMPz+W+qtyKPAUXE0XFKPa7c1GtlI/wK2dvP9qJDZ6bKf0KtA0U/m2HMUOA==} engines: {node: '>= 10'} cpu: [x64] os: [linux] + libc: [glibc] '@nx/nx-linux-x64-musl@16.10.0': resolution: {integrity: sha512-q8sINYLdIJxK/iUx9vRk5jWAWb/2O0PAbOJFwv4qkxBv4rLoN7y+otgCZ5v0xfx/zztFgk/oNY4lg5xYjIso2Q==} engines: {node: '>= 10'} cpu: [x64] os: [linux] + libc: [musl] '@nx/nx-win32-arm64-msvc@16.10.0': resolution: {integrity: sha512-moJkL9kcqxUdJSRpG7dET3UeLIciwrfP08mzBQ12ewo8K8FzxU8ZUsTIVVdNrwt01CXOdXoweGfdQLjJ4qTURA==} @@ -945,56 +962,67 @@ packages: resolution: {integrity: sha512-wC53ZNDgt0pqx5xCAgNunkTzFE8GTgdZ9EwYGVcg+jEjJdZGtq9xPjDnFgfFozQI/Xm1mh+D9YlYtl+ueswNEg==} cpu: [arm] os: [linux] + libc: [glibc] '@rollup/rollup-linux-arm-musleabihf@4.41.1': resolution: {integrity: sha512-jwKCca1gbZkZLhLRtsrka5N8sFAaxrGz/7wRJ8Wwvq3jug7toO21vWlViihG85ei7uJTpzbXZRcORotE+xyrLA==} cpu: [arm] os: [linux] + libc: [musl] '@rollup/rollup-linux-arm64-gnu@4.41.1': resolution: {integrity: sha512-g0UBcNknsmmNQ8V2d/zD2P7WWfJKU0F1nu0k5pW4rvdb+BIqMm8ToluW/eeRmxCared5dD76lS04uL4UaNgpNA==} cpu: [arm64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-arm64-musl@4.41.1': resolution: {integrity: sha512-XZpeGB5TKEZWzIrj7sXr+BEaSgo/ma/kCgrZgL0oo5qdB1JlTzIYQKel/RmhT6vMAvOdM2teYlAaOGJpJ9lahg==} cpu: [arm64] os: [linux] + libc: [musl] '@rollup/rollup-linux-loongarch64-gnu@4.41.1': resolution: {integrity: sha512-bkCfDJ4qzWfFRCNt5RVV4DOw6KEgFTUZi2r2RuYhGWC8WhCA8lCAJhDeAmrM/fdiAH54m0mA0Vk2FGRPyzI+tw==} cpu: [loong64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-powerpc64le-gnu@4.41.1': resolution: {integrity: sha512-3mr3Xm+gvMX+/8EKogIZSIEF0WUu0HL9di+YWlJpO8CQBnoLAEL/roTCxuLncEdgcfJcvA4UMOf+2dnjl4Ut1A==} cpu: [ppc64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-riscv64-gnu@4.41.1': resolution: {integrity: sha512-3rwCIh6MQ1LGrvKJitQjZFuQnT2wxfU+ivhNBzmxXTXPllewOF7JR1s2vMX/tWtUYFgphygxjqMl76q4aMotGw==} cpu: [riscv64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-riscv64-musl@4.41.1': resolution: {integrity: sha512-LdIUOb3gvfmpkgFZuccNa2uYiqtgZAz3PTzjuM5bH3nvuy9ty6RGc/Q0+HDFrHrizJGVpjnTZ1yS5TNNjFlklw==} cpu: [riscv64] os: [linux] + libc: [musl] '@rollup/rollup-linux-s390x-gnu@4.41.1': resolution: {integrity: sha512-oIE6M8WC9ma6xYqjvPhzZYk6NbobIURvP/lEbh7FWplcMO6gn7MM2yHKA1eC/GvYwzNKK/1LYgqzdkZ8YFxR8g==} cpu: [s390x] os: [linux] + libc: [glibc] '@rollup/rollup-linux-x64-gnu@4.41.1': resolution: {integrity: sha512-cWBOvayNvA+SyeQMp79BHPK8ws6sHSsYnK5zDcsC3Hsxr1dgTABKjMnMslPq1DvZIp6uO7kIWhiGwaTdR4Og9A==} cpu: [x64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-x64-musl@4.41.1': resolution: {integrity: sha512-y5CbN44M+pUCdGDlZFzGGBSKCA4A/J2ZH4edTYSSxFg7ce1Xt3GtydbVKWLlzL+INfFIZAEg1ZV6hh9+QQf9YQ==} cpu: [x64] os: [linux] + libc: [musl] '@rollup/rollup-win32-arm64-msvc@4.41.1': resolution: {integrity: sha512-lZkCxIrjlJlMt1dLO/FbpZbzt6J/A8p4DnqzSa4PWqPEUUUnzXLeki/iyPLfV0BmHItlYgHUqJe+3KiyydmiNQ==} @@ -1034,6 +1062,9 @@ packages: '@sinonjs/text-encoding@0.7.2': resolution: {integrity: sha512-sXXKG+uL9IrKqViTtao2Ws6dy0znu9sOaP1di/jKGW1M6VssO8vlpXCQcpZ+jisQ1tTFAC5Jo/EOzFbggBagFQ==} + deprecated: |- + Deprecated: no longer maintained and no longer used by Sinon packages. See + https://github.com/sinonjs/nise/issues/243 for replacement details. '@storybook/addon-actions@8.6.14': resolution: {integrity: sha512-mDQxylxGGCQSK7tJPkD144J8jWh9IU9ziJMHfB84PKpI/V5ZgqMDnpr2bssTrUaGDqU5e1/z8KcRF+Melhs9pQ==} @@ -1440,9 +1471,6 @@ packages: '@types/node@20.14.13': resolution: {integrity: sha512-+bHoGiZb8UiQ0+WEtmph2IWQCjIqg8MDZMAV+ppRRhUZnquF5mQkP/9vpSwJClEiSM/C7fZZExPzfU0vJTyp8w==} - '@types/node@22.0.0': - resolution: {integrity: sha512-VT7KSYudcPOzP5Q0wfbowyNLaVR8QWUdw+088uFWwfvpY6uCWaXpqV6ieLAu9WBcnTa7H4Z5RLK8I5t2FuOcqw==} - '@types/parse-json@4.0.2': resolution: {integrity: sha512-dISoDXWWQwUquiKsyZ4Ng+HX2KsPL7LyHKHQwgGFEA3IaKac4Obd+h2a/a6waisAoepJlBcx9paWqjA8/HVjCw==} @@ -2410,6 +2438,10 @@ packages: resolution: {integrity: sha512-11Ndz7Nv+mvAC1j0ktTa7fAb0vLyGGX+rMHNBYQviQDGU0Hw7lhctJANqbPhu9nV9/izT/IntTgZ7Im/9LJs9g==} engines: {'0': node >=0.6.0} + fast-check@4.9.0: + resolution: {integrity: sha512-7ms6T7SybUev/PQITciI0yLM2pOSFy5zpG8Ty7tQofcVaQUvrMXp6CBwqF6fThLCLOrfBtuHAtwq6Yu4XPCllg==} + engines: {node: '>=12.17.0'} + fast-deep-equal@3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} @@ -3633,6 +3665,9 @@ packages: resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} engines: {node: '>=6'} + pure-rand@8.4.2: + resolution: {integrity: sha512-vvuOGgcuPJAirlHvuQw1TrOiw7ptaIXXmIbNuiNOY6lNGJJH49PQ1Kj4nd783nPdQhQdicgOjVI2yI/9BD6/Ng==} + qs@6.12.3: resolution: {integrity: sha512-AWJm14H1vVaO/iNZ4/hO+HyaTehuy9nRqVdkTqlJt0HWvBiBIEXFmb4C0DGeYo3Xes9rrEW+TxHsaigCbN5ICQ==} engines: {node: '>=0.6'} @@ -4297,6 +4332,7 @@ packages: tsconfck@3.1.6: resolution: {integrity: sha512-ks6Vjr/jEw0P1gmOVwutM3B7fWxoWBL2KRDb1JfqGVawBmO5UsvmWOQFGHBPl5yxYz4eERr19E6L7NMv+Fej4w==} engines: {node: ^18 || >=20} + deprecated: unmaintained hasBin: true peerDependencies: typescript: ^5.0.0 @@ -4392,9 +4428,6 @@ packages: undici-types@5.26.5: resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} - undici-types@6.11.1: - resolution: {integrity: sha512-mIDEX2ek50x0OlRgxryxsenE5XaQD4on5U2inY7RApK3SOJpofyw7uW2AyfMKkhAxXIceo2DeWGVGwyvng1GNQ==} - universalify@0.1.2: resolution: {integrity: sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==} engines: {node: '>= 4.0.0'} @@ -4433,15 +4466,17 @@ packages: uuid@3.4.0: resolution: {integrity: sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==} - deprecated: Please upgrade to version 7 or higher. Older versions may use Math.random() in certain circumstances, which is known to be problematic. See https://v8.dev/blog/math-random for details. + deprecated: uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028). hasBin: true uuid@8.3.2: resolution: {integrity: sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==} + deprecated: uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028). hasBin: true uuid@9.0.1: resolution: {integrity: sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==} + deprecated: uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028). hasBin: true v8-compile-cache-lib@3.0.1: @@ -4744,20 +4779,20 @@ snapshots: '@babel/compat-data@7.27.2': {} - '@babel/core@7.27.1': + '@babel/core@7.27.1(supports-color@8.1.1)': dependencies: '@ampproject/remapping': 2.3.0 '@babel/code-frame': 7.27.1 '@babel/generator': 7.27.1 '@babel/helper-compilation-targets': 7.27.2 - '@babel/helper-module-transforms': 7.27.1(@babel/core@7.27.1) + '@babel/helper-module-transforms': 7.27.1(@babel/core@7.27.1(supports-color@8.1.1))(supports-color@8.1.1) '@babel/helpers': 7.27.1 '@babel/parser': 7.27.5 '@babel/template': 7.27.2 - '@babel/traverse': 7.27.1 + '@babel/traverse': 7.27.1(supports-color@8.1.1) '@babel/types': 7.27.1 convert-source-map: 2.0.0 - debug: 4.4.1 + debug: 4.4.1(supports-color@8.1.1) gensync: 1.0.0-beta.2 json5: 2.2.3 semver: 6.3.1 @@ -4780,19 +4815,19 @@ snapshots: lru-cache: 5.1.1 semver: 6.3.1 - '@babel/helper-module-imports@7.27.1': + '@babel/helper-module-imports@7.27.1(supports-color@8.1.1)': dependencies: - '@babel/traverse': 7.27.1 + '@babel/traverse': 7.27.1(supports-color@8.1.1) '@babel/types': 7.27.1 transitivePeerDependencies: - supports-color - '@babel/helper-module-transforms@7.27.1(@babel/core@7.27.1)': + '@babel/helper-module-transforms@7.27.1(@babel/core@7.27.1(supports-color@8.1.1))(supports-color@8.1.1)': dependencies: - '@babel/core': 7.27.1 - '@babel/helper-module-imports': 7.27.1 + '@babel/core': 7.27.1(supports-color@8.1.1) + '@babel/helper-module-imports': 7.27.1(supports-color@8.1.1) '@babel/helper-validator-identifier': 7.27.1 - '@babel/traverse': 7.27.1 + '@babel/traverse': 7.27.1(supports-color@8.1.1) transitivePeerDependencies: - supports-color @@ -4820,14 +4855,14 @@ snapshots: dependencies: '@babel/types': 7.27.3 - '@babel/plugin-transform-react-jsx-self@7.27.1(@babel/core@7.27.1)': + '@babel/plugin-transform-react-jsx-self@7.27.1(@babel/core@7.27.1(supports-color@8.1.1))': dependencies: - '@babel/core': 7.27.1 + '@babel/core': 7.27.1(supports-color@8.1.1) '@babel/helper-plugin-utils': 7.27.1 - '@babel/plugin-transform-react-jsx-source@7.27.1(@babel/core@7.27.1)': + '@babel/plugin-transform-react-jsx-source@7.27.1(@babel/core@7.27.1(supports-color@8.1.1))': dependencies: - '@babel/core': 7.27.1 + '@babel/core': 7.27.1(supports-color@8.1.1) '@babel/helper-plugin-utils': 7.27.1 '@babel/runtime@7.25.0': @@ -4840,14 +4875,14 @@ snapshots: '@babel/parser': 7.27.5 '@babel/types': 7.27.1 - '@babel/traverse@7.27.1': + '@babel/traverse@7.27.1(supports-color@8.1.1)': dependencies: '@babel/code-frame': 7.27.1 '@babel/generator': 7.27.1 '@babel/parser': 7.27.5 '@babel/template': 7.27.2 '@babel/types': 7.27.1 - debug: 4.4.1 + debug: 4.4.1(supports-color@8.1.1) globals: 11.12.0 transitivePeerDependencies: - supports-color @@ -4864,13 +4899,13 @@ snapshots: '@bcoe/v8-coverage@1.0.2': {} - '@chromatic-com/storybook@3.2.6(react@19.1.0)(storybook@8.6.14(prettier@3.3.3))': + '@chromatic-com/storybook@3.2.6(react@19.1.0)(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1))': dependencies: chromatic: 11.28.3 filesize: 10.1.6 jsonfile: 6.1.0 react-confetti: 6.4.0(react@19.1.0) - storybook: 8.6.14(prettier@3.3.3) + storybook: 8.6.14(prettier@3.3.3)(supports-color@8.1.1) strip-ansi: 7.1.0 transitivePeerDependencies: - '@chromatic-com/cypress' @@ -4889,9 +4924,9 @@ snapshots: enabled: 2.0.0 kuler: 2.0.0 - '@emotion/babel-plugin@11.13.5': + '@emotion/babel-plugin@11.13.5(supports-color@8.1.1)': dependencies: - '@babel/helper-module-imports': 7.27.1 + '@babel/helper-module-imports': 7.27.1(supports-color@8.1.1) '@babel/runtime': 7.25.0 '@emotion/hash': 0.9.2 '@emotion/memoize': 0.9.0 @@ -4921,10 +4956,10 @@ snapshots: '@emotion/memoize@0.9.0': {} - '@emotion/react@11.14.0(@types/react@19.1.5)(react@19.1.0)': + '@emotion/react@11.14.0(@types/react@19.1.5)(react@19.1.0)(supports-color@8.1.1)': dependencies: '@babel/runtime': 7.25.0 - '@emotion/babel-plugin': 11.13.5 + '@emotion/babel-plugin': 11.13.5(supports-color@8.1.1) '@emotion/cache': 11.14.0 '@emotion/serialize': 1.3.3 '@emotion/use-insertion-effect-with-fallbacks': 1.2.0(react@19.1.0) @@ -4947,12 +4982,12 @@ snapshots: '@emotion/sheet@1.4.0': {} - '@emotion/styled@11.14.0(@emotion/react@11.14.0(@types/react@19.1.5)(react@19.1.0))(@types/react@19.1.5)(react@19.1.0)': + '@emotion/styled@11.14.0(@emotion/react@11.14.0(@types/react@19.1.5)(react@19.1.0)(supports-color@8.1.1))(@types/react@19.1.5)(react@19.1.0)(supports-color@8.1.1)': dependencies: '@babel/runtime': 7.25.0 - '@emotion/babel-plugin': 11.13.5 + '@emotion/babel-plugin': 11.13.5(supports-color@8.1.1) '@emotion/is-prop-valid': 1.3.1 - '@emotion/react': 11.14.0(@types/react@19.1.5)(react@19.1.0) + '@emotion/react': 11.14.0(@types/react@19.1.5)(react@19.1.0)(supports-color@8.1.1) '@emotion/serialize': 1.3.3 '@emotion/use-insertion-effect-with-fallbacks': 1.2.0(react@19.1.0) '@emotion/utils': 1.4.2 @@ -5070,12 +5105,12 @@ snapshots: dependencies: '@sinclair/typebox': 0.27.8 - '@joshwooding/vite-plugin-react-docgen-typescript@0.5.0(typescript@5.8.3)(vite@6.3.5(@types/node@22.0.0)(yaml@2.8.0))': + '@joshwooding/vite-plugin-react-docgen-typescript@0.5.0(typescript@5.8.3)(vite@6.3.5(@types/node@20.14.13)(yaml@2.8.0))': dependencies: glob: 10.4.5 magic-string: 0.27.0 react-docgen-typescript: 2.2.2(typescript@5.8.3) - vite: 6.3.5(@types/node@22.0.0)(yaml@2.8.0) + vite: 6.3.5(@types/node@20.14.13)(yaml@2.8.0) optionalDependencies: typescript: 5.8.3 @@ -5112,9 +5147,9 @@ snapshots: sparse-bitfield: 3.0.3 optional: true - '@nrwl/tao@16.10.0': + '@nrwl/tao@16.10.0(debug@4.4.1(supports-color@8.1.1))': dependencies: - nx: 16.10.0 + nx: 16.10.0(debug@4.4.1(supports-color@8.1.1)) tslib: 2.6.3 transitivePeerDependencies: - '@swc-node/register' @@ -5337,112 +5372,112 @@ snapshots: '@sinonjs/text-encoding@0.7.2': {} - '@storybook/addon-actions@8.6.14(storybook@8.6.14(prettier@3.3.3))': + '@storybook/addon-actions@8.6.14(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1))': dependencies: '@storybook/global': 5.0.0 '@types/uuid': 9.0.8 dequal: 2.0.3 polished: 4.3.1 - storybook: 8.6.14(prettier@3.3.3) + storybook: 8.6.14(prettier@3.3.3)(supports-color@8.1.1) uuid: 9.0.1 - '@storybook/addon-backgrounds@8.6.14(storybook@8.6.14(prettier@3.3.3))': + '@storybook/addon-backgrounds@8.6.14(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1))': dependencies: '@storybook/global': 5.0.0 memoizerific: 1.11.3 - storybook: 8.6.14(prettier@3.3.3) + storybook: 8.6.14(prettier@3.3.3)(supports-color@8.1.1) ts-dedent: 2.2.0 - '@storybook/addon-controls@8.6.14(storybook@8.6.14(prettier@3.3.3))': + '@storybook/addon-controls@8.6.14(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1))': dependencies: '@storybook/global': 5.0.0 dequal: 2.0.3 - storybook: 8.6.14(prettier@3.3.3) + storybook: 8.6.14(prettier@3.3.3)(supports-color@8.1.1) ts-dedent: 2.2.0 - '@storybook/addon-docs@8.6.14(@types/react@19.1.5)(storybook@8.6.14(prettier@3.3.3))': + '@storybook/addon-docs@8.6.14(@types/react@19.1.5)(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1))': dependencies: '@mdx-js/react': 3.1.0(@types/react@19.1.5)(react@19.1.0) - '@storybook/blocks': 8.6.14(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(storybook@8.6.14(prettier@3.3.3)) - '@storybook/csf-plugin': 8.6.14(storybook@8.6.14(prettier@3.3.3)) - '@storybook/react-dom-shim': 8.6.14(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(storybook@8.6.14(prettier@3.3.3)) + '@storybook/blocks': 8.6.14(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1)) + '@storybook/csf-plugin': 8.6.14(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1)) + '@storybook/react-dom-shim': 8.6.14(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1)) react: 19.1.0 react-dom: 19.1.0(react@19.1.0) - storybook: 8.6.14(prettier@3.3.3) + storybook: 8.6.14(prettier@3.3.3)(supports-color@8.1.1) ts-dedent: 2.2.0 transitivePeerDependencies: - '@types/react' - '@storybook/addon-essentials@8.6.14(@types/react@19.1.5)(storybook@8.6.14(prettier@3.3.3))': - dependencies: - '@storybook/addon-actions': 8.6.14(storybook@8.6.14(prettier@3.3.3)) - '@storybook/addon-backgrounds': 8.6.14(storybook@8.6.14(prettier@3.3.3)) - '@storybook/addon-controls': 8.6.14(storybook@8.6.14(prettier@3.3.3)) - '@storybook/addon-docs': 8.6.14(@types/react@19.1.5)(storybook@8.6.14(prettier@3.3.3)) - '@storybook/addon-highlight': 8.6.14(storybook@8.6.14(prettier@3.3.3)) - '@storybook/addon-measure': 8.6.14(storybook@8.6.14(prettier@3.3.3)) - '@storybook/addon-outline': 8.6.14(storybook@8.6.14(prettier@3.3.3)) - '@storybook/addon-toolbars': 8.6.14(storybook@8.6.14(prettier@3.3.3)) - '@storybook/addon-viewport': 8.6.14(storybook@8.6.14(prettier@3.3.3)) - storybook: 8.6.14(prettier@3.3.3) + '@storybook/addon-essentials@8.6.14(@types/react@19.1.5)(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1))': + dependencies: + '@storybook/addon-actions': 8.6.14(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1)) + '@storybook/addon-backgrounds': 8.6.14(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1)) + '@storybook/addon-controls': 8.6.14(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1)) + '@storybook/addon-docs': 8.6.14(@types/react@19.1.5)(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1)) + '@storybook/addon-highlight': 8.6.14(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1)) + '@storybook/addon-measure': 8.6.14(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1)) + '@storybook/addon-outline': 8.6.14(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1)) + '@storybook/addon-toolbars': 8.6.14(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1)) + '@storybook/addon-viewport': 8.6.14(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1)) + storybook: 8.6.14(prettier@3.3.3)(supports-color@8.1.1) ts-dedent: 2.2.0 transitivePeerDependencies: - '@types/react' - '@storybook/addon-highlight@8.6.14(storybook@8.6.14(prettier@3.3.3))': + '@storybook/addon-highlight@8.6.14(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1))': dependencies: '@storybook/global': 5.0.0 - storybook: 8.6.14(prettier@3.3.3) + storybook: 8.6.14(prettier@3.3.3)(supports-color@8.1.1) - '@storybook/addon-measure@8.6.14(storybook@8.6.14(prettier@3.3.3))': + '@storybook/addon-measure@8.6.14(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1))': dependencies: '@storybook/global': 5.0.0 - storybook: 8.6.14(prettier@3.3.3) + storybook: 8.6.14(prettier@3.3.3)(supports-color@8.1.1) tiny-invariant: 1.3.3 - '@storybook/addon-outline@8.6.14(storybook@8.6.14(prettier@3.3.3))': + '@storybook/addon-outline@8.6.14(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1))': dependencies: '@storybook/global': 5.0.0 - storybook: 8.6.14(prettier@3.3.3) + storybook: 8.6.14(prettier@3.3.3)(supports-color@8.1.1) ts-dedent: 2.2.0 - '@storybook/addon-toolbars@8.6.14(storybook@8.6.14(prettier@3.3.3))': + '@storybook/addon-toolbars@8.6.14(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1))': dependencies: - storybook: 8.6.14(prettier@3.3.3) + storybook: 8.6.14(prettier@3.3.3)(supports-color@8.1.1) - '@storybook/addon-viewport@8.6.14(storybook@8.6.14(prettier@3.3.3))': + '@storybook/addon-viewport@8.6.14(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1))': dependencies: memoizerific: 1.11.3 - storybook: 8.6.14(prettier@3.3.3) + storybook: 8.6.14(prettier@3.3.3)(supports-color@8.1.1) - '@storybook/blocks@8.6.14(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(storybook@8.6.14(prettier@3.3.3))': + '@storybook/blocks@8.6.14(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1))': dependencies: '@storybook/icons': 1.4.0(react-dom@19.1.0(react@19.1.0))(react@19.1.0) - storybook: 8.6.14(prettier@3.3.3) + storybook: 8.6.14(prettier@3.3.3)(supports-color@8.1.1) ts-dedent: 2.2.0 optionalDependencies: react: 19.1.0 react-dom: 19.1.0(react@19.1.0) - '@storybook/builder-vite@8.6.14(storybook@8.6.14(prettier@3.3.3))(vite@6.3.5(@types/node@22.0.0)(yaml@2.8.0))': + '@storybook/builder-vite@8.6.14(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1))(vite@6.3.5(@types/node@20.14.13)(yaml@2.8.0))': dependencies: - '@storybook/csf-plugin': 8.6.14(storybook@8.6.14(prettier@3.3.3)) + '@storybook/csf-plugin': 8.6.14(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1)) browser-assert: 1.2.1 - storybook: 8.6.14(prettier@3.3.3) + storybook: 8.6.14(prettier@3.3.3)(supports-color@8.1.1) ts-dedent: 2.2.0 - vite: 6.3.5(@types/node@22.0.0)(yaml@2.8.0) + vite: 6.3.5(@types/node@20.14.13)(yaml@2.8.0) - '@storybook/components@8.6.14(storybook@8.6.14(prettier@3.3.3))': + '@storybook/components@8.6.14(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1))': dependencies: - storybook: 8.6.14(prettier@3.3.3) + storybook: 8.6.14(prettier@3.3.3)(supports-color@8.1.1) - '@storybook/core@8.6.14(prettier@3.3.3)(storybook@8.6.14(prettier@3.3.3))': + '@storybook/core@8.6.14(prettier@3.3.3)(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1))(supports-color@8.1.1)': dependencies: - '@storybook/theming': 8.6.14(storybook@8.6.14(prettier@3.3.3)) + '@storybook/theming': 8.6.14(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1)) better-opn: 3.0.2 browser-assert: 1.2.1 esbuild: 0.25.4 - esbuild-register: 3.6.0(esbuild@0.25.4) + esbuild-register: 3.6.0(esbuild@0.25.4)(supports-color@8.1.1) jsdoc-type-pratt-parser: 4.1.0 process: 0.11.10 recast: 0.23.11 @@ -5457,25 +5492,25 @@ snapshots: - supports-color - utf-8-validate - '@storybook/csf-plugin@8.6.14(storybook@8.6.14(prettier@3.3.3))': + '@storybook/csf-plugin@8.6.14(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1))': dependencies: - storybook: 8.6.14(prettier@3.3.3) + storybook: 8.6.14(prettier@3.3.3)(supports-color@8.1.1) unplugin: 1.16.1 - '@storybook/experimental-addon-test@8.6.14(@vitest/browser@3.1.4)(@vitest/runner@3.1.4)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(storybook@8.6.14(prettier@3.3.3))(vitest@3.1.4)': + '@storybook/experimental-addon-test@8.6.14(@vitest/browser@3.1.4)(@vitest/runner@3.1.4)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1))(vitest@3.1.4)': dependencies: '@storybook/global': 5.0.0 '@storybook/icons': 1.4.0(react-dom@19.1.0(react@19.1.0))(react@19.1.0) - '@storybook/instrumenter': 8.6.14(storybook@8.6.14(prettier@3.3.3)) - '@storybook/test': 8.6.14(storybook@8.6.14(prettier@3.3.3)) + '@storybook/instrumenter': 8.6.14(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1)) + '@storybook/test': 8.6.14(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1)) polished: 4.3.1 prompts: 2.4.2 - storybook: 8.6.14(prettier@3.3.3) + storybook: 8.6.14(prettier@3.3.3)(supports-color@8.1.1) ts-dedent: 2.2.0 optionalDependencies: - '@vitest/browser': 3.1.4(playwright@1.52.0)(vite@6.3.5(@types/node@22.0.0)(yaml@2.8.0))(vitest@3.1.4) + '@vitest/browser': 3.1.4(playwright@1.52.0)(vite@6.3.5(@types/node@20.14.13)(yaml@2.8.0))(vitest@3.1.4) '@vitest/runner': 3.1.4 - vitest: 3.1.4(@types/node@22.0.0)(@vitest/browser@3.1.4)(yaml@2.8.0) + vitest: 3.1.4(@types/node@20.14.13)(@vitest/browser@3.1.4)(supports-color@8.1.1)(yaml@2.8.0) transitivePeerDependencies: - react - react-dom @@ -5487,77 +5522,77 @@ snapshots: react: 19.1.0 react-dom: 19.1.0(react@19.1.0) - '@storybook/instrumenter@8.6.14(storybook@8.6.14(prettier@3.3.3))': + '@storybook/instrumenter@8.6.14(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1))': dependencies: '@storybook/global': 5.0.0 '@vitest/utils': 2.1.9 - storybook: 8.6.14(prettier@3.3.3) + storybook: 8.6.14(prettier@3.3.3)(supports-color@8.1.1) - '@storybook/manager-api@8.6.14(storybook@8.6.14(prettier@3.3.3))': + '@storybook/manager-api@8.6.14(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1))': dependencies: - storybook: 8.6.14(prettier@3.3.3) + storybook: 8.6.14(prettier@3.3.3)(supports-color@8.1.1) - '@storybook/preview-api@8.6.14(storybook@8.6.14(prettier@3.3.3))': + '@storybook/preview-api@8.6.14(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1))': dependencies: - storybook: 8.6.14(prettier@3.3.3) + storybook: 8.6.14(prettier@3.3.3)(supports-color@8.1.1) - '@storybook/react-dom-shim@8.6.14(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(storybook@8.6.14(prettier@3.3.3))': + '@storybook/react-dom-shim@8.6.14(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1))': dependencies: react: 19.1.0 react-dom: 19.1.0(react@19.1.0) - storybook: 8.6.14(prettier@3.3.3) + storybook: 8.6.14(prettier@3.3.3)(supports-color@8.1.1) - '@storybook/react-vite@8.6.14(@storybook/test@8.6.14(storybook@8.6.14(prettier@3.3.3)))(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(rollup@4.41.1)(storybook@8.6.14(prettier@3.3.3))(typescript@5.8.3)(vite@6.3.5(@types/node@22.0.0)(yaml@2.8.0))': + '@storybook/react-vite@8.6.14(@storybook/test@8.6.14(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1)))(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(rollup@4.41.1)(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1))(supports-color@8.1.1)(typescript@5.8.3)(vite@6.3.5(@types/node@20.14.13)(yaml@2.8.0))': dependencies: - '@joshwooding/vite-plugin-react-docgen-typescript': 0.5.0(typescript@5.8.3)(vite@6.3.5(@types/node@22.0.0)(yaml@2.8.0)) + '@joshwooding/vite-plugin-react-docgen-typescript': 0.5.0(typescript@5.8.3)(vite@6.3.5(@types/node@20.14.13)(yaml@2.8.0)) '@rollup/pluginutils': 5.1.4(rollup@4.41.1) - '@storybook/builder-vite': 8.6.14(storybook@8.6.14(prettier@3.3.3))(vite@6.3.5(@types/node@22.0.0)(yaml@2.8.0)) - '@storybook/react': 8.6.14(@storybook/test@8.6.14(storybook@8.6.14(prettier@3.3.3)))(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(storybook@8.6.14(prettier@3.3.3))(typescript@5.8.3) + '@storybook/builder-vite': 8.6.14(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1))(vite@6.3.5(@types/node@20.14.13)(yaml@2.8.0)) + '@storybook/react': 8.6.14(@storybook/test@8.6.14(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1)))(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1))(typescript@5.8.3) find-up: 5.0.0 magic-string: 0.30.17 react: 19.1.0 - react-docgen: 7.1.1 + react-docgen: 7.1.1(supports-color@8.1.1) react-dom: 19.1.0(react@19.1.0) resolve: 1.22.8 - storybook: 8.6.14(prettier@3.3.3) + storybook: 8.6.14(prettier@3.3.3)(supports-color@8.1.1) tsconfig-paths: 4.2.0 - vite: 6.3.5(@types/node@22.0.0)(yaml@2.8.0) + vite: 6.3.5(@types/node@20.14.13)(yaml@2.8.0) optionalDependencies: - '@storybook/test': 8.6.14(storybook@8.6.14(prettier@3.3.3)) + '@storybook/test': 8.6.14(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1)) transitivePeerDependencies: - rollup - supports-color - typescript - '@storybook/react@8.6.14(@storybook/test@8.6.14(storybook@8.6.14(prettier@3.3.3)))(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(storybook@8.6.14(prettier@3.3.3))(typescript@5.8.3)': + '@storybook/react@8.6.14(@storybook/test@8.6.14(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1)))(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1))(typescript@5.8.3)': dependencies: - '@storybook/components': 8.6.14(storybook@8.6.14(prettier@3.3.3)) + '@storybook/components': 8.6.14(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1)) '@storybook/global': 5.0.0 - '@storybook/manager-api': 8.6.14(storybook@8.6.14(prettier@3.3.3)) - '@storybook/preview-api': 8.6.14(storybook@8.6.14(prettier@3.3.3)) - '@storybook/react-dom-shim': 8.6.14(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(storybook@8.6.14(prettier@3.3.3)) - '@storybook/theming': 8.6.14(storybook@8.6.14(prettier@3.3.3)) + '@storybook/manager-api': 8.6.14(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1)) + '@storybook/preview-api': 8.6.14(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1)) + '@storybook/react-dom-shim': 8.6.14(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1)) + '@storybook/theming': 8.6.14(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1)) react: 19.1.0 react-dom: 19.1.0(react@19.1.0) - storybook: 8.6.14(prettier@3.3.3) + storybook: 8.6.14(prettier@3.3.3)(supports-color@8.1.1) optionalDependencies: - '@storybook/test': 8.6.14(storybook@8.6.14(prettier@3.3.3)) + '@storybook/test': 8.6.14(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1)) typescript: 5.8.3 - '@storybook/test@8.6.14(storybook@8.6.14(prettier@3.3.3))': + '@storybook/test@8.6.14(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1))': dependencies: '@storybook/global': 5.0.0 - '@storybook/instrumenter': 8.6.14(storybook@8.6.14(prettier@3.3.3)) + '@storybook/instrumenter': 8.6.14(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1)) '@testing-library/dom': 10.4.0 '@testing-library/jest-dom': 6.5.0 '@testing-library/user-event': 14.5.2(@testing-library/dom@10.4.0) '@vitest/expect': 2.0.5 '@vitest/spy': 2.0.5 - storybook: 8.6.14(prettier@3.3.3) + storybook: 8.6.14(prettier@3.3.3)(supports-color@8.1.1) - '@storybook/theming@8.6.14(storybook@8.6.14(prettier@3.3.3))': + '@storybook/theming@8.6.14(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1))': dependencies: - storybook: 8.6.14(prettier@3.3.3) + storybook: 8.6.14(prettier@3.3.3)(supports-color@8.1.1) '@tanstack/react-table@8.21.3(react-dom@19.1.0(react@19.1.0))(react@19.1.0)': dependencies: @@ -5636,9 +5671,9 @@ snapshots: dependencies: '@types/chai': 4.3.16 - '@types/chai-http@4.2.4': + '@types/chai-http@4.2.4(supports-color@5.5.0)': dependencies: - chai-http: 4.4.0 + chai-http: 4.4.0(supports-color@5.5.0) transitivePeerDependencies: - supports-color @@ -5802,7 +5837,7 @@ snapshots: '@types/jsonwebtoken@8.5.9': dependencies: - '@types/node': 22.0.0 + '@types/node': 20.14.13 '@types/jszip@3.4.1': dependencies: @@ -5824,7 +5859,7 @@ snapshots: '@types/node-fetch@2.6.11': dependencies: - '@types/node': 12.20.55 + '@types/node': 20.14.13 form-data: 4.0.0 '@types/node@12.20.55': {} @@ -5833,10 +5868,6 @@ snapshots: dependencies: undici-types: 5.26.5 - '@types/node@22.0.0': - dependencies: - undici-types: 6.11.1 - '@types/parse-json@4.0.2': {} '@types/qs@6.9.15': {} @@ -5860,12 +5891,12 @@ snapshots: '@types/send@0.17.4': dependencies: '@types/mime': 1.3.5 - '@types/node': 22.0.0 + '@types/node': 20.14.13 '@types/serve-static@1.15.7': dependencies: '@types/http-errors': 2.0.4 - '@types/node': 22.0.0 + '@types/node': 20.14.13 '@types/send': 0.17.4 '@types/sinon@10.0.20': @@ -5882,7 +5913,7 @@ snapshots: '@types/superagent@4.1.24': dependencies: '@types/cookiejar': 2.1.5 - '@types/node': 22.0.0 + '@types/node': 20.14.13 '@types/swagger-ui-express@4.1.8': dependencies: @@ -5902,28 +5933,28 @@ snapshots: '@ungap/promise-all-settled@1.1.2': {} - '@vitejs/plugin-react@4.5.0(vite@6.3.5(@types/node@22.0.0)(yaml@2.8.0))': + '@vitejs/plugin-react@4.5.0(supports-color@8.1.1)(vite@6.3.5(@types/node@20.14.13)(yaml@2.8.0))': dependencies: - '@babel/core': 7.27.1 - '@babel/plugin-transform-react-jsx-self': 7.27.1(@babel/core@7.27.1) - '@babel/plugin-transform-react-jsx-source': 7.27.1(@babel/core@7.27.1) + '@babel/core': 7.27.1(supports-color@8.1.1) + '@babel/plugin-transform-react-jsx-self': 7.27.1(@babel/core@7.27.1(supports-color@8.1.1)) + '@babel/plugin-transform-react-jsx-source': 7.27.1(@babel/core@7.27.1(supports-color@8.1.1)) '@rolldown/pluginutils': 1.0.0-beta.9 '@types/babel__core': 7.20.5 react-refresh: 0.17.0 - vite: 6.3.5(@types/node@22.0.0)(yaml@2.8.0) + vite: 6.3.5(@types/node@20.14.13)(yaml@2.8.0) transitivePeerDependencies: - supports-color - '@vitest/browser@3.1.4(playwright@1.52.0)(vite@6.3.5(@types/node@22.0.0)(yaml@2.8.0))(vitest@3.1.4)': + '@vitest/browser@3.1.4(playwright@1.52.0)(vite@6.3.5(@types/node@20.14.13)(yaml@2.8.0))(vitest@3.1.4)': dependencies: '@testing-library/dom': 10.4.0 '@testing-library/user-event': 14.6.1(@testing-library/dom@10.4.0) - '@vitest/mocker': 3.1.4(vite@6.3.5(@types/node@22.0.0)(yaml@2.8.0)) + '@vitest/mocker': 3.1.4(vite@6.3.5(@types/node@20.14.13)(yaml@2.8.0)) '@vitest/utils': 3.1.4 magic-string: 0.30.17 sirv: 3.0.1 tinyrainbow: 2.0.0 - vitest: 3.1.4(@types/node@22.0.0)(@vitest/browser@3.1.4)(yaml@2.8.0) + vitest: 3.1.4(@types/node@20.14.13)(@vitest/browser@3.1.4)(supports-color@8.1.1)(yaml@2.8.0) ws: 8.18.2 optionalDependencies: playwright: 1.52.0 @@ -5933,23 +5964,23 @@ snapshots: - utf-8-validate - vite - '@vitest/coverage-v8@3.1.4(@vitest/browser@3.1.4)(vitest@3.1.4)': + '@vitest/coverage-v8@3.1.4(@vitest/browser@3.1.4)(supports-color@8.1.1)(vitest@3.1.4)': dependencies: '@ampproject/remapping': 2.3.0 '@bcoe/v8-coverage': 1.0.2 - debug: 4.4.1 + debug: 4.4.1(supports-color@8.1.1) istanbul-lib-coverage: 3.2.2 istanbul-lib-report: 3.0.1 - istanbul-lib-source-maps: 5.0.6 + istanbul-lib-source-maps: 5.0.6(supports-color@8.1.1) istanbul-reports: 3.1.7 magic-string: 0.30.17 magicast: 0.3.5 std-env: 3.9.0 test-exclude: 7.0.1 tinyrainbow: 2.0.0 - vitest: 3.1.4(@types/node@22.0.0)(@vitest/browser@3.1.4)(yaml@2.8.0) + vitest: 3.1.4(@types/node@20.14.13)(@vitest/browser@3.1.4)(supports-color@8.1.1)(yaml@2.8.0) optionalDependencies: - '@vitest/browser': 3.1.4(playwright@1.52.0)(vite@6.3.5(@types/node@22.0.0)(yaml@2.8.0))(vitest@3.1.4) + '@vitest/browser': 3.1.4(playwright@1.52.0)(vite@6.3.5(@types/node@20.14.13)(yaml@2.8.0))(vitest@3.1.4) transitivePeerDependencies: - supports-color @@ -5967,13 +5998,13 @@ snapshots: chai: 5.2.0 tinyrainbow: 2.0.0 - '@vitest/mocker@3.1.4(vite@6.3.5(@types/node@22.0.0)(yaml@2.8.0))': + '@vitest/mocker@3.1.4(vite@6.3.5(@types/node@20.14.13)(yaml@2.8.0))': dependencies: '@vitest/spy': 3.1.4 estree-walker: 3.0.3 magic-string: 0.30.17 optionalDependencies: - vite: 6.3.5(@types/node@22.0.0)(yaml@2.8.0) + vite: 6.3.5(@types/node@20.14.13)(yaml@2.8.0) '@vitest/pretty-format@2.0.5': dependencies: @@ -6157,17 +6188,25 @@ snapshots: aws4@1.13.0: {} - axios@1.7.2: + axios@1.7.2(debug@4.4.1(supports-color@8.1.1)): dependencies: - follow-redirects: 1.15.6 + follow-redirects: 1.15.6(debug@4.4.1(supports-color@8.1.1)) form-data: 4.0.0 proxy-from-env: 1.1.0 transitivePeerDependencies: - debug - axios@1.9.0: + axios@1.9.0(debug@4.4.1(supports-color@5.5.0)): + dependencies: + follow-redirects: 1.15.6(debug@4.4.1(supports-color@5.5.0)) + form-data: 4.0.0 + proxy-from-env: 1.1.0 + transitivePeerDependencies: + - debug + + axios@1.9.0(debug@4.4.1(supports-color@8.1.1)): dependencies: - follow-redirects: 1.15.6 + follow-redirects: 1.15.6(debug@4.4.1(supports-color@8.1.1)) form-data: 4.0.0 proxy-from-env: 1.1.0 transitivePeerDependencies: @@ -6204,11 +6243,11 @@ snapshots: inherits: 2.0.4 readable-stream: 3.6.2 - body-parser@2.2.0: + body-parser@2.2.0(supports-color@5.5.0): dependencies: bytes: 3.1.2 content-type: 1.0.5 - debug: 4.4.1 + debug: 4.4.1(supports-color@5.5.0) http-errors: 2.0.0 iconv-lite: 0.6.3 on-finished: 2.4.1 @@ -6319,7 +6358,20 @@ snapshots: chai: 4.5.0 check-error: 1.0.3 - chai-http@4.4.0: + chai-http@4.4.0(supports-color@5.5.0): + dependencies: + '@types/chai': 4.3.16 + '@types/superagent': 4.1.13 + charset: 1.0.1 + cookiejar: 2.1.4 + is-ip: 2.0.0 + methods: 1.1.2 + qs: 6.12.3 + superagent: 8.1.2(supports-color@5.5.0) + transitivePeerDependencies: + - supports-color + + chai-http@4.4.0(supports-color@8.1.1): dependencies: '@types/chai': 4.3.16 '@types/superagent': 4.1.13 @@ -6328,7 +6380,7 @@ snapshots: is-ip: 2.0.0 methods: 1.1.2 qs: 6.12.3 - superagent: 8.1.2 + superagent: 8.1.2(supports-color@8.1.1) transitivePeerDependencies: - supports-color @@ -6599,9 +6651,11 @@ snapshots: dependencies: '@babel/runtime': 7.25.0 - debug@3.1.0: + debug@3.1.0(supports-color@5.5.0): dependencies: ms: 2.0.0 + optionalDependencies: + supports-color: 5.5.0 debug@3.2.7(supports-color@5.5.0): dependencies: @@ -6621,9 +6675,17 @@ snapshots: optionalDependencies: supports-color: 8.1.1 - debug@4.4.1: + debug@4.4.1(supports-color@5.5.0): dependencies: ms: 2.1.3 + optionalDependencies: + supports-color: 5.5.0 + + debug@4.4.1(supports-color@8.1.1): + dependencies: + ms: 2.1.3 + optionalDependencies: + supports-color: 8.1.1 decamelize@1.2.0: {} @@ -6666,7 +6728,7 @@ snapshots: diff@5.2.0: {} - docker-modem@1.0.9: + docker-modem@1.0.9(supports-color@5.5.0): dependencies: JSONStream: 1.3.2 debug: 3.2.7(supports-color@5.5.0) @@ -6675,10 +6737,10 @@ snapshots: transitivePeerDependencies: - supports-color - dockerode@2.5.8: + dockerode@2.5.8(supports-color@5.5.0): dependencies: concat-stream: 1.6.2 - docker-modem: 1.0.9 + docker-modem: 1.0.9(supports-color@5.5.0) tar-fs: 1.16.3 transitivePeerDependencies: - supports-color @@ -6784,9 +6846,9 @@ snapshots: es6-iterator: 2.0.3 es6-symbol: 3.1.4 - esbuild-register@3.6.0(esbuild@0.25.4): + esbuild-register@3.6.0(esbuild@0.25.4)(supports-color@8.1.1): dependencies: - debug: 4.4.1 + debug: 4.4.1(supports-color@8.1.1) esbuild: 0.25.4 transitivePeerDependencies: - supports-color @@ -6867,19 +6929,19 @@ snapshots: expect-type@1.2.1: {} - express@5.1.0: + express@5.1.0(supports-color@5.5.0): dependencies: accepts: 2.0.0 - body-parser: 2.2.0 + body-parser: 2.2.0(supports-color@5.5.0) content-disposition: 1.0.0 content-type: 1.0.5 cookie: 0.7.1 cookie-signature: 1.2.2 - debug: 4.4.1 + debug: 4.4.1(supports-color@5.5.0) encodeurl: 2.0.0 escape-html: 1.0.3 etag: 1.8.1 - finalhandler: 2.1.0 + finalhandler: 2.1.0(supports-color@5.5.0) fresh: 2.0.0 http-errors: 2.0.0 merge-descriptors: 2.0.0 @@ -6890,9 +6952,9 @@ snapshots: proxy-addr: 2.0.7 qs: 6.14.0 range-parser: 1.2.1 - router: 2.2.0 - send: 1.2.0 - serve-static: 2.2.0 + router: 2.2.0(supports-color@5.5.0) + send: 1.2.0(supports-color@5.5.0) + serve-static: 2.2.0(supports-color@5.5.0) statuses: 2.0.1 type-is: 2.0.1 vary: 1.1.2 @@ -6907,6 +6969,10 @@ snapshots: extsprintf@1.3.0: {} + fast-check@4.9.0: + dependencies: + pure-rand: 8.4.2 + fast-deep-equal@3.1.3: {} fast-json-stable-stringify@2.1.0: {} @@ -6931,9 +6997,9 @@ snapshots: dependencies: to-regex-range: 5.0.1 - finalhandler@2.1.0: + finalhandler@2.1.0(supports-color@5.5.0): dependencies: - debug: 4.4.1 + debug: 4.4.1(supports-color@5.5.0) encodeurl: 2.0.0 escape-html: 1.0.3 on-finished: 2.4.1 @@ -6968,7 +7034,13 @@ snapshots: fn.name@1.1.0: {} - follow-redirects@1.15.6: {} + follow-redirects@1.15.6(debug@4.4.1(supports-color@5.5.0)): + optionalDependencies: + debug: 4.4.1(supports-color@5.5.0) + + follow-redirects@1.15.6(debug@4.4.1(supports-color@8.1.1)): + optionalDependencies: + debug: 4.4.1(supports-color@8.1.1) for-each@0.3.5: dependencies: @@ -7334,9 +7406,9 @@ snapshots: dependencies: append-transform: 2.0.0 - istanbul-lib-instrument@4.0.3: + istanbul-lib-instrument@4.0.3(supports-color@8.1.1): dependencies: - '@babel/core': 7.27.1 + '@babel/core': 7.27.1(supports-color@8.1.1) '@istanbuljs/schema': 0.1.3 istanbul-lib-coverage: 3.2.2 semver: 6.3.1 @@ -7358,18 +7430,18 @@ snapshots: make-dir: 4.0.0 supports-color: 7.2.0 - istanbul-lib-source-maps@4.0.1: + istanbul-lib-source-maps@4.0.1(supports-color@8.1.1): dependencies: - debug: 4.4.1 + debug: 4.4.1(supports-color@8.1.1) istanbul-lib-coverage: 3.2.2 source-map: 0.6.1 transitivePeerDependencies: - supports-color - istanbul-lib-source-maps@5.0.6: + istanbul-lib-source-maps@5.0.6(supports-color@8.1.1): dependencies: '@jridgewell/trace-mapping': 0.3.25 - debug: 4.4.1 + debug: 4.4.1(supports-color@8.1.1) istanbul-lib-coverage: 3.2.2 transitivePeerDependencies: - supports-color @@ -7748,13 +7820,13 @@ snapshots: optionalDependencies: '@mongodb-js/saslprep': 1.1.8 - mongoose@7.8.0: + mongoose@7.8.0(supports-color@5.5.0): dependencies: bson: 5.5.1 kareem: 2.5.1 mongodb: 5.9.2 mpath: 0.9.0 - mquery: 5.0.0 + mquery: 5.0.0(supports-color@5.5.0) ms: 2.1.3 sift: 16.0.1 transitivePeerDependencies: @@ -7767,9 +7839,9 @@ snapshots: mpath@0.9.0: {} - mquery@5.0.0: + mquery@5.0.0(supports-color@5.5.0): dependencies: - debug: 4.4.1 + debug: 4.4.1(supports-color@5.5.0) transitivePeerDependencies: - supports-color @@ -7819,9 +7891,9 @@ snapshots: node-releases@2.0.19: {} - node-vault@0.9.22: + node-vault@0.9.22(supports-color@5.5.0): dependencies: - debug: 3.1.0 + debug: 3.1.0(supports-color@5.5.0) mustache: 2.3.2 request: 2.88.0 request-promise-native: 1.0.7(request@2.88.0) @@ -7855,14 +7927,14 @@ snapshots: dependencies: path-key: 3.1.1 - nx@16.10.0: + nx@16.10.0(debug@4.4.1(supports-color@8.1.1)): dependencies: - '@nrwl/tao': 16.10.0 + '@nrwl/tao': 16.10.0(debug@4.4.1(supports-color@8.1.1)) '@parcel/watcher': 2.0.4 '@yarnpkg/lockfile': 1.1.0 '@yarnpkg/parsers': 3.0.0-rc.46 '@zkochan/js-yaml': 0.0.6 - axios: 1.7.2 + axios: 1.7.2(debug@4.4.1(supports-color@8.1.1)) chalk: 4.1.2 cli-cursor: 3.1.0 cli-spinners: 2.6.1 @@ -7907,7 +7979,7 @@ snapshots: transitivePeerDependencies: - debug - nyc@15.1.0: + nyc@15.1.0(supports-color@8.1.1): dependencies: '@istanbuljs/load-nyc-config': 1.1.0 '@istanbuljs/schema': 0.1.3 @@ -7921,10 +7993,10 @@ snapshots: glob: 7.2.3 istanbul-lib-coverage: 3.2.2 istanbul-lib-hook: 3.0.0 - istanbul-lib-instrument: 4.0.3 + istanbul-lib-instrument: 4.0.3(supports-color@8.1.1) istanbul-lib-processinfo: 2.0.3 istanbul-lib-report: 3.0.1 - istanbul-lib-source-maps: 4.0.1 + istanbul-lib-source-maps: 4.0.1(supports-color@8.1.1) istanbul-reports: 3.1.7 make-dir: 3.1.0 node-preload: 0.2.1 @@ -8160,6 +8232,8 @@ snapshots: punycode@2.3.1: {} + pure-rand@8.4.2: {} + qs@6.12.3: dependencies: side-channel: 1.0.6 @@ -8198,10 +8272,10 @@ snapshots: dependencies: typescript: 5.8.3 - react-docgen@7.1.1: + react-docgen@7.1.1(supports-color@8.1.1): dependencies: - '@babel/core': 7.27.1 - '@babel/traverse': 7.27.1 + '@babel/core': 7.27.1(supports-color@8.1.1) + '@babel/traverse': 7.27.1(supports-color@8.1.1) '@babel/types': 7.27.1 '@types/babel__core': 7.20.5 '@types/babel__traverse': 7.20.7 @@ -8407,9 +8481,9 @@ snapshots: '@rollup/rollup-win32-x64-msvc': 4.41.1 fsevents: 2.3.3 - router@2.2.0: + router@2.2.0(supports-color@5.5.0): dependencies: - debug: 4.4.1 + debug: 4.4.1(supports-color@5.5.0) depd: 2.0.0 is-promise: 4.0.0 parseurl: 1.3.3 @@ -8449,9 +8523,9 @@ snapshots: semver@7.6.3: {} - send@1.2.0: + send@1.2.0(supports-color@5.5.0): dependencies: - debug: 4.4.1 + debug: 4.4.1(supports-color@5.5.0) encodeurl: 2.0.0 escape-html: 1.0.3 etag: 1.8.1 @@ -8473,12 +8547,12 @@ snapshots: dependencies: randombytes: 2.1.0 - serve-static@2.2.0: + serve-static@2.2.0(supports-color@5.5.0): dependencies: encodeurl: 2.0.0 escape-html: 1.0.3 parseurl: 1.3.3 - send: 1.2.0 + send: 1.2.0(supports-color@5.5.0) transitivePeerDependencies: - supports-color @@ -8653,9 +8727,9 @@ snapshots: stealthy-require@1.1.1: {} - storybook@8.6.14(prettier@3.3.3): + storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1): dependencies: - '@storybook/core': 8.6.14(prettier@3.3.3)(storybook@8.6.14(prettier@3.3.3)) + '@storybook/core': 8.6.14(prettier@3.3.3)(storybook@8.6.14(prettier@3.3.3)(supports-color@8.1.1))(supports-color@8.1.1) optionalDependencies: prettier: 3.3.3 transitivePeerDependencies: @@ -8746,11 +8820,26 @@ snapshots: stylis@4.2.0: {} - superagent@8.1.2: + superagent@8.1.2(supports-color@5.5.0): + dependencies: + component-emitter: 1.3.1 + cookiejar: 2.1.4 + debug: 4.4.1(supports-color@5.5.0) + fast-safe-stringify: 2.1.1 + form-data: 4.0.0 + formidable: 2.1.2 + methods: 1.1.2 + mime: 2.6.0 + qs: 6.12.3 + semver: 7.6.3 + transitivePeerDependencies: + - supports-color + + superagent@8.1.2(supports-color@8.1.1): dependencies: component-emitter: 1.3.1 cookiejar: 2.1.4 - debug: 4.4.1 + debug: 4.4.1(supports-color@8.1.1) fast-safe-stringify: 2.1.1 form-data: 4.0.0 formidable: 2.1.2 @@ -8783,9 +8872,9 @@ snapshots: dependencies: '@scarf/scarf': 1.4.0 - swagger-ui-express@5.0.1(express@5.1.0): + swagger-ui-express@5.0.1(express@5.1.0(supports-color@5.5.0)): dependencies: - express: 5.1.0 + express: 5.1.0(supports-color@5.5.0) swagger-ui-dist: 5.21.0 tar-fs@1.16.3: @@ -8832,11 +8921,11 @@ snapshots: glob: 10.4.5 minimatch: 9.0.5 - testcontainers@1.3.1: + testcontainers@1.3.1(supports-color@5.5.0): dependencies: byline: 5.0.0 debug: 4.3.6(supports-color@8.1.1) - dockerode: 2.5.8 + dockerode: 2.5.8(supports-color@5.5.0) get-port: 4.2.0 node-duration: 1.0.4 stream-to-array: 2.3.0 @@ -9027,8 +9116,6 @@ snapshots: undici-types@5.26.5: {} - undici-types@6.11.1: {} - universalify@0.1.2: {} universalify@2.0.1: {} @@ -9087,13 +9174,13 @@ snapshots: core-util-is: 1.0.2 extsprintf: 1.3.0 - vite-node@3.1.4(@types/node@22.0.0)(yaml@2.8.0): + vite-node@3.1.4(@types/node@20.14.13)(supports-color@8.1.1)(yaml@2.8.0): dependencies: cac: 6.7.14 - debug: 4.4.1 + debug: 4.4.1(supports-color@8.1.1) es-module-lexer: 1.7.0 pathe: 2.0.3 - vite: 6.3.5(@types/node@22.0.0)(yaml@2.8.0) + vite: 6.3.5(@types/node@20.14.13)(yaml@2.8.0) transitivePeerDependencies: - '@types/node' - jiti @@ -9108,18 +9195,18 @@ snapshots: - tsx - yaml - vite-tsconfig-paths@5.1.4(typescript@5.8.3)(vite@6.3.5(@types/node@22.0.0)(yaml@2.8.0)): + vite-tsconfig-paths@5.1.4(supports-color@8.1.1)(typescript@5.8.3)(vite@6.3.5(@types/node@20.14.13)(yaml@2.8.0)): dependencies: - debug: 4.4.1 + debug: 4.4.1(supports-color@8.1.1) globrex: 0.1.2 tsconfck: 3.1.6(typescript@5.8.3) optionalDependencies: - vite: 6.3.5(@types/node@22.0.0)(yaml@2.8.0) + vite: 6.3.5(@types/node@20.14.13)(yaml@2.8.0) transitivePeerDependencies: - supports-color - typescript - vite@6.3.5(@types/node@22.0.0)(yaml@2.8.0): + vite@6.3.5(@types/node@20.14.13)(yaml@2.8.0): dependencies: esbuild: 0.25.4 fdir: 6.4.4(picomatch@4.0.2) @@ -9128,21 +9215,21 @@ snapshots: rollup: 4.41.1 tinyglobby: 0.2.13 optionalDependencies: - '@types/node': 22.0.0 + '@types/node': 20.14.13 fsevents: 2.3.3 yaml: 2.8.0 - vitest@3.1.4(@types/node@22.0.0)(@vitest/browser@3.1.4)(yaml@2.8.0): + vitest@3.1.4(@types/node@20.14.13)(@vitest/browser@3.1.4)(supports-color@8.1.1)(yaml@2.8.0): dependencies: '@vitest/expect': 3.1.4 - '@vitest/mocker': 3.1.4(vite@6.3.5(@types/node@22.0.0)(yaml@2.8.0)) + '@vitest/mocker': 3.1.4(vite@6.3.5(@types/node@20.14.13)(yaml@2.8.0)) '@vitest/pretty-format': 3.1.4 '@vitest/runner': 3.1.4 '@vitest/snapshot': 3.1.4 '@vitest/spy': 3.1.4 '@vitest/utils': 3.1.4 chai: 5.2.0 - debug: 4.4.1 + debug: 4.4.1(supports-color@8.1.1) expect-type: 1.2.1 magic-string: 0.30.17 pathe: 2.0.3 @@ -9152,12 +9239,12 @@ snapshots: tinyglobby: 0.2.13 tinypool: 1.0.2 tinyrainbow: 2.0.0 - vite: 6.3.5(@types/node@22.0.0)(yaml@2.8.0) - vite-node: 3.1.4(@types/node@22.0.0)(yaml@2.8.0) + vite: 6.3.5(@types/node@20.14.13)(yaml@2.8.0) + vite-node: 3.1.4(@types/node@20.14.13)(supports-color@8.1.1)(yaml@2.8.0) why-is-node-running: 2.3.0 optionalDependencies: - '@types/node': 22.0.0 - '@vitest/browser': 3.1.4(playwright@1.52.0)(vite@6.3.5(@types/node@22.0.0)(yaml@2.8.0))(vitest@3.1.4) + '@types/node': 20.14.13 + '@vitest/browser': 3.1.4(playwright@1.52.0)(vite@6.3.5(@types/node@20.14.13)(yaml@2.8.0))(vitest@3.1.4) transitivePeerDependencies: - jiti - less