#452 Data Generator private package to generate test data from Dictionaries and Schemas

packages/data-generator/.mocharc.json

@@ -0,0 +1,5 @@
+{
+	"extension": ["ts"],
+	"require": "ts-node/register",
+	"spec": "test/**/*.spec.ts"
+}

packages/data-generator/docs/resolving-restrictions.md

@@ -0,0 +1,62 @@
+# Resolving Restrictions for Field Generators
+
+Field generators attempt to generate a value that will be valid given the restrictions defined for the field in the schema. To accomplish this, the generators act in two phases: first collecting all active restrictions from the field definition, then merging each restriction type down to a set of effective constraints used for generation.
+
+## Phase 1 - Collecting active restrictions
+
+A field's restrictions may be a single restriction object, an array of restriction objects, or a mix of plain and conditional objects.
+
+Conditional restrictions (`if/then/else` blocks) are evaluated against the `record` argument passed to the generator. When a condition passes, the `then` branch is used; otherwise the `else` branch is used. Fields absent from `record` are treated as `undefined`, which typically causes an `exists: true` condition to fail and an `exists: false` condition to pass.
+
+All active (non-conditional) restriction values are collected by type before generation begins.
+
+## Phase 2 - Merging restrictions
+
+Each restriction type is merged independently.
+
+### `codeList`
+
+Multiple code lists are intersected. The generated value is drawn from the set of elements present in every active code list.
+
+- **No conflict:** the intersection is non-empty.
+- **Conflict:** the intersection is empty. The generator returns a failure result with a best-effort value drawn from the first code list.
+
+### `range` (integer and number fields)
+
+Multiple ranges are intersected to produce the tightest overlapping subrange. The most restrictive bound from each side (lower and upper) is kept.
+
+- **No conflict:** a valid subrange exists.
+- **Conflict:** the merged lower bound exceeds the upper bound, or both bounds are equal but at least one is exclusive. The generator returns a failure result with a best-effort value drawn from the first range.
+
+### `regex` (string fields)
+
+Multiple regex patterns are combined using lookahead conjunction so that the generated string must match every pattern. This is always syntactically valid, so no conflict is reported at merge time. Patterns that are semantically incompatible (i.e. can never simultaneously match) are not detected; generation will produce a value that matches the combined expression as best it can.
+
+### `codeList` + `range` together (integer and number fields)
+
+When both are present, the code list is filtered to values that fall within the merged range.
+
+- **No conflict:** at least one code list value satisfies the range.
+- **Conflict:** no code list value satisfies the range. The generator returns a failure result with a best-effort value drawn from the unfiltered code list.
+
+### `codeList` + `regex` together (string fields)
+
+When both are present, the code list is filtered to values that match the merged regex pattern.
+
+- **No conflict:** at least one code list value matches the pattern.
+- **Conflict:** no code list value matches the pattern. The generator returns a failure result with a best-effort value drawn from the unfiltered code list.
+
+### `required` and `empty`
+
+These restrictions do not constrain the generated value itself. However, `required: true` combined with `empty: true` across the active restrictions is a conflict - it is impossible for a field to be both required and empty. The generator returns a failure result but still produces a value.
+
+## Failure results
+
+All conflicts produce a failure result rather than throwing an error. The failure includes:
+
+- A best-effort fallback value that may not satisfy all restrictions.
+- A list of conflicts, one per irreconcilable pair of restrictions, each describing the restriction type and the reason the conflict occurred.
+
+When multiple conflicts occur on the same field (e.g. two incompatible code lists and an incompatible range), all conflicts are collected and returned together in a single failure result.
+
+Check `result.success` to distinguish success from failure before using the result value.

packages/data-generator/package.json

@@ -0,0 +1,23 @@
+{
+	"name": "@overture-stack/lectern-data-generator",
+	"version": "0.0.0",
+	"description": "Test data generation utilities for Lectern schemas",
+	"private": true,
+	"main": "dist/index.js",
+	"scripts": {
+		"build": "pnpm nuke:build && tsc -p ./tsconfig.build.json",
+		"format": "prettier --write .",
+		"nuke:build": "npx rimraf dist",
+		"test": "nyc mocha"
+	},
+	"keywords": [],
+	"author": "Ontario Institute for Cancer Research",
+	"license": "AGPL-3.0",
+	"dependencies": {
+		"@overture-stack/lectern-dictionary": "workspace:^",
+		"@overture-stack/lectern-validation": "workspace:^"
+	},
+	"devDependencies": {
+		"fast-check": "^4.9.0"
+	}
+}

packages/data-generator/src/common/fileTypes.ts

@@ -0,0 +1,14 @@
+/** Column delimiter format for data files. `'tsv'` uses tab; `'csv'` uses comma. */
+export type DataFileFormat = 'tsv' | 'csv';
+
+/** Maps each `DataFileFormat` to its column delimiter character. */
+export const COLUMN_DELIMITER = {
+	tsv: '\t',
+	csv: ',',
+};
+
+/** Maps each `DataFileFormat` to its file extension, including the leading dot. */
+export const FILE_EXTENSION = {
+	tsv: '.tsv',
+	csv: '.csv',
+} as const satisfies Record<DataFileFormat, string>;

packages/data-generator/src/common/hash.ts

@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2026 The Ontario Institute for Cancer Research. All rights reserved
+ *
+ * This program and the accompanying materials are made available under the terms of
+ * the GNU Affero General Public License v3.0. You should have received a copy of the
+ * GNU Affero General Public License along with this program.
+ *  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
+ * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+ * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+// Knuth multiplicative hash constant (2^32 / golden ratio, nearest odd integer).
+const KNUTH_MULTIPLIER = 2654435761;
+
+/**
+ * Maps `seed` to a well-distributed 32-bit unsigned integer using a Knuth multiplicative hash
+ * followed by one round of xorshift32. Produces independent draws without fast-check overhead.
+ *
+ * Used as the shared primitive for `shouldGenerateEmpty`, `seededIndexInRange`, and
+ * `deriveRetrySeed` — all of which need a single cheap, seeded, non-colliding hash draw.
+ */
+export const knuthHash = (seed: number): number => {
+	let hash = (seed * KNUTH_MULTIPLIER + 1) >>> 0;
+	hash ^= hash << 13;
+	hash ^= hash >>> 17;
+	hash ^= hash << 5;
+	return hash >>> 0;
+};

packages/data-generator/src/dataFile/dataFileGenerator.ts

@@ -0,0 +1,157 @@
+/*
+ * Copyright (c) 2026 The Ontario Institute for Cancer Research. All rights reserved
+ *
+ * This program and the accompanying materials are made available under the terms of
+ * the GNU Affero General Public License v3.0. You should have received a copy of the
+ * GNU Affero General Public License along with this program.
+ *  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
+ * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+ * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+import type { Dictionary, Result, Schema } from '@overture-stack/lectern-dictionary';
+import { failWith, success } from '@overture-stack/lectern-dictionary';
+import {
+	type DictionaryGeneratorOptions,
+	generateDictionaryRecords,
+} from '../dataGeneration/dictionary/dictionaryGenerator';
+import { type SchemaGeneratorOptions, generateSchemaRecords } from '../dataGeneration/records/schemaGenerator';
+import { closeDataFile, openDataFile, writeRecord } from './dataFileWriter';
+import { FILE_EXTENSION, type DataFileFormat } from '../common/fileTypes';
+
+/** Failure reasons returned by `generateSchemaFile` and `generateDictionaryFiles`. */
+export type GenerateFileError =
+	| { error: 'DIRECTORY_NOT_FOUND'; directory: string }
+	| { error: 'FILE_ALREADY_EXISTS'; filePath: string };
+
+const resolveOutputPath = (outputDir: string, schemaName: string, format: DataFileFormat): string =>
+	path.join(outputDir, schemaName + FILE_EXTENSION[format]);
+
+const DIRECTORY_NOT_FOUND = 'DIRECTORY_NOT_FOUND' as const;
+const FILE_ALREADY_EXISTS = 'FILE_ALREADY_EXISTS' as const;
+
+const checkDirectory = (outputDir: string): Result<void, GenerateFileError> => {
+	if (!fs.existsSync(outputDir) || !fs.statSync(outputDir).isDirectory()) {
+		return failWith(`Output directory does not exist: ${outputDir}`, {
+			error: DIRECTORY_NOT_FOUND,
+			directory: outputDir,
+		});
+	}
+	return success(undefined);
+};
+
+const checkFileAbsent = (filePath: string): Result<void, GenerateFileError> => {
+	if (fs.existsSync(filePath)) {
+		return failWith(`File already exists: ${filePath}`, { error: FILE_ALREADY_EXISTS, filePath });
+	}
+	return success(undefined);
+};
+
+/**
+ * Generates records for `schema` and writes them to a new file in `outputDir`.
+ *
+ * The output file is named `<schema.name>.<format>`. Fails without writing if the directory
+ * does not exist or if the file already exists.
+ */
+export const generateSchemaFile = async (
+	schema: Schema,
+	outputDir: string,
+	format: DataFileFormat,
+	options?: Omit<SchemaGeneratorOptions, 'count'> & { count: number },
+): Promise<Result<void, GenerateFileError>> => {
+	const directoryCheck = checkDirectory(outputDir);
+	if (!directoryCheck.success) {
+		return directoryCheck;
+	}
+
+	const filePath = resolveOutputPath(outputDir, schema.name, format);
+	const fileCheck = checkFileAbsent(filePath);
+	if (!fileCheck.success) {
+		return fileCheck;
+	}
+
+	const handle = await openDataFile(schema, filePath, format);
+	try {
+		for (const record of generateSchemaRecords(schema, options)) {
+			const writeResult = await writeRecord(handle, record);
+			if (!writeResult.success) {
+				throw new Error(`Failed to write record: ${writeResult.data.error}`);
+			}
+		}
+	} finally {
+		await closeDataFile(handle);
+	}
+
+	return success(undefined);
+};
+
+/**
+ * Generates records for all schemas in `dictionary` with a non-zero count and writes each to a
+ * separate file in `outputDir`, named `<schema.name>.<format>`.
+ *
+ * All output file paths are checked before any writing begins. Fails without writing any files
+ * if the directory does not exist or if any expected output file already exists.
+ */
+export const generateDictionaryFiles = async (
+	dictionary: Dictionary,
+	outputDir: string,
+	format: DataFileFormat,
+	options: DictionaryGeneratorOptions,
+): Promise<Result<void, GenerateFileError>> => {
+	const directoryCheck = checkDirectory(outputDir);
+	if (!directoryCheck.success) {
+		return directoryCheck;
+	}
+
+	const includedSchemaNames = Object.entries(options.counts)
+		.filter(([, count]) => count > 0)
+		.map(([name]) => name);
+
+	for (const schemaName of includedSchemaNames) {
+		const filePath = resolveOutputPath(outputDir, schemaName, format);
+		const fileCheck = checkFileAbsent(filePath);
+		if (!fileCheck.success) {
+			return fileCheck;
+		}
+	}
+
+	const schemaByName = new Map(dictionary.schemas.map((schema) => [schema.name, schema]));
+	const handles = new Map<string, Awaited<ReturnType<typeof openDataFile>>>();
+
+	for (const schemaName of includedSchemaNames) {
+		const schema = schemaByName.get(schemaName);
+		if (schema === undefined) {
+			continue;
+		}
+		const filePath = resolveOutputPath(outputDir, schemaName, format);
+		handles.set(schemaName, await openDataFile(schema, filePath, format));
+	}
+
+	try {
+		for (const { schemaName, record } of generateDictionaryRecords(dictionary, options)) {
+			const handle = handles.get(schemaName);
+			if (handle !== undefined) {
+				const writeResult = await writeRecord(handle, record);
+				if (!writeResult.success) {
+					throw new Error(`Failed to write record for schema '${schemaName}': ${writeResult.data.error}`);
+				}
+			}
+		}
+	} finally {
+		for (const handle of handles.values()) {
+			await closeDataFile(handle);
+		}
+	}
+
+	return success(undefined);
+};