[Codex] Add SQLite dictionary storage with experimental legacy migration action

CONTRIBUTING.md

@@ -21,7 +21,7 @@ Below are a few guidelines to ensure contributions have a good level of quality
 
 ## Setup
 
-Yomitan uses [Node.js](https://nodejs.org/) and [npm](https://www.npmjs.com/) tools for building and testing.
+Yomitan uses [Node.js](https://nodejs.org/) and [npm](https://www.npmjs.com/) tools for building and testing. The build also compiles bundled dictionary wasm assets, so you need a wasm32-capable compiler such as `clang` or `zig` available on `PATH` (or pointed to by `YOMITAN_CLANG`/`CLANG`).
 After installing these, the development environment can be set up by running `npm ci` and subsequently `npm run build`.
 
 ## Testing

README.md

@@ -98,7 +98,7 @@ Feel free to join us on the [Yomitan Discord](https://discord.gg/YkQrXW6TXF).
 
 ## Building Yomitan
 
-1. Install [Node.js](https://nodejs.org/) and [npm](https://www.npmjs.com/).
+1. Install [Node.js](https://nodejs.org/) and [npm](https://www.npmjs.com/). You also need a wasm32-capable compiler such as `clang` or `zig` on `PATH` to build the bundled dictionary wasm assets. If needed, set `YOMITAN_CLANG` or `CLANG` to the compiler binary you want Yomitan to use.
 
 2. Run `npm ci` to set up the environment.
 

dev/build-libs.js

@@ -21,6 +21,8 @@ import standaloneCode from 'ajv/dist/standalone/index.js';
 import esbuild from 'esbuild';
 import fs from 'fs';
 import {createRequire} from 'module';
+import {execFileSync} from 'node:child_process';
+import os from 'os';
 import path from 'path';
 import {fileURLToPath} from 'url';
 import {parseJson} from './json.js';
@@ -29,6 +31,127 @@ const require = createRequire(import.meta.url);
 
 const dirname = path.dirname(fileURLToPath(import.meta.url));
 const extDir = path.join(dirname, '..', 'ext');
+const dictionaryWasmTarget = 'wasm32-freestanding';
+
+/**
+ * @typedef {{command: string, args?: string[]}} CompilerCommand
+ */
+
+/**
+ * @param {string|undefined} value
+ * @returns {value is string}
+ */
+function isNonEmptyString(value) {
+    return typeof value === 'string' && value.length > 0;
+}
+
+/**
+ * @param {string} command
+ * @returns {CompilerCommand}
+ */
+function createCompilerCommand(command) {
+    const name = path.basename(command).toLowerCase();
+    return (
+        name === 'zig' || name === 'zig.exe' ?
+            {command, args: ['cc']} :
+            {command}
+    );
+}
+
+/**
+ * @returns {CompilerCommand[]}
+ */
+function getWindowsWingetCompilerCommands() {
+    if (process.platform !== 'win32') { return []; }
+
+    const localAppData = process.env.LOCALAPPDATA;
+    if (!isNonEmptyString(localAppData)) { return []; }
+
+    const packagesDir = path.join(localAppData, 'Microsoft', 'WinGet', 'Packages');
+    if (!fs.existsSync(packagesDir)) { return []; }
+
+    /** @type {CompilerCommand[]} */
+    const commands = [];
+    for (const pkgEntry of fs.readdirSync(packagesDir, {withFileTypes: true})) {
+        if (!pkgEntry.isDirectory() || !pkgEntry.name.toLowerCase().startsWith('zig.zig')) { continue; }
+        const packageDir = path.join(packagesDir, pkgEntry.name);
+        for (const versionEntry of fs.readdirSync(packageDir, {withFileTypes: true})) {
+            if (!versionEntry.isDirectory()) { continue; }
+            const zigPath = path.join(packageDir, versionEntry.name, 'zig.exe');
+            if (fs.existsSync(zigPath)) {
+                commands.push(createCompilerCommand(zigPath));
+            }
+        }
+    }
+    return commands;
+}
+
+/**
+ * @param {CompilerCommand} compiler
+ * @returns {boolean}
+ */
+function canBuildWasmTarget(compiler) {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'yomitan-wasm-'));
+    const sourcePath = path.join(tempDir, 'probe.c');
+    const outputPath = path.join(tempDir, 'probe.wasm');
+    try {
+        fs.writeFileSync(sourcePath, 'void probe(void) {}\n', 'utf8');
+        execFileSync(
+            compiler.command,
+            [
+                ...(compiler.args ?? []),
+                `--target=${dictionaryWasmTarget}`,
+                '-nostdlib',
+                '-Wl,--no-entry',
+                '-Wl,--export=probe',
+                '-Wl,--strip-all',
+                '-o',
+                outputPath,
+                sourcePath,
+            ],
+            {stdio: 'ignore'},
+        );
+        return true;
+    } catch {
+        return false;
+    } finally {
+        fs.rmSync(tempDir, {recursive: true, force: true});
+    }
+}
+
+/**
+ * @returns {CompilerCommand}
+ * @throws {Error}
+ */
+function getWasmCapableCompiler() {
+    const candidates = /** @type {CompilerCommand[]} */ ([
+        process.env.YOMITAN_CLANG,
+        process.env.CLANG,
+        'clang',
+        'clang-18',
+        'clang-17',
+        'zig',
+        '/opt/homebrew/opt/llvm/bin/clang',
+        '/usr/bin/clang',
+        ...getWindowsWingetCompilerCommands().map(({command}) => command),
+    ]
+        .filter(isNonEmptyString)
+        .map((value) => createCompilerCommand(value)));
+    for (const candidate of candidates) {
+        try {
+            execFileSync(candidate.command, [...(candidate.args ?? []), '--version'], {stdio: 'ignore'});
+        } catch {
+            continue;
+        }
+        if (canBuildWasmTarget(candidate)) {
+            return candidate;
+        }
+    }
+    throw new Error(
+        'Missing a wasm32-capable compiler required to build dictionary wasm assets. ' +
+        `Set YOMITAN_CLANG or CLANG to a compiler that can link --target=${dictionaryWasmTarget}, such as clang or zig.`,
+    );
+}
 
 /**
  * @param {string} out
@@ -40,6 +163,73 @@ async function copyWasm(out) {
     fs.copyFileSync(wasmPath, path.join(out, 'resvg.wasm'));
 }
 
+/**
+ * @param {string} out
+ */
+async function copySqliteWasm(out) {
+    const sqliteWasmPath = path.dirname(require.resolve('@sqlite.org/sqlite-wasm/package.json'));
+    const sqliteDistPath = path.join(sqliteWasmPath, 'dist');
+    const sqliteOutPath = path.join(out, 'sqlite');
+    fs.mkdirSync(sqliteOutPath, {recursive: true});
+    for (const fileName of fs.readdirSync(sqliteDistPath)) {
+        const source = path.join(sqliteDistPath, fileName);
+        const destination = path.join(sqliteOutPath, fileName);
+        fs.copyFileSync(source, destination);
+    }
+}
+
+/**
+ * @param {string} out
+ */
+async function copyZstdAssets(out) {
+    const zstdEntryPath = require.resolve('@bokuweb/zstd-wasm');
+    const zstdPkgPath = path.resolve(path.dirname(zstdEntryPath), '..', '..');
+    const zstdWasmPath = path.join(zstdPkgPath, 'dist/esm/zstd.wasm');
+    fs.copyFileSync(zstdWasmPath, path.join(out, 'zstd.wasm'));
+
+    const zstdDictOutPath = path.join(out, 'zstd-dicts');
+    fs.mkdirSync(zstdDictOutPath, {recursive: true});
+    const jmdictDictPath = path.join(dirname, 'data', 'zstd-dicts', 'jmdict.zdict');
+    if (!fs.existsSync(jmdictDictPath)) {
+        throw new Error(`Missing vendored zstd dictionary asset: ${jmdictDictPath}`);
+    }
+    fs.copyFileSync(jmdictDictPath, path.join(zstdDictOutPath, 'jmdict.zdict'));
+}
+
+/**
+ * @param {string} out
+ */
+async function buildDictionaryWasm(out) {
+    const wasmSources = [
+        {
+            sourcePath: path.join(extDir, 'js', 'dictionary', 'wasm', 'term-bank-parser.c'),
+            outputPath: path.join(out, 'term-bank-parser.wasm'),
+            exports: ['wasm_reset_heap', 'wasm_alloc', 'parse_term_bank', 'encode_term_content'],
+        },
+        {
+            sourcePath: path.join(extDir, 'js', 'dictionary', 'wasm', 'term-record-encoder.c'),
+            outputPath: path.join(out, 'term-record-encoder.wasm'),
+            exports: ['wasm_reset_heap', 'wasm_alloc', 'calc_encoded_size', 'encode_records'],
+        },
+    ];
+
+    const compiler = getWasmCapableCompiler();
+
+    for (const target of wasmSources) {
+        const args = [
+            `--target=${dictionaryWasmTarget}`,
+            '-O3',
+            '-nostdlib',
+            '-Wl,--no-entry',
+        ];
+        for (const exportName of target.exports) {
+            args.push(`-Wl,--export=${exportName}`);
+        }
+        args.push('-Wl,--strip-all', '-o', target.outputPath, target.sourcePath);
+        execFileSync(compiler.command, [...(compiler.args ?? []), ...args], {stdio: 'inherit'});
+    }
+}
+
 
 /**
  * @param {string} scriptPath
@@ -95,4 +285,7 @@ export async function buildLibs() {
     fs.writeFileSync(path.join(extDir, 'lib/validate-schemas.js'), patchedModuleCode);
 
     await copyWasm(path.join(extDir, 'lib'));
+    await copySqliteWasm(path.join(extDir, 'lib'));
+    await copyZstdAssets(path.join(extDir, 'lib'));
+    await buildDictionaryWasm(path.join(extDir, 'lib'));
 }

dev/data/manifest-variants.json

@@ -106,7 +106,8 @@
                     "popup.html",
                     "template-renderer.html",
                     "js/*",
-                    "lib/resvg.wasm"
+                    "lib/resvg.wasm",
+                    "lib/sqlite/*"
                 ],
                 "matches": [
                     "<all_urls>"

dev/lib/zstd-wasm.js

@@ -0,0 +1,18 @@
+/*
+ * Copyright (C) 2023-2025  Yomitan Authors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+export * from '@bokuweb/zstd-wasm';

ext/js/background/backend.js

@@ -170,6 +170,8 @@ export class Backend {
             ['getZoom',                      this._onApiGetZoom.bind(this)],
             ['getDefaultAnkiFieldTemplates', this._onApiGetDefaultAnkiFieldTemplates.bind(this)],
             ['getDictionaryInfo',            this._onApiGetDictionaryInfo.bind(this)],
+            ['exportDictionaryDatabase',     this._onApiExportDictionaryDatabase.bind(this)],
+            ['importDictionaryDatabase',     this._onApiImportDictionaryDatabase.bind(this)],
             ['purgeDatabase',                this._onApiPurgeDatabase.bind(this)],
             ['getMedia',                     this._onApiGetMedia.bind(this)],
             ['logGenericErrorBackend',       this._onApiLogGenericErrorBackend.bind(this)],
@@ -927,6 +929,17 @@ export class Backend {
         return await this._dictionaryDatabase.getDictionaryInfo();
     }
 
+    /** @type {import('api').ApiHandler<'exportDictionaryDatabase'>} */
+    async _onApiExportDictionaryDatabase() {
+        return await this._dictionaryDatabase.exportDatabase();
+    }
+
+    /** @type {import('api').ApiHandler<'importDictionaryDatabase'>} */
+    async _onApiImportDictionaryDatabase({content}) {
+        await this._dictionaryDatabase.importDatabase(content);
+        this._triggerDatabaseUpdated('dictionary', 'import');
+    }
+
     /** @type {import('api').ApiHandler<'purgeDatabase'>} */
     async _onApiPurgeDatabase() {
         await this._dictionaryDatabase.purge();

ext/js/background/offscreen-proxy.js

@@ -18,7 +18,7 @@
 
 import {ExtensionError} from '../core/extension-error.js';
 import {isObjectNotArray} from '../core/object-utilities.js';
-import {base64ToArrayBuffer} from '../data/array-buffer-util.js';
+import {arrayBufferToBase64, base64ToArrayBuffer} from '../data/array-buffer-util.js';
 
 /**
  * This class is responsible for creating and communicating with an offscreen document.
@@ -190,6 +190,25 @@ export class DictionaryDatabaseProxy {
         return this._offscreen.sendMessagePromise({action: 'getDictionaryInfoOffscreen'});
     }
 
+    /**
+     * @returns {Promise<ArrayBuffer>}
+     */
+    async exportDatabase() {
+        const content = await this._offscreen.sendMessagePromise({action: 'exportDictionaryDatabaseOffscreen'});
+        return base64ToArrayBuffer(content);
+    }
+
+    /**
+     * @param {ArrayBuffer} content
+     * @returns {Promise<void>}
+     */
+    async importDatabase(content) {
+        await this._offscreen.sendMessagePromise({
+            action: 'importDictionaryDatabaseOffscreen',
+            params: {content: arrayBufferToBase64(content)},
+        });
+    }
+
     /**
      * @returns {Promise<boolean>}
      */

ext/js/background/offscreen.js

@@ -22,7 +22,7 @@ import {createApiMap, invokeApiMapHandler} from '../core/api-map.js';
 import {ExtensionError} from '../core/extension-error.js';
 import {log} from '../core/log.js';
 import {sanitizeCSS} from '../core/utilities.js';
-import {arrayBufferToBase64} from '../data/array-buffer-util.js';
+import {arrayBufferToBase64, base64ToArrayBuffer} from '../data/array-buffer-util.js';
 import {DictionaryDatabase} from '../dictionary/dictionary-database.js';
 import {WebExtension} from '../extension/web-extension.js';
 import {Translator} from '../language/translator.js';
@@ -55,6 +55,8 @@ export class Offscreen {
             ['clipboardSetBrowserOffscreen',   this._setClipboardBrowser.bind(this)],
             ['databasePrepareOffscreen',       this._prepareDatabaseHandler.bind(this)],
             ['getDictionaryInfoOffscreen',     this._getDictionaryInfoHandler.bind(this)],
+            ['exportDictionaryDatabaseOffscreen', this._exportDictionaryDatabaseHandler.bind(this)],
+            ['importDictionaryDatabaseOffscreen', this._importDictionaryDatabaseHandler.bind(this)],
             ['databasePurgeOffscreen',         this._purgeDatabaseHandler.bind(this)],
             ['databaseGetMediaOffscreen',      this._getMediaHandler.bind(this)],
             ['translatorPrepareOffscreen',     this._prepareTranslatorHandler.bind(this)],
@@ -117,6 +119,16 @@ export class Offscreen {
         return await this._dictionaryDatabase.getDictionaryInfo();
     }
 
+    /** @type {import('offscreen').ApiHandler<'exportDictionaryDatabaseOffscreen'>} */
+    async _exportDictionaryDatabaseHandler() {
+        return arrayBufferToBase64(await this._dictionaryDatabase.exportDatabase());
+    }
+
+    /** @type {import('offscreen').ApiHandler<'importDictionaryDatabaseOffscreen'>} */
+    async _importDictionaryDatabaseHandler({content}) {
+        await this._dictionaryDatabase.importDatabase(base64ToArrayBuffer(content));
+    }
+
     /** @type {import('offscreen').ApiHandler<'databasePurgeOffscreen'>} */
     async _purgeDatabaseHandler() {
         return await this._dictionaryDatabase.purge();

ext/js/comm/api.js

@@ -248,6 +248,21 @@ export class API {
         return this._invoke('getDictionaryInfo', void 0);
     }
 
+    /**
+     * @returns {Promise<import('api').ApiReturn<'exportDictionaryDatabase'>>}
+     */
+    exportDictionaryDatabase() {
+        return this._invoke('exportDictionaryDatabase', void 0);
+    }
+
+    /**
+     * @param {import('api').ApiParam<'importDictionaryDatabase', 'content'>} content
+     * @returns {Promise<import('api').ApiReturn<'importDictionaryDatabase'>>}
+     */
+    importDictionaryDatabase(content) {
+        return this._invoke('importDictionaryDatabase', {content});
+    }
+
     /**
      * @returns {Promise<import('api').ApiReturn<'purgeDatabase'>>}
      */