diff --git a/packages/kit/package.json b/packages/kit/package.json index 313eb3c88..2f54e6b24 100644 --- a/packages/kit/package.json +++ b/packages/kit/package.json @@ -67,6 +67,7 @@ "license": "MIT", "devDependencies": { "@types/node": "^22.13.17", + "fake-indexeddb": "^6.2.5", "openai": "^6.34.0", "tsup": "^8.0.1", "typescript": "^5.8.2", diff --git a/packages/kit/src/index.ts b/packages/kit/src/index.ts index 1456bf582..644772a61 100644 --- a/packages/kit/src/index.ts +++ b/packages/kit/src/index.ts @@ -1,6 +1,7 @@ export { AIClient } from './client' export { BaseModelProvider } from './providers/base' export { OpenAIProvider } from './providers/openai' +export * from './skills/storage' export * from './storage' export * from './types' export { extractTextFromResponse, formatMessages, handleSSEStream, sseStreamToGenerator } from './utils' diff --git a/packages/kit/src/node.ts b/packages/kit/src/node.ts index d72db2267..05eec1da1 100644 --- a/packages/kit/src/node.ts +++ b/packages/kit/src/node.ts @@ -1,4 +1,5 @@ export { loadSkill, loadSkillWithDetails } from './skills/loader/node' +export * from './skills/storage/node' export type { FsSkillLoadOptions, GithubSkillLoadOptions, diff --git a/packages/kit/src/skills/storage/fs.ts b/packages/kit/src/skills/storage/fs.ts new file mode 100644 index 000000000..445d6c6dc --- /dev/null +++ b/packages/kit/src/skills/storage/fs.ts @@ -0,0 +1,328 @@ +import { mkdir, readFile, readdir, rm, stat, writeFile } from 'node:fs/promises' +import { dirname, join, relative } from 'node:path' +import { stringify as stringifyYaml } from 'yaml' +import { loadSkillWithDetails } from '../loader/node' +import type { SkillLoadOptions } from '../loader/node' +import { + getRecord, + getString, + isTextSkillFilePath, + normalizeSkillPath, + parseMarkdownFrontmatter, +} from '../loader/utils' +import type { SkillDefinition, SkillResourceDescriptor } from '../types' +import { createImportSkill } from './importSkill' +import type { SkillStorage } from './types' + +/** 一个标准 skill 目录集合的文件系统 storage。 */ +export interface FsSkillStorageOptions { + root: string + /** 只读 storage 不允许 add/import 写入,也不允许 delete。 */ + readonly?: boolean +} + +const entryFile = 'SKILL.md' +const importSkill = createImportSkill(loadSkillWithDetails) + +export class FsSkillStorage implements SkillStorage { + readonly root: string + readonly readonly: boolean + + constructor(options: FsSkillStorageOptions) { + this.root = options.root + this.readonly = options.readonly ?? false + } + + async add(skill: SkillDefinition) { + this.assertWritable() + + const directory = this.getSkillDirectory(skill.name) + await rm(directory, { + recursive: true, + force: true, + }) + await mkdir(directory, { + recursive: true, + }) + await writeFile(join(directory, entryFile), serializeSkillEntry(skill), 'utf8') + + for (const resource of skill.resources ?? []) { + await this.writeResource(directory, resource) + } + + const storedSkill = await this.get(skill.name) + if (!storedSkill) { + throw new Error(`Failed to store skill "${skill.name}".`) + } + + return storedSkill + } + + async get(name: string) { + const directory = this.getSkillDirectory(name) + + try { + return await this.readSkillDirectory(directory) + } catch (error) { + if (isFileNotFoundError(error)) { + return undefined + } + + throw error + } + } + + async has(name: string) { + return Boolean(await this.get(name)) + } + + async delete(name: string) { + this.assertWritable() + + const directory = this.getSkillDirectory(name) + const exists = await this.has(name) + + if (!exists) { + return false + } + + await rm(directory, { + recursive: true, + force: true, + }) + return true + } + + async list() { + const entries = await readdir(this.root, { + withFileTypes: true, + }).catch((error: unknown) => { + if (isFileNotFoundError(error)) { + return [] + } + + throw error + }) + const summaries = await Promise.all( + entries + .filter((entry) => entry.isDirectory() && !entry.name.startsWith('.')) + .map(async (entry) => this.get(entry.name)), + ) + + return summaries + .filter((skill): skill is SkillDefinition => Boolean(skill)) + .map((skill) => ({ + name: skill.name, + description: skill.description, + resourceCount: skill.resources?.length ?? 0, + metadata: skill.metadata, + })) + .sort((a, b) => a.name.localeCompare(b.name)) + } + + import(options: SkillLoadOptions) { + this.assertWritable() + const task = importSkill(options) + + return Object.assign( + task.then(async (result) => { + const skill = await this.add(result.skill) + return { + ...result, + name: skill.name, + skill, + } + }), + { cancel: task.cancel }, + ) + } + + private async readSkillDirectory(directory: string): Promise { + const entryPath = join(directory, entryFile) + const entryContent = await readFile(entryPath, 'utf8') + const { frontmatter, body } = parseMarkdownFrontmatter(entryContent) + const instructions = body.trim() + + if (!instructions) { + throw new Error(`Skill entry file "${entryFile}" must contain instructions.`) + } + + const resources = await this.readResourceDescriptors(directory) + + return { + name: getString(frontmatter.name) || directory.split(/[\\/]/).at(-1) || '', + description: getString(frontmatter.description) || '', + instructions, + resources: resources.length ? resources : undefined, + metadata: { + ...getRecord(frontmatter.metadata), + ...(getString(frontmatter.homepage) ? { homepage: getString(frontmatter.homepage) } : {}), + }, + } + } + + private async readResourceDescriptors(directory: string) { + const resources: SkillResourceDescriptor[] = [] + + const walk = async (currentDirectory: string) => { + const entries = await readdir(currentDirectory, { + withFileTypes: true, + }) + + for (const entry of entries) { + if (entry.name.startsWith('.')) { + continue + } + + const fullPath = join(currentDirectory, entry.name) + + if (entry.isDirectory()) { + await walk(fullPath) + continue + } + + if (!entry.isFile()) { + continue + } + + const path = normalizeSkillPath(relative(directory, fullPath)) + if (!path || path === entryFile) { + continue + } + + const fileStat = await stat(fullPath) + const kind = isTextSkillFilePath(path) ? 'text' : 'binary' + const base = { + path, + kind, + resourceId: path, + size: fileStat.size, + lastModified: fileStat.mtimeMs, + } + + resources.push( + kind === 'text' + ? { + ...base, + kind, + readText: async () => readFile(fullPath, 'utf8'), + readBinary: async () => new Uint8Array(await readFile(fullPath)), + } + : { + ...base, + kind, + readBinary: async () => new Uint8Array(await readFile(fullPath)), + readText: async () => new TextDecoder().decode(await readFile(fullPath)), + }, + ) + } + } + + await walk(directory) + return resources.sort((a, b) => a.path.localeCompare(b.path)) + } + + private async writeResource(directory: string, resource: SkillResourceDescriptor) { + const path = normalizeSkillPath(resource.path) + + if (!path || path === entryFile) { + return + } + + const fullPath = join(directory, path) + await mkdir(dirname(fullPath), { + recursive: true, + }) + + if (resource.kind === 'text') { + await writeFile(fullPath, await getResourceText(resource), 'utf8') + return + } + + await writeFile(fullPath, await getResourceBinary(resource)) + } + + private getSkillDirectory(name: string) { + const directoryName = normalizeSkillPath(name) + + if (!directoryName || directoryName.includes('/')) { + throw new Error(`Invalid skill name for file storage: ${name}`) + } + + return join(this.root, directoryName) + } + + private assertWritable() { + if (this.readonly) { + throw new Error('File system skill storage is readonly.') + } + } +} + +export function createFsSkillStorage(options: FsSkillStorageOptions) { + return new FsSkillStorage(options) +} + +function serializeSkillEntry(skill: SkillDefinition) { + const metadata = { ...skill.metadata } + const homepage = typeof metadata.homepage === 'string' ? metadata.homepage : undefined + delete metadata.homepage + const frontmatter: Record = { + name: skill.name, + description: skill.description, + } + + if (homepage) { + frontmatter.homepage = homepage + } + + if (Object.keys(metadata).length > 0) { + frontmatter.metadata = metadata + } + + return `---\n${stringifyYaml(frontmatter).trimEnd()}\n---\n\n${skill.instructions.trim()}\n` +} + +async function getResourceText(resource: SkillResourceDescriptor) { + if (resource.text !== undefined) { + return resource.text + } + + if (resource.readText) { + return resource.readText() + } + + if (resource.binary) { + return new TextDecoder().decode(resource.binary) + } + + if (resource.readBinary) { + return new TextDecoder().decode(await resource.readBinary()) + } + + throw new Error(`Skill resource "${resource.path}" has no text content.`) +} + +async function getResourceBinary(resource: SkillResourceDescriptor) { + if (resource.binary) { + return resource.binary + } + + if (resource.readBinary) { + return resource.readBinary() + } + + if (resource.text !== undefined) { + return new TextEncoder().encode(resource.text) + } + + if (resource.readText) { + return new TextEncoder().encode(await resource.readText()) + } + + throw new Error(`Skill resource "${resource.path}" has no binary content.`) +} + +function isFileNotFoundError(error: unknown) { + return Boolean(error && typeof error === 'object' && 'code' in error && error.code === 'ENOENT') +} diff --git a/packages/kit/src/skills/storage/importSkill.ts b/packages/kit/src/skills/storage/importSkill.ts new file mode 100644 index 000000000..88a786add --- /dev/null +++ b/packages/kit/src/skills/storage/importSkill.ts @@ -0,0 +1,26 @@ +import type { SkillLoadJob, SkillLoadResult } from '../loader/type' +import type { SkillImporter, SkillImportJob, SkillImportResult } from './types' + +export function createImportSkill( + loadSkill: (options: TImportOptions) => SkillLoadJob, +): SkillImporter { + return (options) => { + const loadJob = loadSkill(options) + + const task = (async (): Promise => { + const { skill, warnings } = await loadJob + + return { + name: skill.name, + skill, + warnings, + } + })() as SkillImportJob + + task.cancel = () => { + loadJob.cancel() + } + + return task + } +} diff --git a/packages/kit/src/skills/storage/index.ts b/packages/kit/src/skills/storage/index.ts new file mode 100644 index 000000000..fb9bcf887 --- /dev/null +++ b/packages/kit/src/skills/storage/index.ts @@ -0,0 +1,18 @@ +import { loadSkillWithDetails } from '../loader' +import type { SkillLoadOptions } from '../loader' +import type { SkillStorage as SkillStorageBase } from './types' +import { createImportSkill } from './importSkill' +import { createMemorySkillStorage as createMemorySkillStorageBase } from './memory' + +export type { SkillImportJob, SkillImportResult } from './types' +export type SkillImportOptions = SkillLoadOptions +export type SkillStorage = SkillStorageBase +export { createIndexedDBSkillStorage, IndexedDBSkillStorage } from './indexedDB' +export type { IndexedDBSkillStorageOptions } from './indexedDB' +export { MemorySkillStorage } from './memory' + +export const importSkill = createImportSkill(loadSkillWithDetails) + +export function createMemorySkillStorage() { + return createMemorySkillStorageBase(importSkill) +} diff --git a/packages/kit/src/skills/storage/indexedDB.ts b/packages/kit/src/skills/storage/indexedDB.ts new file mode 100644 index 000000000..5a63ac107 --- /dev/null +++ b/packages/kit/src/skills/storage/indexedDB.ts @@ -0,0 +1,344 @@ +import { openDB, type DBSchema, type IDBPDatabase, type IDBPTransaction } from 'idb' +import { loadSkillWithDetails } from '../loader' +import type { SkillLoadOptions } from '../loader' +import type { SkillDefinition, SkillResourceDescriptor } from '../types' +import { createImportSkill } from './importSkill' +import type { SkillImportJob, SkillImporter, SkillStorage, SkillSummary } from './types' + +const defaultVersion = 1 +const defaultSkillStoreName = 'skills' +const defaultResourceStoreName = 'resources' + +export interface IndexedDBSkillStorageOptions { + /** + * IndexedDB database name. Tests should pass a unique name to avoid cross-test state. + */ + databaseName: string +} + +interface IndexedDBSkillStorageSkillRecord { + name: string + description: string + instructions: string + metadata?: Record + resources?: IndexedDBSkillStorageResourceMetadata[] +} + +interface IndexedDBSkillStorageResourceMetadata { + path: string + kind: SkillResourceDescriptor['kind'] + resourceId: string + mimeType?: string + size?: number + lastModified?: number + metadata?: Record +} + +interface IndexedDBSkillStorageResourceRecord { + skillName: string + resourceId: string + kind: SkillResourceDescriptor['kind'] + text?: string + binary?: Uint8Array +} + +interface IndexedDBSkillStorageSchema extends DBSchema { + skills: { + key: string + value: IndexedDBSkillStorageSkillRecord + } + resources: { + key: [string, string] + value: IndexedDBSkillStorageResourceRecord + indexes: { + skillName: string + } + } +} + +type IndexedDBSkillStorageTransaction = IDBPTransaction< + IndexedDBSkillStorageSchema, + ['skills', 'resources'], + 'readwrite' +> + +type SkillImportOptions = SkillLoadOptions +const importSkill = createImportSkill(loadSkillWithDetails) + +export class IndexedDBSkillStorage implements SkillStorage { + readonly databaseName: string + readonly skillStoreName = defaultSkillStoreName + readonly resourceStoreName = defaultResourceStoreName + private dbPromise?: Promise> + + constructor( + options: IndexedDBSkillStorageOptions, + private readonly importer: SkillImporter = importSkill as SkillImporter, + ) { + this.databaseName = options.databaseName + } + + private getDB() { + this.dbPromise ??= openDB(this.databaseName, defaultVersion, { + upgrade: (db) => { + if (!db.objectStoreNames.contains(this.skillStoreName)) { + db.createObjectStore(defaultSkillStoreName, { + keyPath: 'name', + }) + } + + if (!db.objectStoreNames.contains(this.resourceStoreName)) { + const resourceStore = db.createObjectStore(defaultResourceStoreName, { + keyPath: ['skillName', 'resourceId'], + }) + resourceStore.createIndex('skillName', 'skillName') + } + }, + }) + + return this.dbPromise + } + + async add(skill: SkillDefinition) { + const resourceRecords = await Promise.all( + (skill.resources ?? []).map((resource) => toResourceRecord(skill.name, resource)), + ) + const db = await this.getDB() + const tx = db.transaction([defaultSkillStoreName, defaultResourceStoreName], 'readwrite') + const skillStore = tx.objectStore(defaultSkillStoreName) + const resourceStore = tx.objectStore(defaultResourceStoreName) + + await this.deleteResourceRecords(tx, skill.name) + await skillStore.put(toSkillRecord(skill)) + + for (const resourceRecord of resourceRecords) { + await resourceStore.put(resourceRecord) + } + + await tx.done + return this.getStoredSkill(skill.name) + } + + async get(name: string) { + const db = await this.getDB() + const record = await db.get(defaultSkillStoreName, name) + + return record ? this.toSkillDefinition(record) : undefined + } + + async has(name: string) { + const db = await this.getDB() + return (await db.count(defaultSkillStoreName, name)) > 0 + } + + async delete(name: string) { + const db = await this.getDB() + const tx = db.transaction([defaultSkillStoreName, defaultResourceStoreName], 'readwrite') + const skillStore = tx.objectStore(defaultSkillStoreName) + const existed = (await skillStore.count(name)) > 0 + + await skillStore.delete(name) + await this.deleteResourceRecords(tx, name) + await tx.done + + return existed + } + + async list(): Promise { + const db = await this.getDB() + const records = await db.getAll(defaultSkillStoreName) + + return records + .map((record) => ({ + name: record.name, + description: record.description, + resourceCount: record.resources?.length ?? 0, + metadata: record.metadata, + })) + .sort((a, b) => a.name.localeCompare(b.name)) + } + + import(options: TImportOptions): SkillImportJob { + const task = this.importer(options) + + return Object.assign( + task.then(async (result) => { + await this.add(result.skill) + return result + }), + { cancel: task.cancel }, + ) + } + + private async getStoredSkill(name: string) { + const skill = await this.get(name) + + if (!skill) { + throw new Error(`Failed to store skill "${name}".`) + } + + return skill + } + + private toSkillDefinition(record: IndexedDBSkillStorageSkillRecord): SkillDefinition { + return { + name: record.name, + description: record.description, + instructions: record.instructions, + metadata: record.metadata ? { ...record.metadata } : undefined, + resources: record.resources?.map((resource) => this.toSkillResource(record.name, resource)), + } + } + + private toSkillResource(skillName: string, resource: IndexedDBSkillStorageResourceMetadata): SkillResourceDescriptor { + const base = { + path: resource.path, + resourceId: resource.resourceId, + mimeType: resource.mimeType, + size: resource.size, + lastModified: resource.lastModified, + metadata: resource.metadata ? { ...resource.metadata } : undefined, + } + + if (resource.kind === 'text') { + return { + ...base, + kind: resource.kind, + readText: async () => this.readResourceText(skillName, resource.resourceId), + readBinary: async () => this.readResourceBinary(skillName, resource.resourceId), + } + } + + return { + ...base, + kind: resource.kind, + readBinary: async () => this.readResourceBinary(skillName, resource.resourceId), + readText: async () => this.readResourceText(skillName, resource.resourceId), + } + } + + private async readResourceText(skillName: string, resourceId: string) { + const resource = await this.getResourceRecord(skillName, resourceId) + + if (typeof resource.text === 'string') { + return resource.text + } + + if (resource.binary) { + return new TextDecoder().decode(resource.binary) + } + + throw new Error(`Skill resource "${resourceId}" has no text content.`) + } + + private async readResourceBinary(skillName: string, resourceId: string) { + const resource = await this.getResourceRecord(skillName, resourceId) + + if (resource.binary) { + return new Uint8Array(resource.binary) + } + + if (typeof resource.text === 'string') { + return new TextEncoder().encode(resource.text) + } + + throw new Error(`Skill resource "${resourceId}" has no binary content.`) + } + + private async getResourceRecord(skillName: string, resourceId: string) { + const db = await this.getDB() + const resource = await db.get(defaultResourceStoreName, [skillName, resourceId]) + + if (!resource) { + throw new Error(`Skill resource "${resourceId}" was not found.`) + } + + return resource + } + + private async deleteResourceRecords(tx: IndexedDBSkillStorageTransaction, skillName: string) { + const resourceStore = tx.objectStore(defaultResourceStoreName) + const resourceKeys = await resourceStore.index('skillName').getAllKeys(skillName) + + await Promise.all(resourceKeys.map((key) => resourceStore.delete(key))) + } +} + +export function createIndexedDBSkillStorage(options: IndexedDBSkillStorageOptions) { + return new IndexedDBSkillStorage(options) +} + +function toSkillRecord(skill: SkillDefinition): IndexedDBSkillStorageSkillRecord { + return { + name: skill.name, + description: skill.description, + instructions: skill.instructions, + metadata: skill.metadata ? { ...skill.metadata } : undefined, + resources: skill.resources?.map((resource) => ({ + path: resource.path, + kind: resource.kind, + resourceId: resource.resourceId, + mimeType: resource.mimeType, + size: resource.size, + lastModified: resource.lastModified, + metadata: resource.metadata ? { ...resource.metadata } : undefined, + })), + } +} + +async function toResourceRecord( + skillName: string, + resource: SkillResourceDescriptor, +): Promise { + if (resource.kind === 'text') { + const text = resource.text ?? (await readTextContent(resource)) + + if (typeof text !== 'string') { + throw new Error(`Skill resource "${resource.resourceId}" has no text content to store.`) + } + + return { + skillName, + resourceId: resource.resourceId, + kind: resource.kind, + text, + } + } + + const binary = resource.binary ?? (await readBinaryContent(resource)) + + if (!binary) { + throw new Error(`Skill resource "${resource.resourceId}" has no binary content to store.`) + } + + return { + skillName, + resourceId: resource.resourceId, + kind: resource.kind, + binary: new Uint8Array(binary), + } +} + +async function readTextContent(resource: SkillResourceDescriptor) { + if (resource.readText) { + return resource.readText() + } + + if (resource.binary) { + return new TextDecoder().decode(resource.binary) + } + + return undefined +} + +async function readBinaryContent(resource: SkillResourceDescriptor) { + if (resource.readBinary) { + return resource.readBinary() + } + + if (resource.text) { + return new TextEncoder().encode(resource.text) + } + + return undefined +} diff --git a/packages/kit/src/skills/storage/memory.ts b/packages/kit/src/skills/storage/memory.ts new file mode 100644 index 000000000..5ae401226 --- /dev/null +++ b/packages/kit/src/skills/storage/memory.ts @@ -0,0 +1,117 @@ +import type { SkillDefinition, SkillResourceDescriptor } from '../types' +import type { SkillImporter, SkillStorage, SkillSummary } from './types' + +const toSummary = (skill: SkillDefinition): SkillSummary => ({ + name: skill.name, + description: skill.description, + resourceCount: skill.resources?.length ?? 0, + metadata: skill.metadata, +}) + +const cloneSkill = (skill: SkillDefinition): SkillDefinition => ({ + name: skill.name, + description: skill.description, + instructions: skill.instructions, + metadata: skill.metadata ? { ...skill.metadata } : undefined, + resources: skill.resources?.map(cloneResource), +}) + +const cloneResource = (resource: SkillResourceDescriptor): SkillResourceDescriptor => { + const base = { + path: resource.path, + resourceId: resource.resourceId, + mimeType: resource.mimeType, + size: resource.size, + lastModified: resource.lastModified, + metadata: resource.metadata ? { ...resource.metadata } : undefined, + } + + if (resource.kind === 'text') { + const content = { + binary: resource.binary ? new Uint8Array(resource.binary) : undefined, + readBinary: resource.readBinary, + } + + return resource.text !== undefined + ? { + ...base, + ...content, + kind: resource.kind, + text: resource.text, + readText: resource.readText, + } + : { + ...base, + ...content, + kind: resource.kind, + readText: resource.readText!, + } + } + + const content = { + text: resource.text, + readText: resource.readText, + } + + return resource.binary + ? { + ...base, + ...content, + kind: resource.kind, + binary: new Uint8Array(resource.binary), + readBinary: resource.readBinary, + } + : { + ...base, + ...content, + kind: resource.kind, + readBinary: resource.readBinary!, + } +} + +export class MemorySkillStorage implements SkillStorage { + private skills = new Map() + + constructor(private readonly importer: SkillImporter) {} + + async add(skill: SkillDefinition) { + const saved = cloneSkill(skill) + this.skills.set(skill.name, saved) + return cloneSkill(saved) + } + + async get(name: string) { + const skill = this.skills.get(name) + return skill ? cloneSkill(skill) : undefined + } + + async has(name: string) { + return this.skills.has(name) + } + + async delete(name: string) { + return this.skills.delete(name) + } + + async list() { + return Array.from(this.skills.values()) + .map(toSummary) + .sort((a, b) => a.name.localeCompare(b.name)) + } + + import(options: TImportOptions) { + const task = this.importer(options) + + return Object.assign( + task.then(async (result) => { + await this.add(result.skill) + return result + }), + { cancel: task.cancel }, + ) + } +} + +export function createMemorySkillStorage(importer: SkillImporter) { + return new MemorySkillStorage(importer) +} diff --git a/packages/kit/src/skills/storage/node.ts b/packages/kit/src/skills/storage/node.ts new file mode 100644 index 000000000..0555270c1 --- /dev/null +++ b/packages/kit/src/skills/storage/node.ts @@ -0,0 +1,18 @@ +import { loadSkillWithDetails } from '../loader/node' +import type { SkillLoadOptions } from '../loader/node' +import type { SkillStorage as SkillStorageBase } from './types' +import { createImportSkill } from './importSkill' +import { createMemorySkillStorage as createMemorySkillStorageBase } from './memory' + +export type { SkillImportJob, SkillImportResult } from './types' +export type SkillImportOptions = SkillLoadOptions +export type SkillStorage = SkillStorageBase +export { MemorySkillStorage } from './memory' + +export const importSkill = createImportSkill(loadSkillWithDetails) + +export function createMemorySkillStorage() { + return createMemorySkillStorageBase(importSkill) +} +export { createFsSkillStorage, FsSkillStorage } from './fs' +export type { FsSkillStorageOptions } from './fs' diff --git a/packages/kit/src/skills/storage/types.ts b/packages/kit/src/skills/storage/types.ts new file mode 100644 index 000000000..c6a269bf4 --- /dev/null +++ b/packages/kit/src/skills/storage/types.ts @@ -0,0 +1,48 @@ +import type { SkillDefinition } from '../types' +import type { SkillLoadWarning } from '../loader/type' + +/** skill 摘要,用于 list()。 */ +export interface SkillSummary { + name: string + description: string + resourceCount: number + metadata?: Record +} + +/** + * skill 持久化与导入。 + * + * @example + * await storage.add(skill) + * const saved = await storage.get('weather') + * const summaries = await storage.list() + */ +export interface SkillStorage { + add(skill: SkillDefinition): Promise + get(name: string): Promise + has(name: string): Promise + delete(name: string): Promise + list(): Promise + import(options: TImportOptions): SkillImportJob +} + +export interface SkillImportResult { + name: string + skill: SkillDefinition + warnings: SkillLoadWarning[] +} + +/** + * 进行中的导入操作;await 得到 SkillImportResult。 + * + * @example + * const job = storage.import({ source: 'browser', fileList: input.files }) + * job.cancel() + * const { name, warnings } = await job + */ +export type SkillImportJob = Promise & { + /** 中止导入。 */ + cancel(): void +} + +export type SkillImporter = (options: TImportOptions) => SkillImportJob diff --git a/packages/kit/src/skills/test/fsStorage.test.ts b/packages/kit/src/skills/test/fsStorage.test.ts new file mode 100644 index 000000000..718352a06 --- /dev/null +++ b/packages/kit/src/skills/test/fsStorage.test.ts @@ -0,0 +1,106 @@ +import { cp, mkdtemp, readFile, writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import { fileURLToPath } from 'node:url' +import { describe, expect, it } from 'vitest' +import { createFsSkillStorage } from '../storage/node' + +const createTempRoot = () => mkdtemp(join(tmpdir(), 'tiny-robot-skill-storage-')) + +describe('FsSkillStorage', () => { + it('adds and restores skills in native directory format with lazy resources', async () => { + const root = await createTempRoot() + const storage = createFsSkillStorage({ root }) + + await storage.add({ + name: 'demo', + description: 'Demo skill', + instructions: '# Demo\n\nUse this skill.', + metadata: { + homepage: 'https://example.com/demo', + version: '1.0.0', + }, + resources: [ + { + path: 'references/guide.md', + kind: 'text', + resourceId: 'references/guide.md', + text: '# Guide', + }, + { + path: 'assets/icon.bin', + kind: 'binary', + resourceId: 'assets/icon.bin', + binary: new Uint8Array([1, 2, 3]), + }, + ], + }) + + await expect(readFile(join(root, 'demo', 'SKILL.md'), 'utf8')).resolves.toContain( + 'homepage: https://example.com/demo', + ) + await expect(readFile(join(root, 'demo', 'references', 'guide.md'), 'utf8')).resolves.toBe('# Guide') + + const storedSkill = await storage.get('demo') + const guide = storedSkill?.resources?.find((resource) => resource.path === 'references/guide.md') + + expect(storedSkill).toMatchObject({ + name: 'demo', + description: 'Demo skill', + instructions: '# Demo\n\nUse this skill.', + metadata: { + homepage: 'https://example.com/demo', + version: '1.0.0', + }, + }) + expect(guide).toMatchObject({ + path: 'references/guide.md', + kind: 'text', + }) + expect(guide).not.toHaveProperty('text') + + await writeFile(join(root, 'demo', 'references', 'guide.md'), '# Updated', 'utf8') + await expect(guide?.readText?.()).resolves.toBe('# Updated') + }) + + it('lists existing skill directories, imports another skill, and deletes skills', async () => { + const root = await createTempRoot() + const weatherRoot = fileURLToPath(new URL('./.cache/weather', import.meta.url)) + const vueRoot = fileURLToPath(new URL('./.cache/vue-best-practices', import.meta.url)) + await cp(weatherRoot, join(root, 'weather'), { + recursive: true, + }) + + const storage = createFsSkillStorage({ root }) + + await expect(storage.list()).resolves.toEqual([ + expect.objectContaining({ + name: 'weather', + description: expect.stringContaining('weather'), + }), + ]) + const existingSkill = await storage.get('weather') + expect(existingSkill?.instructions).toContain('# Weather Skill') + + const result = await storage.import({ + source: 'fs', + root: vueRoot, + }) + + expect(result.skill.name).toBe('vue-best-practices') + await expect(storage.list()).resolves.toEqual( + expect.arrayContaining([ + expect.objectContaining({ + name: 'weather', + }), + expect.objectContaining({ + name: 'vue-best-practices', + }), + ]), + ) + expect(await storage.has('weather')).toBe(true) + expect(await storage.delete('weather')).toBe(true) + expect(await storage.get('weather')).toBeUndefined() + expect(await storage.has('vue-best-practices')).toBe(true) + }) +}) diff --git a/packages/kit/src/skills/test/indexedDBStorage.test.ts b/packages/kit/src/skills/test/indexedDBStorage.test.ts new file mode 100644 index 000000000..03a15a658 --- /dev/null +++ b/packages/kit/src/skills/test/indexedDBStorage.test.ts @@ -0,0 +1,183 @@ +import 'fake-indexeddb/auto' +import { describe, expect, it } from 'vitest' +import { createIndexedDBSkillStorage } from '../storage' +import type { SkillDefinition } from '../types' + +const databaseName = () => `tiny-robot-skills-test-${crypto.randomUUID()}` + +describe('IndexedDBSkillStorage', () => { + it('adds, gets, lists, checks, and deletes skills', async () => { + const storage = createIndexedDBSkillStorage({ databaseName: databaseName() }) + + const saved = await storage.add({ + name: 'demo', + description: 'Demo skill', + instructions: '# Demo', + metadata: { + homepage: 'https://example.com', + }, + }) + + expect(saved).toMatchObject({ + name: 'demo', + description: 'Demo skill', + instructions: '# Demo', + metadata: { + homepage: 'https://example.com', + }, + }) + expect(await storage.has('demo')).toBe(true) + expect(await storage.list()).toEqual([ + { + name: 'demo', + description: 'Demo skill', + resourceCount: 0, + metadata: { + homepage: 'https://example.com', + }, + }, + ]) + + expect(await storage.delete('demo')).toBe(true) + expect(await storage.delete('demo')).toBe(false) + expect(await storage.has('demo')).toBe(false) + expect(await storage.get('demo')).toBeUndefined() + }) + + it('restores resources as lazy readers without eager content', async () => { + const storage = createIndexedDBSkillStorage({ databaseName: databaseName() }) + + await storage.add({ + name: 'docs', + description: 'Docs skill', + instructions: '# Docs', + resources: [ + { + path: 'references/guide.md', + kind: 'text', + resourceId: 'references/guide.md', + text: '# Guide', + readText: async () => '# Guide', + }, + { + path: 'assets/icon.png', + kind: 'binary', + resourceId: 'assets/icon.png', + binary: new Uint8Array([1, 2, 3]), + readBinary: async () => new Uint8Array([1, 2, 3]), + mimeType: 'image/png', + }, + ], + }) + + const skill = await storage.get('docs') + const textResource = skill?.resources?.find((resource) => resource.path === 'references/guide.md') + const binaryResource = skill?.resources?.find((resource) => resource.path === 'assets/icon.png') + + expect(textResource).toMatchObject({ + path: 'references/guide.md', + kind: 'text', + resourceId: 'references/guide.md', + }) + expect(textResource).not.toHaveProperty('text') + await expect(textResource?.readText?.()).resolves.toBe('# Guide') + await expect(textResource?.readBinary?.()).resolves.toEqual(new TextEncoder().encode('# Guide')) + + expect(binaryResource).toMatchObject({ + path: 'assets/icon.png', + kind: 'binary', + resourceId: 'assets/icon.png', + mimeType: 'image/png', + }) + expect(binaryResource).not.toHaveProperty('binary') + await expect(binaryResource?.readBinary?.()).resolves.toEqual(new Uint8Array([1, 2, 3])) + await expect(binaryResource?.readText?.()).resolves.toBe(new TextDecoder().decode(new Uint8Array([1, 2, 3]))) + }) + + it('overwrites stale resource records when replacing a skill', async () => { + const storage = createIndexedDBSkillStorage({ databaseName: databaseName() }) + + await storage.add({ + name: 'docs', + description: 'Docs skill', + instructions: '# Docs', + resources: [ + { + path: 'old.md', + kind: 'text', + resourceId: 'old.md', + text: 'old', + }, + ], + }) + + await storage.add({ + name: 'docs', + description: 'Updated docs skill', + instructions: '# Updated', + resources: [ + { + path: 'new.md', + kind: 'text', + resourceId: 'new.md', + text: 'new', + }, + ], + }) + + const skill = await storage.get('docs') + + expect(skill?.description).toBe('Updated docs skill') + expect(skill?.resources?.map((resource) => resource.path)).toEqual(['new.md']) + await expect(skill?.resources?.[0]?.readText?.()).resolves.toBe('new') + }) + + it('imports skills from browser sources', async () => { + const storage = createIndexedDBSkillStorage({ databaseName: databaseName() }) + + const file = new File( + [['---', 'name: browser-docs', 'description: Browser docs skill', '---', '', '# Browser Docs'].join('\n')], + 'SKILL.md', + { type: 'text/markdown' }, + ) + + const result = await storage.import({ + source: 'browser', + fileList: [file], + }) + + expect(result.name).toBe('browser-docs') + expect(await storage.get('browser-docs')).toMatchObject({ + name: 'browser-docs', + instructions: '# Browser Docs', + }) + }) + + it('persists resources through lazy readers during add', async () => { + const storage = createIndexedDBSkillStorage({ databaseName: databaseName() }) + const skill: SkillDefinition = { + name: 'lazy', + description: 'Lazy skill', + instructions: '# Lazy', + resources: [ + { + path: 'lazy.md', + kind: 'text', + resourceId: 'lazy.md', + readText: async () => { + await new Promise((resolve) => setTimeout(resolve, 0)) + return 'lazy text' + }, + }, + ], + } + + await storage.add(skill) + + const storedSkill = await storage.get('lazy') + const resource = storedSkill?.resources?.[0] + + expect(resource).not.toHaveProperty('text') + await expect(resource?.readText?.()).resolves.toBe('lazy text') + }) +}) diff --git a/packages/kit/src/skills/test/memoryStorage.test.ts b/packages/kit/src/skills/test/memoryStorage.test.ts new file mode 100644 index 000000000..6101a2494 --- /dev/null +++ b/packages/kit/src/skills/test/memoryStorage.test.ts @@ -0,0 +1,130 @@ +import { describe, expect, it } from 'vitest' +import { createMemorySkillStorage, importSkill } from '../storage' + +type TestFile = File & { + webkitRelativePath?: string +} + +const createTestFile = (path: string, content: string): TestFile => + ({ + name: path.split('/').at(-1) ?? path, + webkitRelativePath: path, + type: 'text/markdown', + size: content.length, + lastModified: 123, + text: async () => content, + arrayBuffer: async () => { + const bytes = new TextEncoder().encode(content) + return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength) + }, + }) as TestFile + +describe('MemorySkillStorage', () => { + it('add, get, has, delete, and list', async () => { + const storage = createMemorySkillStorage() + + const saved = await storage.add({ + name: 'demo', + description: 'Demo skill', + instructions: '# Demo', + }) + + expect(saved.name).toBe('demo') + expect(await storage.has('demo')).toBe(true) + expect(await storage.get('demo')).toMatchObject({ + name: 'demo', + description: 'Demo skill', + instructions: '# Demo', + }) + + const summaries = await storage.list() + expect(summaries).toEqual([ + { + name: 'demo', + description: 'Demo skill', + resourceCount: 0, + metadata: undefined, + }, + ]) + + expect(await storage.delete('demo')).toBe(true) + expect(await storage.has('demo')).toBe(false) + expect(await storage.get('demo')).toBeUndefined() + }) + + it('imports skill from browser source', async () => { + const storage = createMemorySkillStorage() + + const { name, skill, warnings } = await storage.import({ + source: 'browser', + fileList: [ + createTestFile( + 'weather/SKILL.md', + ['---', 'name: weather', 'description: Weather skill', '---', '', '# Weather Skill'].join('\n'), + ), + ], + }) + + expect(name).toBe('weather') + expect(skill.name).toBe('weather') + expect(skill.instructions).toContain('# Weather Skill') + expect(warnings).toEqual([]) + + const storedSkill = await storage.get('weather') + expect(storedSkill?.instructions).toContain('# Weather Skill') + expect(storedSkill?.resources?.some((resource) => resource.path === 'SKILL.md')).toBeFalsy() + }) + + it('imports multi-file skill with readable resources', async () => { + const storage = createMemorySkillStorage() + + await storage.import({ + source: 'browser', + fileList: [ + createTestFile( + 'vue-best-practices/SKILL.md', + [ + '---', + 'name: vue-best-practices', + 'description: Vue.js tasks', + '---', + '', + '# Vue Best Practices Workflow', + ].join('\n'), + ), + createTestFile('vue-best-practices/references/reactivity.md', '# Reactivity'), + ], + }) + + const skill = await storage.get('vue-best-practices') + const resource = skill?.resources?.find((item) => item.path === 'references/reactivity.md') + + expect(resource).toBeDefined() + await expect(resource?.readText?.()).resolves.toContain('# Reactivity') + }) + + it('supports cancel on import task', async () => { + let releaseWait!: () => void + const waitForText = new Promise((resolve) => { + releaseWait = () => resolve('# Cancelled') + }) + const task = importSkill({ + source: 'browser', + fileList: [ + { + ...createTestFile( + 'cancelled/SKILL.md', + ['---', 'name: cancelled', 'description: Cancelled skill', '---', '', '# Cancelled'].join('\n'), + ), + text: async () => waitForText, + }, + ], + }) + task.cancel() + releaseWait() + + await expect(task).rejects.toMatchObject({ + name: 'SkillLoadCancelledError', + }) + }) +})