Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions skills/faceless-explainer/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -105,9 +105,9 @@ Goal: Generate narration, word timings, music, and audio metadata from the appro

Start audio after Step 3 approval. Run it in the background, then continue to Step 4. (Sign-in status was already shown in Step 0; the engine falls back automatically.)

**Choose the narration voice from the user's ask before invoking.** If the request named a voice, gender, or tone, pick a matching voice id and pass it with `--voice <id>`. The pipeline default is otherwise **Marcia (female)** on HeyGen / `am_michael` on Kokoro — so a request like "a male voice" is silently ignored unless you pass the flag. Voice ids are provider-specific; resolve against whichever provider Step 0's sign-in status selected: **HeyGen** (signed in) via `node ../media-use/audio/scripts/heygen-tts.mjs --list` (or `GET /v3/voices?engine=starfish`); **Kokoro** (offline) via the voice table in `../media-use/audio/references/tts.md` (prefixes `am_`/`bm_` male, `af_`/`bf_` female). Omit `--voice` only when the user expressed no preference.
**Choose the narration provider and voice from the user's ask before invoking.** Pass an explicit offline or alternate provider with `--provider kokoro|elevenlabs|heygen` (or `HF_TTS_PROVIDER`); the CLI flag takes precedence. If the request named a voice, gender, or tone, pick a matching voice id and pass it with `--voice <id>`. The pipeline default is otherwise **Marcia (female)** on HeyGen / `am_michael` on Kokoro — so a request like "a male voice" is silently ignored unless you pass the flag. Voice ids are provider-specific; resolve against whichever provider Step 0's sign-in status selected: **HeyGen** (signed in) via `node ../media-use/audio/scripts/heygen-tts.mjs --list` (or `GET /v3/voices?engine=starfish`); **Kokoro** (offline) via the voice table in `../media-use/audio/references/tts.md` (prefixes `am_`/`bm_` male, `af_`/`bf_` female). Omit `--voice` only when the user expressed no preference.

`node <SKILL_DIR>/scripts/audio.mjs --script ./SCRIPT.md --storyboard ./STORYBOARD.md --hyperframes . --out ./audio_meta.json --voice <voice-id> &`
`node <SKILL_DIR>/scripts/audio.mjs --script ./SCRIPT.md --storyboard ./STORYBOARD.md --hyperframes . --out ./audio_meta.json --provider <provider> --voice <voice-id> &`

The audio script handles narration, word timings, BGM lookup from HeyGen's music library, and timing metadata. BGM mood comes from the storyboard's `music:` field. This uses the HeyGen Audio API for retrieval, not generation, and the same `~/.heygen` credential as TTS. For provider details, read `../media-use/audio/references/tts.md`.

Expand Down
3 changes: 2 additions & 1 deletion skills/faceless-explainer/scripts/audio.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,7 @@ function runGenerate(argv) {
const storyboardPath = resolve(flag(argv, "storyboard", join(hyperframesDir, "STORYBOARD.md")));
const scriptPath = resolve(flag(argv, "script", join(hyperframesDir, "SCRIPT.md")));
const outPath = resolve(flag(argv, "out", join(hyperframesDir, "audio_meta.json")));
const provider = flag(argv, "provider", process.env.HF_TTS_PROVIDER || "auto");
const userVoice = flag(argv, "voice", null);
const speed = Number(flag(argv, "speed", "1.0")) || 1.0;

Expand All @@ -144,7 +145,7 @@ function runGenerate(argv) {
// strict here (no wait-bgm step downstream).
const query = (g.extra && g.extra.music) || g.message || g.arc || "calm cinematic underscore";
const request = {
provider: "auto",
provider,
speed,
lines,
bgm: { mode: "retrieve", query, blob: g.message || "", arc: g.arc || "" },
Expand Down
50 changes: 50 additions & 0 deletions skills/faceless-explainer/scripts/audio.test.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
import assert from "node:assert/strict";
import { spawnSync } from "node:child_process";
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import test from "node:test";

const script = new URL("./audio.mjs", import.meta.url).pathname;

function runAudio({ args = [], env = {} } = {}) {
const dir = mkdtempSync(join(tmpdir(), "faceless-audio-"));
const engine = join(dir, "engine.mjs");
try {
writeFileSync(join(dir, "STORYBOARD.md"), "---\nmessage: Test\n---\n");
writeFileSync(
engine,
`import { readFileSync, writeFileSync } from "node:fs";
const argv = process.argv.slice(2);
const flag = (name) => argv[argv.indexOf(name) + 1];
const request = JSON.parse(readFileSync(flag("--request"), "utf8"));
writeFileSync(new URL("request.json", import.meta.url), JSON.stringify(request));
writeFileSync(flag("--out"), JSON.stringify({ voices: [], bgm: null, sfx: [] }));
`,
);
const result = spawnSync(
process.execPath,
[script, "--hyperframes", dir, "--storyboard", join(dir, "STORYBOARD.md"), ...args],
{ encoding: "utf8", env: { ...process.env, HF_MEDIA_ENGINE: engine, ...env } },
);
assert.equal(result.status, 0, result.stderr);
return JSON.parse(readFileSync(join(dir, "request.json"), "utf8"));
} finally {
rmSync(dir, { recursive: true, force: true });
}
}

test("passes --provider to the shared audio engine", () => {
assert.equal(runAudio({ args: ["--provider", "kokoro"] }).provider, "kokoro");
});

test("uses HF_TTS_PROVIDER when --provider is omitted", () => {
assert.equal(runAudio({ env: { HF_TTS_PROVIDER: "elevenlabs" } }).provider, "elevenlabs");
});

test("--provider takes precedence over HF_TTS_PROVIDER", () => {
assert.equal(
runAudio({ args: ["--provider", "kokoro"], env: { HF_TTS_PROVIDER: "elevenlabs" } }).provider,
"kokoro",
);
});
5 changes: 3 additions & 2 deletions skills/media-use/audio/scripts/audio.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -111,8 +111,9 @@ const speed = Number(speedOverride ?? request.speed ?? 1.0) || 1.0;

// ── env + HeyGen availability (the single switch) ─────────────────────────────
loadEnvFromDir(hyperframesDir);
const heygenOK = heygenCredential() !== null;
const headers = heygenOK ? heygenAuthHeaders() : null;
const heygenCred = heygenCredential();
const headers = heygenCred?.headers ? heygenAuthHeaders() : null;
const heygenOK = headers !== null;

// ── merge base: preserve sections not selected by --only ──────────────────────
const prev = existsSync(outPath) ? JSON.parse(readFileSync(outPath, "utf8")) : {};
Expand Down
48 changes: 47 additions & 1 deletion skills/media-use/audio/scripts/audio.test.mjs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { strict as assert } from "node:assert";
import { spawnSync } from "node:child_process";
import { test } from "node:test";
import { mkdtempSync, rmSync, existsSync } from "node:fs";
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { join, dirname } from "node:path";
import { tmpdir } from "node:os";
import { fileURLToPath } from "node:url";
Expand All @@ -13,6 +14,51 @@ import { resolveSfx } from "./lib/sfx.mjs";
const HERE = dirname(fileURLToPath(import.meta.url));
const sfxLibDir = join(HERE, "..", "assets", "sfx"); // same offset the engine uses

test("explicit offline TTS provider bypasses expired HeyGen OAuth", () => {
const dir = mkdtempSync(join(tmpdir(), "mu-audio-expired-auth-"));
const configDir = join(dir, "config");
const requestPath = join(dir, "audio_request.json");
const outPath = join(dir, "audio_meta.json");
const engine = join(HERE, "audio.mjs");
try {
mkdirSync(configDir, { recursive: true });
writeFileSync(
join(configDir, "credentials"),
JSON.stringify({
oauth: { access_token: "expired", expires_at: "2000-01-01T00:00:00Z" },
}),
);
writeFileSync(
requestPath,
JSON.stringify({ provider: "kokoro", lines: [], bgm: { mode: "none" } }),
);
const env = { ...process.env, HEYGEN_CONFIG_DIR: configDir };
delete env.HEYGEN_API_KEY;
delete env.HYPERFRAMES_API_KEY;
const result = spawnSync(
process.execPath,
[
engine,
"--request",
requestPath,
"--hyperframes",
dir,
"--out",
outPath,
"--only",
"tts",
"--provider",
"kokoro",
],
{ encoding: "utf8", env },
);
assert.equal(result.status, 0, result.stderr);
assert.equal(JSON.parse(readFileSync(outPath, "utf8")).tts_provider, null);
} finally {
rmSync(dir, { recursive: true, force: true });
}
});

test("bundled SFX library resolves from the relocated path", async () => {
assert.ok(existsSync(join(sfxLibDir, "manifest.json")), "moved manifest is present");
const dir = mkdtempSync(join(tmpdir(), "mu-audio-"));
Expand Down
2 changes: 1 addition & 1 deletion skills/media-use/audio/scripts/lib/tts.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ import { pythonInvocation } from "./python.mjs";

// ── provider detection ────────────────────────────────────────────────────────
export function heygenAvailable() {
return heygenCredential() !== null;
return heygenCredential()?.headers != null;
}
export function elevenlabsAvailable() {
if (!process.env.ELEVENLABS_API_KEY) return false;
Expand Down
30 changes: 30 additions & 0 deletions skills/media-use/audio/scripts/lib/tts.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,43 @@ import { mkdtempSync, writeFileSync, chmodSync, rmSync, existsSync } from "node:
import { join, dirname } from "node:path";
import { tmpdir } from "node:os";
import {
heygenAvailable,
parseFfmpegDurationBanner,
ffprobeDuration,
synthesizeOne,
synthesizeHeygen,
synthResult,
} from "./tts.mjs";

test("expired HeyGen OAuth is not an available TTS provider", () => {
const dir = mkdtempSync(join(tmpdir(), "tts-expired-heygen-"));
const saved = {
apiKey: process.env.HEYGEN_API_KEY,
hyperframesApiKey: process.env.HYPERFRAMES_API_KEY,
configDir: process.env.HEYGEN_CONFIG_DIR,
};
try {
delete process.env.HEYGEN_API_KEY;
delete process.env.HYPERFRAMES_API_KEY;
process.env.HEYGEN_CONFIG_DIR = dir;
writeFileSync(
join(dir, "credentials"),
JSON.stringify({
oauth: { access_token: "expired", expires_at: "2000-01-01T00:00:00Z" },
}),
);
assert.equal(heygenAvailable(), false);
} finally {
if (saved.apiKey === undefined) delete process.env.HEYGEN_API_KEY;
else process.env.HEYGEN_API_KEY = saved.apiKey;
if (saved.hyperframesApiKey === undefined) delete process.env.HYPERFRAMES_API_KEY;
else process.env.HYPERFRAMES_API_KEY = saved.hyperframesApiKey;
if (saved.configDir === undefined) delete process.env.HEYGEN_CONFIG_DIR;
else process.env.HEYGEN_CONFIG_DIR = saved.configDir;
rmSync(dir, { recursive: true, force: true });
}
});

test("parseFfmpegDurationBanner reads ffmpeg's stderr Duration line", () => {
const stderr = [
"ffmpeg version 6.0",
Expand Down
Loading