diff --git a/src/apphosting/localbuilds.spec.ts b/src/apphosting/localbuilds.spec.ts index 34cbd0f4383..18f9c421b44 100644 --- a/src/apphosting/localbuilds.spec.ts +++ b/src/apphosting/localbuilds.spec.ts @@ -1,15 +1,21 @@ import * as sinon from "sinon"; import { expect } from "chai"; import * as localBuildModule from "@apphosting/build"; -import { localBuild } from "./localbuilds"; +import { localBuild, runUniversalMaker } from "./localbuilds"; import * as secrets from "./secrets"; import { EnvMap } from "./yaml"; +import * as childProcess from "child_process"; +import * as fs from "fs"; +import * as experiments from "../experiments"; +import * as universalMakerDownload from "./universalMakerDownload"; describe("localBuild", () => { + beforeEach(() => { + sinon.stub(experiments, "isEnabled").returns(false); + }); afterEach(() => { sinon.restore(); }); - it("returns the expected output", async () => { const bundleConfig = { version: "v1" as const, @@ -202,4 +208,104 @@ describe("localBuild", () => { expect(confirmStub).to.have.been.calledOnce; }); }); + + describe("runUniversalMaker", () => { + let downloadStub: sinon.SinonStub; + let readFileSyncStub: sinon.SinonStub; + + beforeEach(() => { + downloadStub = sinon + .stub(universalMakerDownload, "getOrDownloadUniversalMaker") + .resolves("/path/to/universal_maker"); + readFileSyncStub = sinon.stub(fs, "readFileSync").callsFake((pathStr: any) => { + if (typeof pathStr === "string" && pathStr.endsWith("bundle.yaml")) { + return ` + runConfig: + runCommand: npm run start + outputFiles: + serverApp: + include: + - .next/standalone + `; + } + if (typeof pathStr === "string" && pathStr.endsWith("build_output.json")) { + return JSON.stringify({ + command: "npm", + args: ["run", "start"], + language: "nodejs", + runtime: "nodejs22", + envVars: { + PORT: "3000", + }, + }); + } + return ""; + }); + }); + + it("should successfully execute Universal Maker and parse output", async () => { + const spawnStub = sinon.stub(childProcess, "spawnSync").returns({ + status: 0, + output: ["", "mock output", ""], + pid: 12345, + stdout: "mock stdout", + stderr: "mock stderr", + signal: null, + }); + + sinon.stub(fs, "existsSync").returns(true); + sinon.stub(fs, "mkdirSync"); + sinon.stub(fs, "readdirSync").returns(["bundle.yaml"] as any); + sinon.stub(fs, "renameSync"); + sinon.stub(fs, "rmSync"); + sinon.stub(fs, "rmdirSync"); + sinon.stub(fs, "unlinkSync"); + const output = await runUniversalMaker("./", "nextjs"); + + expect(output).to.deep.equal({ + metadata: { + language: "nodejs", + runtime: "nodejs22", + framework: "nextjs", + }, + runConfig: { + runCommand: "npm run start", + environmentVariables: [{ variable: "PORT", value: "3000", availability: ["RUNTIME"] }], + }, + outputFiles: { + serverApp: { + include: [".next/standalone"], + }, + }, + }); + + sinon.assert.calledOnce(spawnStub); + sinon.assert.calledWith( + spawnStub, + "/path/to/universal_maker", + ["-application_dir", "./", "-output_dir", "./", "-output_format", "json"], + sinon.match({ + env: sinon.match({ + X_GOOGLE_TARGET_PLATFORM: "fah", + }), + }), + ); + sinon.assert.calledTwice(readFileSyncStub); + sinon.assert.calledOnce(downloadStub); + }); + + it("should raise clear FirebaseError on permission errors within child execution", async () => { + sinon.stub(childProcess, "spawnSync").callsFake(() => { + const err = new Error("EACCES exception") as NodeJS.ErrnoException; + err.code = "EACCES"; + + throw err; + }); + + await expect(runUniversalMaker("./")).to.be.rejectedWith( + "Failed to execute the Universal Maker binary due to permission constraints. Please assure you have set chmod +x on your file.", + ); + sinon.assert.calledOnce(downloadStub); + }); + }); }); diff --git a/src/apphosting/localbuilds.ts b/src/apphosting/localbuilds.ts index fcc19687a3d..38dc17aa3f0 100644 --- a/src/apphosting/localbuilds.ts +++ b/src/apphosting/localbuilds.ts @@ -1,14 +1,181 @@ -import { BuildConfig, Env } from "../gcp/apphosting"; +import * as childProcess from "child_process"; +import * as fs from "fs"; +import * as path from "path"; +import { Availability, BuildConfig, Env } from "../gcp/apphosting"; + import { localBuild as localAppHostingBuild } from "@apphosting/build"; import { EnvMap } from "./yaml"; import { loadSecret } from "./secrets"; import { confirm } from "../prompt"; import { FirebaseError } from "../error"; +import * as experiments from "../experiments"; +import { logger } from "../logger"; +import { wrappedSafeLoad } from "../utils"; +import { getOrDownloadUniversalMaker } from "./universalMakerDownload"; + +interface UniversalMakerOutput { + command: string; + args: string[]; + language: string; + runtime: string; + envVars?: Record; +} + +/** + * Runs the Universal Maker binary to build the project. + */ +export async function runUniversalMaker( + projectRoot: string, + framework?: string, +): Promise { + const universalMakerBinary = await getOrDownloadUniversalMaker(); + + try { + const bundleOutput = path.join(projectRoot, "bundle_output"); + if (fs.existsSync(bundleOutput)) { + fs.rmSync(bundleOutput, { recursive: true, force: true }); + } + fs.mkdirSync(bundleOutput, { recursive: true }); + + const res = childProcess.spawnSync( + universalMakerBinary, + ["-application_dir", projectRoot, "-output_dir", projectRoot, "-output_format", "json"], + { + env: { + ...process.env, + X_GOOGLE_TARGET_PLATFORM: "fah", + FIREBASE_OUTPUT_BUNDLE_DIR: bundleOutput, + }, + stdio: "pipe", + }, + ); + + if (res.stdout) { + logger.debug("[Universal Maker stdout]:\n" + res.stdout.toString()); + } + if (res.stderr) { + logger.debug("[Universal Maker stderr]:\n" + res.stderr.toString()); + } + + if (res.error) { + throw res.error; + } + if (res.status !== 0) { + throw new FirebaseError(`Universal Maker failed with exit code ${res.status ?? "unknown"}.`); + } + } catch (e) { + if (e && typeof e === "object" && "code" in e && e.code === "EACCES") { + throw new FirebaseError( + "Failed to execute the Universal Maker binary due to permission constraints. Please assure you have set chmod +x on your file.", + ); + } + throw e; + } + + const outputFilePath = path.join(projectRoot, "build_output.json"); + if (!fs.existsSync(outputFilePath)) { + throw new FirebaseError( + `Universal Maker did not produce the expected output file at ${outputFilePath}`, + ); + } + const outputRaw = fs.readFileSync(outputFilePath, "utf-8"); + fs.unlinkSync(outputFilePath); // Clean up temporary metadata file + + const bundleOutput = path.join(projectRoot, "bundle_output"); + const targetAppHosting = path.join(projectRoot, ".apphosting"); + + // Universal Maker has a bug where it accidentally empties bundle.yaml if we tell it to output directly to .apphosting. + // To avoid this, we output to bundle_output first, and then safely move the files over. + if (fs.existsSync(bundleOutput)) { + if (!fs.existsSync(targetAppHosting)) { + fs.mkdirSync(targetAppHosting, { recursive: true }); + } + const files = fs.readdirSync(bundleOutput); + for (const file of files) { + const dest = path.join(targetAppHosting, file); + if (fs.existsSync(dest)) { + fs.rmSync(dest, { recursive: true, force: true }); + } + fs.renameSync(path.join(bundleOutput, file), dest); + } + fs.rmdirSync(bundleOutput); + } + + let umOutput: UniversalMakerOutput; + try { + umOutput = JSON.parse(outputRaw) as UniversalMakerOutput; + } catch (e) { + throw new FirebaseError(`Failed to parse build_output.json: ${(e as Error).message}`); + } + + let finalRunCommand = `${umOutput.command} ${umOutput.args.join(" ")}`; + let finalOutputFiles: string[] = [".apphosting"]; // Fallback + const bundleYamlPath = path.join(projectRoot, ".apphosting", "bundle.yaml"); + if (fs.existsSync(bundleYamlPath)) { + try { + const bundleRaw = fs.readFileSync(bundleYamlPath, "utf-8"); + // Safely parse the YAML string + const bundleData = wrappedSafeLoad(bundleRaw); + + if (bundleData?.runConfig?.runCommand) { + finalRunCommand = bundleData.runConfig.runCommand; + } + + if (bundleData?.outputFiles?.serverApp?.include) { + finalOutputFiles = bundleData.outputFiles.serverApp.include; + } + } catch (e: any) { + logger.debug(`Failed to parse bundle.yaml: ${e.message}`); + } + } + + return { + metadata: { + language: umOutput.language, + runtime: umOutput.runtime, + framework: framework || "nextjs", + }, + runConfig: { + runCommand: finalRunCommand, + environmentVariables: Object.entries(umOutput.envVars || {}) + .filter(([k]) => k !== "FIREBASE_OUTPUT_BUNDLE_DIR") + .map(([k, v]) => ({ + variable: k, + value: String(v), + availability: ["RUNTIME"], + })), + }, + outputFiles: { + serverApp: { + include: finalOutputFiles, + }, + }, + }; +} + +export interface AppHostingBuildOutput { + metadata: Record; + + runConfig: { + runCommand?: string; + environmentVariables?: Array<{ + variable: string; + value: string; + availability: string[]; + }>; + }; + outputFiles?: { + serverApp: { + include: string[]; + }; + }; +} /** * Triggers a local build of your App Hosting codebase. * * This function orchestrates the build process using the App Hosting build adapter. + * * It detects the framework (though currently defaults/assumes 'nextjs' in some contexts), * generates the necessary build artifacts, and returns metadata about the build. * @param projectId - The project ID to use for resolving secrets. @@ -62,9 +229,26 @@ export async function localBuild( process.env[key] = value; } - let apphostingBuildOutput; + let apphostingBuildOutput: AppHostingBuildOutput; try { - apphostingBuildOutput = await localAppHostingBuild(projectRoot, framework); + if (experiments.isEnabled("universalMaker")) { + apphostingBuildOutput = await runUniversalMaker(projectRoot, framework); + logger.debug( + `[apphosting] Universal Maker build outputFiles include: ${JSON.stringify(apphostingBuildOutput.outputFiles?.serverApp?.include ?? [])}`, + ); + } else { + const buildResult = await localAppHostingBuild(projectRoot, framework); + apphostingBuildOutput = { + metadata: Object.fromEntries( + Object.entries(buildResult.metadata || {}).map(([k, v]) => [ + k, + v as string | number | boolean, + ]), + ), + runConfig: buildResult.runConfig, + outputFiles: buildResult.outputFiles, + }; + } } finally { for (const key in process.env) { if (!(key in originalEnv)) { @@ -82,11 +266,16 @@ export async function localBuild( const discoveredEnv: Env[] | undefined = apphostingBuildOutput.runConfig.environmentVariables?.map( - ({ variable, value, availability }) => ({ - variable, - value, - availability, - }), + ({ variable, value, availability }) => { + const validAvail = availability.filter( + (a): a is Availability => a === "BUILD" || a === "RUNTIME", + ); + return { + variable, + value, + availability: validAvail, + }; + }, ); return { diff --git a/src/apphosting/universalMakerDownload.spec.ts b/src/apphosting/universalMakerDownload.spec.ts new file mode 100644 index 00000000000..3f94d968e73 --- /dev/null +++ b/src/apphosting/universalMakerDownload.spec.ts @@ -0,0 +1,82 @@ +import * as sinon from "sinon"; +import { expect } from "chai"; +import * as fs from "fs-extra"; +import * as downloadUtils from "../downloadUtils"; +import { getOrDownloadUniversalMaker } from "./universalMakerDownload"; + +describe("universalMakerDownload", () => { + let existsSyncStub: sinon.SinonStub; + let copySyncStub: sinon.SinonStub; + let chmodSyncStub: sinon.SinonStub; + let downloadToTmpStub: sinon.SinonStub; + let validateSizeStub: sinon.SinonStub; + let validateChecksumStub: sinon.SinonStub; + + let originalPlatform: string; + let originalArch: string; + + beforeEach(() => { + originalPlatform = process.platform; + originalArch = process.arch; + Object.defineProperty(process, "platform", { value: "linux", configurable: true }); + Object.defineProperty(process, "arch", { value: "x64", configurable: true }); + + existsSyncStub = sinon.stub(fs, "existsSync"); + copySyncStub = sinon.stub(fs, "copySync"); + chmodSyncStub = sinon.stub(fs, "chmodSync"); + downloadToTmpStub = sinon.stub(downloadUtils, "downloadToTmp"); + validateSizeStub = sinon.stub(downloadUtils, "validateSize"); + validateChecksumStub = sinon.stub(downloadUtils, "validateChecksum"); + }); + + afterEach(() => { + sinon.restore(); + Object.defineProperty(process, "platform", { value: originalPlatform, configurable: true }); + Object.defineProperty(process, "arch", { value: originalArch, configurable: true }); + }); + + it("should return cached binary if valid", async () => { + existsSyncStub.returns(true); + validateSizeStub.resolves(); + validateChecksumStub.resolves(); + + const result = await getOrDownloadUniversalMaker(); + expect(existsSyncStub).to.have.been.calledOnce; + expect(validateSizeStub).to.have.been.calledOnce; + expect(validateChecksumStub).to.have.been.calledOnce; + expect(downloadToTmpStub).to.not.have.been.called; + expect(result).to.include("universal-maker-linux-x64"); + }); + + it("should redownload if cached binary fails validation", async () => { + existsSyncStub.returns(true); + // Fail on first call (cache check), succeed on second (downloaded file) + validateSizeStub.onFirstCall().rejects(new Error("Invalid size")); + validateSizeStub.onSecondCall().resolves(); + + downloadToTmpStub.resolves("/tmp/downloaded_file"); + validateChecksumStub.resolves(); // For the new file + + const result = await getOrDownloadUniversalMaker(); + expect(existsSyncStub).to.have.been.calledOnce; + expect(validateSizeStub).to.have.been.calledTwice; + expect(downloadToTmpStub).to.have.been.calledOnce; + expect(copySyncStub).to.have.been.calledOnce; + expect(chmodSyncStub).to.have.been.calledOnce; + expect(result).to.include("universal-maker-linux-x64"); + }); + + it("should download if binary not in cache", async () => { + existsSyncStub.returns(false); + downloadToTmpStub.resolves("/tmp/downloaded_file"); + validateSizeStub.resolves(); + validateChecksumStub.resolves(); + + const result = await getOrDownloadUniversalMaker(); + expect(existsSyncStub).to.have.been.calledOnce; + expect(downloadToTmpStub).to.have.been.calledOnce; + expect(copySyncStub).to.have.been.calledOnce; + expect(chmodSyncStub).to.have.been.calledOnce; + expect(result).to.include("universal-maker-linux-x64"); + }); +}); diff --git a/src/apphosting/universalMakerDownload.ts b/src/apphosting/universalMakerDownload.ts new file mode 100644 index 00000000000..61a4b00340f --- /dev/null +++ b/src/apphosting/universalMakerDownload.ts @@ -0,0 +1,103 @@ +import * as fs from "fs-extra"; +import * as os from "os"; +import * as path from "path"; + +import { FirebaseError } from "../error"; +import * as downloadUtils from "../downloadUtils"; +import { logger } from "../logger"; +import * as universalMakerInfo from "./universalMakerInfo.json"; + +const CACHE_DIR = + process.env.FIREBASE_UNIVERSAL_MAKER_PATH || + path.join(os.homedir(), ".cache", "firebase", "universal-maker"); + +interface UniversalMakerUpdateDetails { + version: string; + expectedSize: number; + expectedChecksumSHA256: string; + remoteUrl: string; + downloadPathRelativeToCacheDir: string; +} + +const UNIVERSAL_MAKER_UPDATE_DETAILS: Record = + universalMakerInfo; + +function getPlatformInfo(): UniversalMakerUpdateDetails { + let platformKey = ""; + if (process.platform === "darwin") { + if (process.arch === "arm64") { + platformKey = "darwin_arm64"; + } else { + throw new FirebaseError( + "macOS Intel (darwin_x64) is not currently supported for Universal Maker.", + ); + } + } else if (process.platform === "linux") { + if (process.arch === "x64") { + platformKey = "linux_x64"; + } else { + throw new FirebaseError( + "Linux ARM (linux_arm64) is not currently supported for Universal Maker.", + ); + } + } else if (process.platform === "win32") { + throw new FirebaseError("Windows (win32) is not currently supported for Universal Maker."); + } else { + throw new FirebaseError( + `Unsupported platform for Universal Maker: ${process.platform} ${process.arch}`, + ); + } + + const details = UNIVERSAL_MAKER_UPDATE_DETAILS[platformKey]; + if (!details) { + throw new FirebaseError(`Could not find download details for platform: ${platformKey}`); + } + + return details; +} + +/** + * Gets the path to the Universal Maker binary, downloading it if necessary. + */ +export async function getOrDownloadUniversalMaker(): Promise { + const details = getPlatformInfo(); + const downloadPath = path.join(CACHE_DIR, details.downloadPathRelativeToCacheDir); + + const hasBinary = fs.existsSync(downloadPath); + + if (hasBinary) { + logger.debug(`[apphosting] Universal Maker binary found at cache: ${downloadPath}`); + try { + await downloadUtils.validateSize(downloadPath, details.expectedSize); + await downloadUtils.validateChecksum(downloadPath, details.expectedChecksumSHA256, "sha256"); + return downloadPath; + } catch (err) { + logger.warn( + `[apphosting] Cached Universal Maker binary failed verification: ${(err as Error).message}. Proceeding to redownload...`, + ); + } + } + + logger.info( + "Downloading Universal Maker, a tool required to build your App Hosting application locally...", + ); + fs.ensureDirSync(CACHE_DIR); + + let tmpfile: string; + try { + tmpfile = await downloadUtils.downloadToTmp(details.remoteUrl); + } catch (err: any) { + throw new FirebaseError(`Failed to download Universal Maker: ${(err as Error).message}`); + } + + await downloadUtils.validateSize(tmpfile, details.expectedSize); + await downloadUtils.validateChecksum(tmpfile, details.expectedChecksumSHA256, "sha256"); + + // Move to cache dir + fs.copySync(tmpfile, downloadPath); + + // Make it executable + fs.chmodSync(downloadPath, 0o755); + + return downloadPath; +} diff --git a/src/apphosting/universalMakerInfo.json b/src/apphosting/universalMakerInfo.json new file mode 100644 index 00000000000..751a36cd902 --- /dev/null +++ b/src/apphosting/universalMakerInfo.json @@ -0,0 +1,16 @@ +{ + "darwin_arm64": { + "version": "1.0.0", + "expectedSize": 16111618, + "expectedChecksumSHA256": "4b77d02a5f80f26d9bd1428f388c293c1fb264995d75b51c7d50fec7c87bcf58", + "remoteUrl": "https://artifactregistry.googleapis.com/download/v1/projects/serverless-runtimes-qa/locations/us-central1/repositories/universal-maker/files/darwin-arm64%3A1.0.0%3Auniversal_maker:download?alt=media", + "downloadPathRelativeToCacheDir": "universal-maker-darwin-arm64-1.0.0" + }, + "linux_x64": { + "version": "1.0.0", + "expectedSize": 16856277, + "expectedChecksumSHA256": "dfc8357b8ce23ef1897e5590d4390f166e27734f241b770f721d68273118845c", + "remoteUrl": "https://artifactregistry.googleapis.com/download/v1/projects/serverless-runtimes-qa/locations/us-central1/repositories/universal-maker/files/x86-64%3A1.0.0%3Auniversal_maker:download?alt=media", + "downloadPathRelativeToCacheDir": "universal-maker-linux-x64-1.0.0" + } +} diff --git a/src/downloadUtils.ts b/src/downloadUtils.ts index 9e311108785..a5e0c13b260 100644 --- a/src/downloadUtils.ts +++ b/src/downloadUtils.ts @@ -1,4 +1,5 @@ import { URL } from "url"; +import * as crypto from "crypto"; import * as fs from "fs-extra"; import * as ProgressBar from "progress"; import * as tmp from "tmp"; @@ -12,7 +13,7 @@ import { FirebaseError } from "./error"; * @param remoteUrl URL to download. * @param auth Whether to include an access token in the download request. Defaults to false. */ -export async function downloadToTmp(remoteUrl: string, auth: boolean = false): Promise { +export async function downloadToTmp(remoteUrl: string, auth = false): Promise { const u = new URL(remoteUrl); const c = new Client({ urlPrefix: u.origin, auth }); const tmpfile = tmp.fileSync(); @@ -47,3 +48,46 @@ export async function downloadToTmp(remoteUrl: string, auth: boolean = false): P return tmpfile.name; } + +/** + * Checks whether the file at `filepath` has the expected size. + */ +export function validateSize(filepath: string, expectedSize: number): Promise { + return new Promise((resolve, reject) => { + const stat = fs.statSync(filepath); + return stat.size === expectedSize + ? resolve() + : reject( + new FirebaseError( + `download failed, expected ${expectedSize} bytes but got ${stat.size}`, + { exit: 1 }, + ), + ); + }); +} + +/** + * Checks whether the file at `filepath` has the expected checksum. + */ +export function validateChecksum( + filepath: string, + expectedChecksum: string, + algorithm: "md5" | "sha256" = "md5", +): Promise { + return new Promise((resolve, reject) => { + const hash = crypto.createHash(algorithm); + const stream = fs.createReadStream(filepath); + stream.on("data", (data: any) => hash.update(data)); + stream.on("end", () => { + const checksum = hash.digest("hex"); + return checksum === expectedChecksum + ? resolve() + : reject( + new FirebaseError( + `download failed, expected checksum ${expectedChecksum} but got ${checksum}`, + { exit: 1 }, + ), + ); + }); + }); +} diff --git a/src/emulator/download.ts b/src/emulator/download.ts index 062f827a930..4d3dbaa1747 100644 --- a/src/emulator/download.ts +++ b/src/emulator/download.ts @@ -1,4 +1,3 @@ -import * as crypto from "crypto"; import * as fs from "fs-extra"; import * as path from "path"; import * as tmp from "tmp"; @@ -51,8 +50,8 @@ export async function downloadEmulator(name: DownloadableEmulators): Promise { - return new Promise((resolve, reject) => { - const stat = fs.statSync(filepath); - return stat.size === expectedSize - ? resolve() - : reject( - new FirebaseError( - `download failed, expected ${expectedSize} bytes but got ${stat.size}`, - { exit: 1 }, - ), - ); - }); -} - -/** - * Checks whether the file at `filepath` has the expected checksum. - */ -function validateChecksum(filepath: string, expectedChecksum: string): Promise { - return new Promise((resolve, reject) => { - const hash = crypto.createHash("md5"); - const stream = fs.createReadStream(filepath); - stream.on("data", (data: any) => hash.update(data)); - stream.on("end", () => { - const checksum = hash.digest("hex"); - return checksum === expectedChecksum - ? resolve() - : reject( - new FirebaseError( - `download failed, expected checksum ${expectedChecksum} but got ${checksum}`, - { exit: 1 }, - ), - ); - }); - }); -} diff --git a/src/experiments.ts b/src/experiments.ts index e8bbcdf3b2a..de1601ffb1e 100644 --- a/src/experiments.ts +++ b/src/experiments.ts @@ -148,6 +148,12 @@ export const ALL_EXPERIMENTS = experiments({ default: false, public: false, }, + universalMaker: { + shortDescription: "Opt-in to Universal Maker standalone binary local builds", + default: false, + public: false, + }, + abiu: { shortDescription: "Enable App Hosting ABIU and runtime selection", default: false,