Integrate FAH Local Builds with Universal Maker

src/apphosting/localbuilds.spec.ts

@@ -1,9 +1,11 @@
 import * as sinon from "sinon";
 import { expect } from "chai";
 import * as localBuildModule from "@apphosting/build";
-import { localBuild } from "./localbuilds";
+import { localBuild, runUniversalMaker } from "./localbuilds";
 import * as secrets from "./secrets";
 import { EnvMap } from "./yaml";
+import * as childProcess from "child_process";
+import * as fs from "fs";
 
 describe("localBuild", () => {
   afterEach(() => {
@@ -13,6 +15,7 @@ describe("localBuild", () => {
   it("returns the expected output", async () => {
     const bundleConfig = {
       version: "v1" as const,
+
       runConfig: {
         runCommand: "npm run build:prod",
       },
@@ -202,4 +205,69 @@ describe("localBuild", () => {
       expect(confirmStub).to.have.been.calledOnce;
     });
   });
+
+  describe("runUniversalMaker", () => {
+    it("should successfully execute Universal Maker and parse output", () => {
+      process.env.UNIVERSAL_MAKER_BINARY = "/path/to/universal_maker";
+      const spawnStub = sinon
+        .stub(childProcess, "spawnSync")
+        .returns({} as unknown as childProcess.SpawnSyncReturns<string>);
+      sinon.stub(fs, "existsSync").returns(true);
+      const readFileSyncStub = sinon.stub(fs, "readFileSync").returns(
+        JSON.stringify({
+          command: "npm",
+          args: ["run", "start"],
+          language: "nodejs",
+          runtime: "nodejs22",
+          envVars: { PORT: 3000 },
+        }),
+      );
+
+      const output = runUniversalMaker("./", "nextjs");
+
+      expect(output).to.deep.equal({
+        metadata: {
+          language: "nodejs",
+          runtime: "nodejs22",
+          framework: "nextjs",
+        },
+        runConfig: {
+          runCommand: "npm run start",
+          environmentVariables: [{ variable: "PORT", value: "3000", availability: ["RUNTIME"] }],
+        },
+        outputFiles: {
+          serverApp: {
+            include: [".apphosting"],
+          },
+        },
+      });
+
+      sinon.assert.calledOnce(spawnStub);
+      sinon.assert.calledOnce(readFileSyncStub);
+      delete process.env.UNIVERSAL_MAKER_BINARY;
+    });
+
+    it("should raise clear FirebaseError when UNIVERSAL_MAKER_BINARY is undefined", () => {
+      delete process.env.UNIVERSAL_MAKER_BINARY;
+
+      expect(() => runUniversalMaker("./")).to.throw(
+        "Please specify the path to your Universal Maker binary by establishing the UNIVERSAL_MAKER_BINARY environment variable.",
+      );
+    });
+
+    it("should raise clear FirebaseError on permission errors within child execution", () => {
+      process.env.UNIVERSAL_MAKER_BINARY = "/path/to/universal_maker";
+      sinon.stub(childProcess, "spawnSync").callsFake(() => {
+        const err = new Error("EACCES exception") as NodeJS.ErrnoException;
+        err.code = "EACCES";
+
+        throw err;
+      });
+
+      expect(() => runUniversalMaker("./")).to.throw(
+        "Failed to execute the Universal Maker binary due to permission constraints. Please assure you have set chmod +x on your file.",
+      );
+      delete process.env.UNIVERSAL_MAKER_BINARY;
+    });
+  });
 });

src/apphosting/localbuilds.ts

@@ -1,14 +1,163 @@
+import * as childProcess from "child_process";
+import * as fs from "fs";
+import * as path from "path";
 import { BuildConfig, Env } from "../gcp/apphosting";
 import { localBuild as localAppHostingBuild } from "@apphosting/build";
 import { EnvMap } from "./yaml";
 import { loadSecret } from "./secrets";
 import { confirm } from "../prompt";
 import { FirebaseError } from "../error";
+import * as experiments from "../experiments";
+import { logger } from "../logger";
+
+interface UniversalMakerOutput {
+  command: string;
+  args: string[];
+  language: string;
+  runtime: string;
+  envVars?: Record<string, string | number | boolean>;
+}
+
+/**
+ * Runs the Universal Maker binary to build the project.
+ */
+export function runUniversalMaker(projectRoot: string, framework?: string): AppHostingBuildOutput {
+  if (!process.env.UNIVERSAL_MAKER_BINARY) {
+    throw new FirebaseError(
+      "Please specify the path to your Universal Maker binary by establishing the UNIVERSAL_MAKER_BINARY environment variable.",
+    );
+  }
+
+  const bundleYamlPath = path.join(projectRoot, ".apphosting", "bundle.yaml");
+  let cachedBundleContent: string | null = null;
+
+  if (!fs.existsSync(path.dirname(bundleYamlPath))) {
+    fs.mkdirSync(path.dirname(bundleYamlPath), { recursive: true });
+  }
+
+  // Watch for the file being created by the Next.js adapter
+  const watcher = fs.watch(path.dirname(bundleYamlPath), (eventType, filename) => {
+    if (filename === "bundle.yaml") {
+      try {
+        if (fs.existsSync(bundleYamlPath)) {
+          const currentText = fs.readFileSync(bundleYamlPath, "utf-8");
+          if (currentText.trim().length > 0) {
+            cachedBundleContent = currentText;
+          }
+        }
+      } catch (e) {
+      }
+    }
+  });
+
+  try {
+    childProcess.spawnSync(
+      process.env.UNIVERSAL_MAKER_BINARY,
+      ["-application_dir", projectRoot, "-output_dir", projectRoot, "-output_format", "json"],
+      {
+        env: {
+          ...process.env,
+          X_GOOGLE_TARGET_PLATFORM: "fah",
+          FIREBASE_OUTPUT_BUNDLE_DIR: ".apphosting",
+          NPM_CONFIG_REGISTRY: "https://registry.npmjs.org/",
+        },
+        stdio: "inherit",
+      },
+    );
+
+    // Close the background watcher safely
+    watcher.close();
+
+    // Restore safely if wiped out in the final seconds
+    if (cachedBundleContent && fs.existsSync(bundleYamlPath)) {
+      const lastText = fs.readFileSync(bundleYamlPath, "utf-8");
+      if (lastText.trim().length === 0) {
+        fs.writeFileSync(bundleYamlPath, cachedBundleContent, "utf-8");
+      }
+    }
+  } catch (e) {
+    if (e && typeof e === "object" && "code" in e && e.code === "EACCES") {
+      throw new FirebaseError(
+        "Failed to execute the Universal Maker binary due to permission constraints. Please assure you have set chmod +x on your file.",
+      );
+    }
+    throw e;
+  }
+
+  const outputFilePath = path.join(projectRoot, "build_output.json");
+  if (!fs.existsSync(outputFilePath)) {
+    throw new FirebaseError(
+      `Universal Maker did not produce the expected output file at ${outputFilePath}`,
+    );
+  }
+
+  const outputRaw = fs.readFileSync(outputFilePath, "utf-8");
+  let umOutput: UniversalMakerOutput;
+  try {
+    umOutput = JSON.parse(outputRaw) as UniversalMakerOutput;
+  } catch (e) {
+    throw new FirebaseError(`Failed to parse build_output.json: ${(e as Error).message}`);
+  }
+
+  let finalRunCommand = `${umOutput.command} ${umOutput.args.join(" ")}`;
+  if (fs.existsSync(bundleYamlPath)) {
+    try {
+      const bundleRaw = fs.readFileSync(bundleYamlPath, "utf-8");
+      // Safely parse the YAML string
+      const bundleData = require("yaml").parse(bundleRaw);
+      if (bundleData?.runConfig?.runCommand) {
+        finalRunCommand = bundleData.runConfig.runCommand;
+      }
+    } catch (e) {
+      // Fall back gracefully if parser fails
+    }
+  }
+
+  return {
+    metadata: {
+      language: umOutput.language,
+      runtime: umOutput.runtime,
+      framework: framework || "nextjs",
+    },
+    runConfig: {
+      runCommand: finalRunCommand,
+      environmentVariables: Object.entries(umOutput.envVars || {}).map(([k, v]) => ({
+        variable: k,
+        value: String(v),
+        availability: ["RUNTIME"],
+      })),
+    },
+    outputFiles: {
+      serverApp: {
+        include: [".apphosting"],
+      },
+    },
+  };
+}
+
+export interface AppHostingBuildOutput {
+  metadata: Record<string, string | number | boolean>;
+
+  runConfig: {
+    runCommand?: string;
+    environmentVariables?: Array<{
+      variable: string;
+      value: string;
+      availability: string[];
+    }>;
+  };
+  outputFiles?: {
+    serverApp: {
+      include: string[];
+    };
+  };
+}
 
 /**
  * Triggers a local build of your App Hosting codebase.
  *
  * This function orchestrates the build process using the App Hosting build adapter.
+ *
  * It detects the framework (though currently defaults/assumes 'nextjs' in some contexts),
  * generates the necessary build artifacts, and returns metadata about the build.
  * @param projectId - The project ID to use for resolving secrets.
@@ -62,9 +211,19 @@ export async function localBuild(
     process.env[key] = value;
   }
 
-  let apphostingBuildOutput;
+  let apphostingBuildOutput: AppHostingBuildOutput;
   try {
-    apphostingBuildOutput = await localAppHostingBuild(projectRoot, framework);
+    if (experiments.isEnabled("universalMaker")) {
+      apphostingBuildOutput = runUniversalMaker(projectRoot, framework);
+      logger.debug(
+        `[apphosting] Universal Maker build outputFiles include: ${JSON.stringify(apphostingBuildOutput.outputFiles?.serverApp?.include ?? [])}`,
+      );
+    } else {
+      apphostingBuildOutput = (await localAppHostingBuild(
+        projectRoot,
+        framework,
+      )) as unknown as AppHostingBuildOutput;
+    }
   } finally {
     for (const key in process.env) {
       if (!(key in originalEnv)) {
@@ -87,7 +246,7 @@ export async function localBuild(
         value,
         availability,
       }),
-    );
+    ) as unknown as Env[] | undefined;
 
   return {
     outputFiles: apphostingBuildOutput.outputFiles?.serverApp.include ?? [],

src/deploy/apphosting/util.ts

@@ -27,47 +27,93 @@ export async function createLocalBuildTarArchive(
 ): Promise<string> {
   const tmpFile = tmp.fileSync({ prefix: `${config.backendId}-`, postfix: ".tar.gz" }).name;
 
-  const targetDir = targetSubDir ? path.join(rootDir, targetSubDir) : rootDir;
+  const isAppHostingDir =
+    targetSubDir === ".apphosting" ||
+    (!!targetSubDir && path.basename(targetSubDir) === ".apphosting");
+  const targetDir = targetSubDir
+    ? path.isAbsolute(targetSubDir)
+      ? targetSubDir
+      : path.join(rootDir, targetSubDir)
+    : rootDir;
   const ignore = ["firebase-debug.log", "firebase-debug.*.log", ".git"];
-  const rdrFiles = await fsAsync.readdirRecursive({
-    path: targetDir,
-    ignore: ignore,
-    isGitIgnore: true,
-  });
-  const allFiles: string[] = rdrFiles.map((rdrf) => path.relative(rootDir, rdrf.name));
 
-  if (targetSubDir) {
-    const defaultFiles = fs.readdirSync(rootDir).filter((file) => {
-      return APPHOSTING_YAML_FILE_REGEX.test(file);
+  let archiveCwd = rootDir;
+  let pathsToPack: string[];
+
+  if (isAppHostingDir) {
+    // create temporary directory to bundle things flattened
+    const tempDir = tmp.dirSync({ unsafeCleanup: true }).name;
+    fs.cpSync(targetDir, tempDir, { recursive: true });
+
+    const rootPackageJson = path.join(rootDir, "package.json");
+    if (fs.existsSync(rootPackageJson)) {
+      fs.copyFileSync(rootPackageJson, path.join(tempDir, "package.json"));
+    }
+    const rootFiles = fs.readdirSync(rootDir);
+    for (const file of rootFiles) {
+      if (APPHOSTING_YAML_FILE_REGEX.test(file)) {
+        fs.copyFileSync(path.join(rootDir, file), path.join(tempDir, file));
+      }
+    }
+    const rootNext = path.join(rootDir, ".next");
+    if (fs.existsSync(rootNext)) {
+      fs.cpSync(rootNext, path.join(tempDir, ".next"), { recursive: true });
+    }
+    const rootNodeModules = path.join(rootDir, "node_modules");
+    if (fs.existsSync(rootNodeModules)) {
+      fs.cpSync(rootNodeModules, path.join(tempDir, "node_modules"), { recursive: true });
+    }
+
+    const rdrFiles = await fsAsync.readdirRecursive({
+      path: tempDir,
+      ignore: ignore,
+      isGitIgnore: false,
+    });
+    pathsToPack = rdrFiles.map((rdrf) => path.relative(tempDir, rdrf.name));
+    archiveCwd = tempDir;
+  } else {
+    const rdrFiles = await fsAsync.readdirRecursive({
+      path: targetDir,
+      ignore: ignore,
+      isGitIgnore: !targetSubDir, // Disable gitignore if we are anchored to a build output subdirectory
     });
-    for (const file of defaultFiles) {
-      if (!allFiles.includes(file)) {
-        allFiles.push(file);
+    pathsToPack = rdrFiles.map((rdrf) => path.relative(rootDir, rdrf.name));
+
+    if (targetSubDir) {
+      const defaultFiles = fs.readdirSync(rootDir).filter((file) => {
+        return APPHOSTING_YAML_FILE_REGEX.test(file);
+      });
+      for (const file of defaultFiles) {
+        const relativePath = path.relative(rootDir, path.join(rootDir, file));
+        if (!pathsToPack.includes(relativePath)) {
+          pathsToPack.push(relativePath);
+        }
       }
     }
   }
 
-  // `tar` returns a `TypeError` if `allFiles` is empty. Let's check a feww things.
   try {
-    fs.statSync(rootDir);
+    fs.statSync(archiveCwd);
   } catch (err: unknown) {
     if (err instanceof Error && "code" in err && err.code === "ENOENT") {
-      throw new FirebaseError(`Could not read directory "${rootDir}"`);
+      throw new FirebaseError(`Could not read directory "${archiveCwd}"`);
     }
     throw err;
   }
-  if (!allFiles.length) {
-    throw new FirebaseError(`Cannot create a tar archive with 0 files from directory "${rootDir}"`);
+  if (!pathsToPack.length) {
+    throw new FirebaseError(
+      `Cannot create a tar archive with 0 files from directory "${archiveCwd}"`,
+    );
   }
 
   await tar.create(
     {
       gzip: true,
       file: tmpFile,
-      cwd: rootDir,
+      cwd: archiveCwd,
       portable: true,
     },
-    allFiles,
+    pathsToPack,
   );
   return tmpFile;
 }