ci: add ci.1focus host runner mode and tighten workflow performance

Author: nikivdev

.github/workflows/canary.yml

@@ -18,6 +18,7 @@ env:
 
 jobs:
   build:
+    timeout-minutes: 40
     env:
       # Don't reference `secrets.*` in `if:` expressions; GitHub rejects that at workflow-parse time.
       MACOS_SIGN_P12_B64: ${{ secrets.MACOS_SIGN_P12_B64 }}
@@ -45,6 +46,12 @@ jobs:
         with:
           targets: ${{ matrix.target }}
 
+      - name: Cache Rust build
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: ". -> target"
+          cache-on-failure: true
+
       - name: Install cross-compilation tools (Linux ARM)
         if: matrix.target == 'aarch64-unknown-linux-gnu'
         run: |
@@ -88,6 +95,7 @@ jobs:
 
   build-linux-host-simd:
     if: ${{ false }}
+    timeout-minutes: 40
     runs-on: ubuntu-latest
     env:
       RUSTFLAGS: -C target-cpu=native
@@ -100,6 +108,12 @@ jobs:
         with:
           targets: x86_64-unknown-linux-gnu
 
+      - name: Cache Rust build
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: ". -> target"
+          cache-on-failure: true
+
       - name: Build (Linux host SIMD)
         run: cargo build --release --target x86_64-unknown-linux-gnu --features linux-host-simd-json
 
@@ -119,6 +133,7 @@ jobs:
 
   release:
     needs: build
+    timeout-minutes: 20
     runs-on: ubuntu-latest
     steps:
       - name: Checkout

.github/workflows/release.yml

@@ -13,6 +13,7 @@ env:
 
 jobs:
   build:
+    timeout-minutes: 40
     env:
       # Don't reference `secrets.*` in `if:` expressions; GitHub rejects that at workflow-parse time.
       MACOS_SIGN_P12_B64: ${{ secrets.MACOS_SIGN_P12_B64 }}
@@ -40,6 +41,12 @@ jobs:
         with:
           targets: ${{ matrix.target }}
 
+      - name: Cache Rust build
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: ". -> target"
+          cache-on-failure: true
+
       - name: Install cross-compilation tools (Linux ARM)
         if: matrix.target == 'aarch64-unknown-linux-gnu'
         run: |
@@ -83,6 +90,7 @@ jobs:
 
   build-linux-host-simd:
     if: ${{ false }}
+    timeout-minutes: 40
     runs-on: ubuntu-latest
     env:
       RUSTFLAGS: -C target-cpu=native
@@ -95,6 +103,12 @@ jobs:
         with:
           targets: x86_64-unknown-linux-gnu
 
+      - name: Cache Rust build
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: ". -> target"
+          cache-on-failure: true
+
       - name: Build (Linux host SIMD)
         run: cargo build --release --target x86_64-unknown-linux-gnu --features linux-host-simd-json
 
@@ -114,6 +128,7 @@ jobs:
 
   release:
     needs: build
+    timeout-minutes: 20
     runs-on: ubuntu-latest
     steps:
       - name: Checkout

flow.toml

@@ -126,7 +126,7 @@ description = "Upload the latest darwin/arm64 release tarball to the release hos
 [[tasks]]
 name = "ci-blacksmith-status"
 command = "python3 ./scripts/ci_blacksmith.py status"
-description = "Show CI runner mode for canary/release workflows (github or blacksmith)"
+description = "Show CI runner mode for canary/release workflows (github, blacksmith, or host)"
 
 [[tasks]]
 name = "ci-blacksmith-enable"
@@ -148,6 +148,41 @@ name = "ci-blacksmith-disable-apply"
 command = "python3 ./scripts/ci_blacksmith.py disable --commit --push"
 description = "Disable Blacksmith CI mode, commit workflow updates, and push"
 
+[[tasks]]
+name = "ci-host-enable"
+command = "python3 ./scripts/ci_blacksmith.py host"
+description = "Keep standard Linux CI lanes on GitHub runners, but enable Linux host SIMD build on ci-1focus self-hosted runner"
+
+[[tasks]]
+name = "ci-host-enable-apply"
+command = "python3 ./scripts/ci_blacksmith.py host --commit --push"
+description = "Enable host SIMD CI mode, commit workflow updates, and push"
+
+[[tasks]]
+name = "ci-host-runner-status"
+command = "python3 ./scripts/ci_host_runner.py status --repo nikivdev/flow"
+description = "Show ci-1focus runner service status on infra host and GitHub runner registration state"
+
+[[tasks]]
+name = "ci-host-runner-install"
+command = "python3 ./scripts/ci_host_runner.py install --repo nikivdev/flow"
+description = "Install/register self-hosted GitHub runner on configured infra Linux host (label: ci-1focus)"
+
+[[tasks]]
+name = "ci-host-runner-remove"
+command = "python3 ./scripts/ci_host_runner.py remove --repo nikivdev/flow"
+description = "Unregister ci-1focus self-hosted GitHub runner and remove runner service on infra host"
+
+[[tasks]]
+name = "ci-host-bootstrap"
+command = "python3 ./scripts/ci_host_runner.py install --repo nikivdev/flow && python3 ./scripts/ci_blacksmith.py host"
+description = "One-command setup: install ci-1focus runner on infra host and switch workflows to host SIMD mode"
+
+[[tasks]]
+name = "ci-host-bootstrap-apply"
+command = "python3 ./scripts/ci_host_runner.py install --repo nikivdev/flow && python3 ./scripts/ci_blacksmith.py host --commit --push"
+description = "Install ci-1focus runner, switch workflows to host SIMD mode, then commit and push workflow updates"
+
 [[tasks]]
 name = "test-flow-task-tracing"
 command = "bun i-run.ts"

readme.md

@@ -87,20 +87,28 @@ Stable releases are cut by pushing a git tag that starts with `v`:
 Canary releases are published automatically on every push to `main` via
 `.github/workflows/canary.yml` (GitHub release tag: `canary`).
 
-Optional Linux host SIMD lane:
-
-- Flow has first-class tasks for CI runner mode (no GitHub env/repo vars needed):
-  - `f ci-blacksmith-status`
-  - `f ci-blacksmith-enable`
-  - `f ci-blacksmith-enable-apply`
-  - `f ci-blacksmith-disable`
-  - `f ci-blacksmith-disable-apply`
-- `ci-blacksmith-enable` switches Linux jobs to Blacksmith runners and enables
-  the Linux host SIMD build job (`--features linux-host-simd-json` with
-  `RUSTFLAGS=-C target-cpu=native`).
-- `ci-blacksmith-disable` reverts to GitHub-hosted Linux runners and keeps the
-  SIMD lane disabled by default for reliability.
-- `*-apply` variants also commit and push workflow changes in one command.
+CI runner modes (no workflow env var toggles):
+
+- `f ci-blacksmith-status` shows the active mode (`github`, `blacksmith`, or `host`).
+- `f ci-blacksmith-enable` / `f ci-blacksmith-enable-apply` switch Linux lanes to Blacksmith runners and enable the SIMD lane.
+- `f ci-host-enable` / `f ci-host-enable-apply` keep normal Linux lanes on GitHub-hosted runners, but run the SIMD lane on `ci-1focus` self-hosted runner.
+- `f ci-blacksmith-disable` / `f ci-blacksmith-disable-apply` revert to default GitHub mode with SIMD lane disabled.
+
+Set up `ci.1focus.ai` host runner (painless path):
+
+1. Point infra to your Linux host: `infra host set <user@ip>`
+2. Make sure GitHub CLI auth is ready: `gh auth status`
+3. Install/register runner on host: `f ci-host-runner-install`
+4. Verify runner service + registration: `f ci-host-runner-status`
+5. Switch workflows to host SIMD mode: `f ci-host-enable-apply`
+
+Or run all in one command:
+
+- `f ci-host-bootstrap-apply`
+
+Notes:
+
+- Host mode is a compile-time balance: expensive SIMD builds run on your Linux host, while default Linux matrix jobs stay on GitHub-hosted runners for throughput.
 - Blacksmith setup and runner tags:
   - https://docs.blacksmith.sh/github-actions/quickstart
   - https://docs.blacksmith.sh/github-actions/runner-tags

scripts/ci_blacksmith.py

@@ -1,11 +1,17 @@
 #!/usr/bin/env python3
 """
-Toggle CI workflows between GitHub-hosted and Blacksmith runner profiles.
+Toggle CI workflows between runner profiles.
+
+Profiles:
+  - github: Default GitHub-hosted Linux jobs, SIMD lane disabled.
+  - blacksmith: Blacksmith Linux jobs, SIMD lane enabled on Blacksmith.
+  - host: GitHub Linux jobs, SIMD lane enabled on ci.1focus.ai self-hosted runner.
 
 Usage:
   python3 scripts/ci_blacksmith.py status
   python3 scripts/ci_blacksmith.py enable
   python3 scripts/ci_blacksmith.py enable --commit --push
+  python3 scripts/ci_blacksmith.py host
   python3 scripts/ci_blacksmith.py disable
 """
 
@@ -28,21 +34,33 @@
 BLACKSMITH_X64 = "blacksmith-2vcpu-ubuntu-2404"
 BLACKSMITH_ARM = "blacksmith-2vcpu-ubuntu-2404-arm"
 BLACKSMITH_SIMD = "blacksmith-4vcpu-ubuntu-2404"
+HOST_SIMD = "[self-hosted, linux, x64, ci-1focus]"
 
 WORKFLOW_REL_PATHS = [
     ".github/workflows/canary.yml",
     ".github/workflows/release.yml",
 ]
 
 
-def rewrite_workflow(path: Path, enable: bool) -> bool:
+def rewrite_workflow(path: Path, mode: str) -> bool:
     content = path.read_text(encoding="utf-8")
     original = content
 
-    linux_x64 = BLACKSMITH_X64 if enable else GITHUB_X64
-    linux_arm = BLACKSMITH_ARM if enable else GITHUB_ARM
-    simd_runs_on = BLACKSMITH_SIMD if enable else "ubuntu-latest"
-    simd_if_line = "" if enable else "    if: ${{ false }}\n"
+    if mode == "blacksmith":
+        linux_x64 = BLACKSMITH_X64
+        linux_arm = BLACKSMITH_ARM
+        simd_runs_on = BLACKSMITH_SIMD
+        simd_if_line = ""
+    elif mode == "host":
+        linux_x64 = GITHUB_X64
+        linux_arm = GITHUB_ARM
+        simd_runs_on = HOST_SIMD
+        simd_if_line = ""
+    else:
+        linux_x64 = GITHUB_X64
+        linux_arm = GITHUB_ARM
+        simd_runs_on = "ubuntu-latest"
+        simd_if_line = "    if: ${{ false }}\n"
 
     content = re.sub(
         r"(- target: x86_64-unknown-linux-gnu\s*\n\s*os:\s*)([^\n]+)",
@@ -57,12 +75,35 @@ def rewrite_workflow(path: Path, enable: bool) -> bool:
         count=1,
     )
 
-    content = re.sub(
-        r"(  build-linux-host-simd:\n)(?:\s+if:.*\n)?(\s+runs-on:.*\n)",
-        rf"\1{simd_if_line}    runs-on: {simd_runs_on}\n",
-        content,
-        count=1,
+    simd_block = re.compile(
+        r"(^  build-linux-host-simd:\n)(?P<body>(?:^    .*\n)*)",
+        re.MULTILINE,
     )
+    block_match = simd_block.search(content)
+    if block_match:
+        body_lines = block_match.group("body").splitlines(keepends=True)
+        rewritten_body: list[str] = []
+        has_runs_on = False
+        for line in body_lines:
+            if re.match(r"^    if:", line):
+                continue
+            if re.match(r"^    runs-on:", line):
+                rewritten_body.append(f"    runs-on: {simd_runs_on}\n")
+                has_runs_on = True
+                continue
+            rewritten_body.append(line)
+
+        if not has_runs_on:
+            rewritten_body.insert(0, f"    runs-on: {simd_runs_on}\n")
+        if simd_if_line:
+            rewritten_body.insert(0, simd_if_line)
+
+        replacement = block_match.group(1) + "".join(rewritten_body)
+        content = (
+            content[: block_match.start()]
+            + replacement
+            + content[block_match.end() :]
+        )
 
     changed = content != original
     if changed:
@@ -74,6 +115,8 @@ def detect_profile(path: Path) -> str:
     content = path.read_text(encoding="utf-8")
     if BLACKSMITH_X64 in content and BLACKSMITH_ARM in content:
         return "blacksmith"
+    if HOST_SIMD in content and "if: ${{ false }}" not in content:
+        return "host"
     if GITHUB_X64 in content and "blacksmith-" not in content:
         return "github"
     if GITHUB_X64 in content and GITHUB_ARM in content:
@@ -130,19 +173,18 @@ def maybe_commit_and_push(mode: str, commit: bool, push: bool) -> int:
     return 0
 
 
-def set_mode(enable: bool, commit: bool, push: bool) -> int:
+def set_mode(mode: str, commit: bool, push: bool) -> int:
     if push and not commit:
         print("--push requires --commit", file=sys.stderr)
         return 2
 
     changed_any = False
     for wf in WORKFLOWS:
-        changed = rewrite_workflow(wf, enable)
+        changed = rewrite_workflow(wf, mode=mode)
         changed_any = changed_any or changed
         state = "updated" if changed else "unchanged"
         print(f"{wf.relative_to(ROOT)}: {state}")
 
-    mode = "blacksmith" if enable else "github"
     if changed_any:
         print(f"CI runner mode set to: {mode}")
     else:
@@ -154,8 +196,8 @@ def main() -> int:
     parser = argparse.ArgumentParser(description="Manage CI runner profile.")
     parser.add_argument(
         "command",
-        choices=["status", "enable", "disable"],
-        help="status | enable (Blacksmith) | disable (GitHub-hosted)",
+        choices=["status", "enable", "disable", "host"],
+        help="status | enable (Blacksmith) | host (self-hosted SIMD lane) | disable (GitHub-hosted)",
     )
     parser.add_argument(
         "--commit",
@@ -172,9 +214,11 @@ def main() -> int:
     if args.command == "status":
         return status()
     if args.command == "enable":
-        return set_mode(enable=True, commit=args.commit, push=args.push)
+        return set_mode(mode="blacksmith", commit=args.commit, push=args.push)
+    if args.command == "host":
+        return set_mode(mode="host", commit=args.commit, push=args.push)
     if args.command == "disable":
-        return set_mode(enable=False, commit=args.commit, push=args.push)
+        return set_mode(mode="github", commit=args.commit, push=args.push)
     return 2