From 5f73bff539cd94c9c8336bf95be4ffb7ccd4fff3 Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Mon, 9 Nov 2026 00:25:48 -0800 Subject: [PATCH] fixing ci failures --- .github/actions/setup-node/action.yml | 42 +++++++++ .github/maturin-curl/.curlrc | 9 ++ .github/workflows/node-sdk.yml | 32 ++++--- .github/workflows/python-sdk.yml | 9 +- scripts/tests/test_node_sdk_release.py | 74 +++++++++++++++ scripts/tests/test_python_sdk_release.py | 110 +++++++++++++++++++++++ 6 files changed, 261 insertions(+), 15 deletions(-) create mode 100644 .github/actions/setup-node/action.yml create mode 100644 .github/maturin-curl/.curlrc diff --git a/.github/actions/setup-node/action.yml b/.github/actions/setup-node/action.yml new file mode 100644 index 00000000..6ddc5930 --- /dev/null +++ b/.github/actions/setup-node/action.yml @@ -0,0 +1,42 @@ +name: Set up Node with safe Windows ARM64 extraction +description: Keep the pinned installer while avoiding the hosted ARM64 PowerShell 7 ZIP crash. +inputs: + node-version: + description: Exact Node version to install. + required: true + cache: + description: Package manager cache passed to setup-node. + cache-dependency-path: + description: Dependency lockfile paths passed to setup-node. + package-manager-cache: + description: Enable setup-node's automatic package manager cache. + default: 'true' +runs: + using: composite + steps: + - name: Select the Windows ARM64 archive extractor + if: runner.os == 'Windows' && runner.arch == 'ARM64' + id: extractor + shell: powershell + run: | + $ErrorActionPreference = 'Stop' + # Node-spawned pwsh can abort before tool-cache's ZIP fallback runs: + # https://github.com/actions/runner-images/issues/15107 + # Hide every pwsh directory only from this setup-node invocation. The + # existing tool-cache fallback then uses Windows PowerShell instead. + Get-Command powershell.exe -CommandType Application -ErrorAction Stop | Out-Null + $path = (($env:PATH -split ';') | Where-Object { + $directory = $_.Trim().Trim('"') + $directory -and -not (Test-Path -LiteralPath (Join-Path $directory 'pwsh.exe')) + }) -join ';' + $overrides = @{ PATH = $path } + $json = ConvertTo-Json -InputObject $overrides -Compress + "env=$json" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append + - name: Set up the pinned Node runtime + uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 + env: ${{ fromJSON(steps.extractor.outputs.env || '{}') }} + with: + node-version: ${{ inputs.node-version }} + cache: ${{ inputs.cache }} + cache-dependency-path: ${{ inputs.cache-dependency-path }} + package-manager-cache: ${{ inputs.package-manager-cache }} diff --git a/.github/maturin-curl/.curlrc b/.github/maturin-curl/.curlrc new file mode 100644 index 00000000..19e71950 --- /dev/null +++ b/.github/maturin-curl/.curlrc @@ -0,0 +1,9 @@ +# Used only by the Python SDK's manylinux containers, via explicit CURL_HOME. +# Do not pass HTTP error pages to the action's curl | tar installer. +fail +retry = 4 +retry-delay = 2 +retry-max-time = 120 +retry-connrefused +connect-timeout = 30 +max-time = 180 diff --git a/.github/workflows/node-sdk.yml b/.github/workflows/node-sdk.yml index 3e39dec3..40081eb2 100644 --- a/.github/workflows/node-sdk.yml +++ b/.github/workflows/node-sdk.yml @@ -4,9 +4,9 @@ on: push: branches: [main, development] tags: ['npm-v*'] - paths: ['javascript/**', 'crates/**', 'Cargo.toml', 'Cargo.lock', 'LICENSE', 'NOTICE', 'THIRD_PARTY_NOTICES', 'third-party/kingfisher-vectorscan/**', 'scripts/third-party-notices.cjs', 'scripts/tests/third-party-notices.test.cjs', 'scripts/license-overrides/**', 'Makefile', '.gitattributes', 'docs/JAVASCRIPT.md', 'scripts/tests/test_node_sdk*.py', '.github/actions/vectorscan-prebuilt/**', 'scripts/fetch-vectorscan.py', 'scripts/tests/test_fetch_vectorscan.py', '.github/workflows/node-sdk.yml'] + paths: ['javascript/**', 'crates/**', 'Cargo.toml', 'Cargo.lock', 'LICENSE', 'NOTICE', 'THIRD_PARTY_NOTICES', 'third-party/kingfisher-vectorscan/**', 'scripts/third-party-notices.cjs', 'scripts/tests/third-party-notices.test.cjs', 'scripts/license-overrides/**', 'Makefile', '.gitattributes', 'docs/JAVASCRIPT.md', 'scripts/tests/test_node_sdk*.py', '.github/actions/setup-node/**', '.github/actions/vectorscan-prebuilt/**', 'scripts/fetch-vectorscan.py', 'scripts/tests/test_fetch_vectorscan.py', '.github/workflows/node-sdk.yml'] pull_request: - paths: ['javascript/**', 'crates/**', 'Cargo.toml', 'Cargo.lock', 'LICENSE', 'NOTICE', 'THIRD_PARTY_NOTICES', 'third-party/kingfisher-vectorscan/**', 'scripts/third-party-notices.cjs', 'scripts/tests/third-party-notices.test.cjs', 'scripts/license-overrides/**', 'Makefile', '.gitattributes', 'docs/JAVASCRIPT.md', 'scripts/tests/test_node_sdk*.py', '.github/actions/vectorscan-prebuilt/**', 'scripts/fetch-vectorscan.py', 'scripts/tests/test_fetch_vectorscan.py', '.github/workflows/node-sdk.yml'] + paths: ['javascript/**', 'crates/**', 'Cargo.toml', 'Cargo.lock', 'LICENSE', 'NOTICE', 'THIRD_PARTY_NOTICES', 'third-party/kingfisher-vectorscan/**', 'scripts/third-party-notices.cjs', 'scripts/tests/third-party-notices.test.cjs', 'scripts/license-overrides/**', 'Makefile', '.gitattributes', 'docs/JAVASCRIPT.md', 'scripts/tests/test_node_sdk*.py', '.github/actions/setup-node/**', '.github/actions/vectorscan-prebuilt/**', 'scripts/fetch-vectorscan.py', 'scripts/tests/test_fetch_vectorscan.py', '.github/workflows/node-sdk.yml'] workflow_dispatch: permissions: @@ -83,7 +83,7 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: { persist-credentials: false } - - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 + - uses: ./.github/actions/setup-node with: node-version: '22.23.4' cache: npm @@ -122,6 +122,10 @@ jobs: - uses: ./.github/actions/vectorscan-prebuilt with: target: ${{ matrix.target }} + - name: Check the Windows Node archive selector + if: runner.os == 'Windows' + working-directory: . + run: python -X utf8 -m unittest discover -s scripts/tests -p test_node_sdk_release.py -v - name: Rust formatting and binding lint run: | cargo fmt --all --check @@ -142,31 +146,31 @@ jobs: run: | npm test npm run test:types - - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 + - uses: ./.github/actions/setup-node with: { node-version: '24.22.2' } - name: Node 24 tests against the same native artifact run: | npm test npm run test:types - - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 + - uses: ./.github/actions/setup-node with: { node-version: '22.13.0' } - name: Minimum Node 22 runtime against the same native artifact run: | npm test npm run test:types - - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 + - uses: ./.github/actions/setup-node with: { node-version: '24.0.0' } - name: Minimum Node 24 runtime against the same native artifact run: | npm test npm run test:types - - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 + - uses: ./.github/actions/setup-node with: { node-version: '26.0.0' } - name: Minimum Node 26 runtime against the same native artifact run: | npm test npm run test:types - - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 + - uses: ./.github/actions/setup-node with: { node-version: '26.13.0' } - name: Node 26 integration and TypeScript API against the same native artifact run: | @@ -230,27 +234,27 @@ jobs: with: name: node-sdk-registry-staging path: javascript/release - - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 + - uses: ./.github/actions/setup-node with: { node-version: '22.23.4' } - name: Clean offline Node 22 consumer of the exact publish archives run: npm run test:package - - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 + - uses: ./.github/actions/setup-node with: { node-version: '24.22.2' } - name: Clean offline Node 24 consumer of the same archives run: npm run test:package - - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 + - uses: ./.github/actions/setup-node with: { node-version: '22.13.0' } - name: Clean offline minimum Node 22 consumer of the same archives run: npm run test:package - - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 + - uses: ./.github/actions/setup-node with: { node-version: '24.0.0' } - name: Clean offline minimum Node 24 consumer of the same archives run: npm run test:package - - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 + - uses: ./.github/actions/setup-node with: { node-version: '26.0.0' } - name: Clean offline minimum Node 26 consumer of the same archives run: npm run test:package - - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 + - uses: ./.github/actions/setup-node with: { node-version: '26.13.0' } - name: Clean offline Node 26 consumer of the same archives run: npm run test:package diff --git a/.github/workflows/python-sdk.yml b/.github/workflows/python-sdk.yml index 52a94641..ed446f4f 100644 --- a/.github/workflows/python-sdk.yml +++ b/.github/workflows/python-sdk.yml @@ -122,7 +122,14 @@ jobs: # Published Vectorscan archives require glibc 2.35. Build from the # vendored source inside manylinux to keep the advertised 2.28 floor. before-script-linux: yum install -y boost-devel ragel perl-core - docker-options: -e VECTORSCAN_BUILD_FROM_SOURCE=1 -e CARGO_PROFILE_RELEASE_PANIC=unwind -e CARGO_PROFILE_RELEASE_LTO=false + # The action downloads Maturin before before-script-linux runs. Its + # curl | tar installer needs HTTP-error handling and bounded retries; + # the checkout is already mounted at github.workspace in the container. + docker-options: >- + -e VECTORSCAN_BUILD_FROM_SOURCE=1 + -e CARGO_PROFILE_RELEASE_PANIC=unwind + -e CARGO_PROFILE_RELEASE_LTO=false + -e CURL_HOME="${{ github.workspace }}/.github/maturin-curl" # manylinux builds Vectorscan from source to preserve the glibc 2.28 floor. - uses: ./.github/actions/vectorscan-prebuilt if: runner.os != 'Linux' diff --git a/scripts/tests/test_node_sdk_release.py b/scripts/tests/test_node_sdk_release.py index 6b8fed81..1ec5b9d2 100644 --- a/scripts/tests/test_node_sdk_release.py +++ b/scripts/tests/test_node_sdk_release.py @@ -1,6 +1,12 @@ """Check native SDK release guards without GitHub or npm registry writes.""" from pathlib import Path +import json +import os import re +import shutil +import subprocess +import tempfile +import textwrap import tomllib import unittest @@ -114,6 +120,74 @@ def test_release_native_runtime_baselines_are_explicit_and_audited(self): self.assertIn("npm test", minimum) self.assertIn("npm run test:types", minimum) + def test_all_native_and_sealed_runtime_installs_use_the_safe_wrapper(self): + workflow = (ROOT / ".github/workflows/node-sdk.yml").read_text() + for name in ("build", "verify"): + with self.subTest(job=name): + body = job(workflow, name) + self.assertEqual(body.count("uses: ./.github/actions/setup-node"), 6) + self.assertNotIn("uses: actions/setup-node@", body) + self.assertEqual(re.findall(r"node-version: '([^']+)'", body), [ + "22.23.4", "24.22.2", "22.13.0", "24.0.0", "26.0.0", "26.13.0", + ]) + for name in ("source", "package", "publish"): + self.assertIn("actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38", + job(workflow, name)) + self.assertIn("cache: npm", job(workflow, "build")) + self.assertIn("cache-dependency-path: javascript/package-lock.json", job(workflow, "build")) + self.assertIn("if: runner.os == 'Windows'\n working-directory: .\n" + " run: python -X utf8 -m unittest discover -s scripts/tests " + "-p test_node_sdk_release.py -v", job(workflow, "build")) + triggers = workflow.split("permissions:", 1)[0] + self.assertEqual(triggers.count("'.github/actions/setup-node/**'"), 2) + + def test_archive_workaround_is_scoped_to_each_arm64_windows_installer(self): + action = (ROOT / ".github/actions/setup-node/action.yml").read_text() + selector, installer = action.split(" - name: Set up the pinned Node runtime\n", 1) + self.assertIn("if: runner.os == 'Windows' && runner.arch == 'ARM64'", selector) + self.assertIn("shell: powershell", selector) + self.assertIn("Get-Command powershell.exe -CommandType Application -ErrorAction Stop", selector) + self.assertIn("$_.Trim().Trim", selector) + self.assertIn("Test-Path -LiteralPath (Join-Path $directory 'pwsh.exe')", selector) + self.assertNotIn("GITHUB_ENV", action) + self.assertNotIn("GITHUB_PATH", action) + self.assertNotIn("continue-on-error", action) + self.assertIn("uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38", installer) + self.assertIn("env: ${{ fromJSON(steps.extractor.outputs.env || '{}') }}", installer) + for name in ("node-version", "cache", "cache-dependency-path", "package-manager-cache"): + self.assertIn(f"{name}: ${{{{ inputs.{name} }}}}", installer) + self.assertIn("default: 'true'", selector) + + @unittest.skipUnless(os.name == "nt", "The archive selector uses Windows PowerShell") + def test_windows_selector_preserves_compiler_and_node_paths_and_hides_all_pwsh(self): + powershell = shutil.which("powershell") + self.assertIsNotNone(powershell, "Windows PowerShell is required by tool-cache's fallback") + action = (ROOT / ".github/actions/setup-node/action.yml").read_text() + script = textwrap.dedent(action.split(" run: |\n", 1)[1].split(" - name:", 1)[0]) + with tempfile.TemporaryDirectory(prefix="kingfisher Node extractor ") as temporary: + root = Path(temporary) + compiler, node = root / "VS 14.44", root / "Node 22" + pwsh_directories = [root / "PowerShell 7", root / "Other PowerShell"] + for directory in (compiler, node, *pwsh_directories): + directory.mkdir() + for directory in pwsh_directories: + (directory / "pwsh.exe").write_bytes(b"fixture, never executed") + retained = [str(Path(powershell).parent), str(compiler), str(node)] + hidden = [str(pwsh_directories[0]) + "\\", str(pwsh_directories[0]).upper(), + '"' + str(pwsh_directories[1]) + '\\"'] + for index, entries in enumerate((retained, retained + hidden)): + with self.subTest(pwsh_present=bool(index)): + output = root / f"output-{index}.txt" + result = subprocess.run([ + powershell, "-NoLogo", "-NoProfile", "-NonInteractive", + "-ExecutionPolicy", "Bypass", "-Command", script, + ], env={**os.environ, "PATH": ";".join(entries), "GITHUB_OUTPUT": str(output)}, + capture_output=True, timeout=30, check=False) + self.assertEqual(result.returncode, 0, result.stderr.decode(errors="replace")) + line = output.read_text(encoding="utf-8-sig").strip() + self.assertTrue(line.startswith("env=")) + self.assertEqual(json.loads(line.removeprefix("env=")), {"PATH": ";".join(retained)}) + def test_fuzz_lockfile_tracks_workspace_crate_versions(self): lock = tomllib.loads((ROOT / "fuzz/Cargo.lock").read_text()) for crate in ("core", "rules", "scanner"): diff --git a/scripts/tests/test_python_sdk_release.py b/scripts/tests/test_python_sdk_release.py index 00b70475..33b9848c 100644 --- a/scripts/tests/test_python_sdk_release.py +++ b/scripts/tests/test_python_sdk_release.py @@ -2,9 +2,16 @@ import importlib.util import io import json +import os import re +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path +import shlex +import shutil +import subprocess +import tarfile import tempfile +import threading import unittest from unittest.mock import patch import urllib.error @@ -128,5 +135,108 @@ def test_transient_registry_failure_retries(self): sleep.assert_called_once_with(1) +class MaturinDownloadTests(unittest.TestCase): + def container_environment(self, workspace): + workflow = (ROOT / ".github/workflows/python-sdk.yml").read_text(encoding="utf-8") + build = workflow.split(" - name: Build portable Linux wheel\n", 1)[1] + build = build.split(" # manylinux builds", 1)[0] + self.assertIn("if: runner.os == 'Linux'", build) + self.assertIn("PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b", build) + self.assertIn("manylinux: '2_28'", build) + self.assertIn("maturin-version: 'v1.16.0'", build) + self.assertNotIn("CURL_HOME", workflow.split("jobs:\n", 1)[0]) + folded = re.search(r"(?m)^ docker-options: >-\n((?: .+\n)+)", build) + self.assertIsNotNone(folded) + options = " ".join(line.strip() for line in folded.group(1).splitlines()) + options = shlex.split(options.replace("${{ github.workspace }}", workspace.as_posix())) + self.assertEqual(options[::2], ["-e"] * 4) + environment = dict(option.split("=", 1) for option in options[1::2]) + self.assertEqual(environment, { + "VECTORSCAN_BUILD_FROM_SOURCE": "1", + "CARGO_PROFILE_RELEASE_PANIC": "unwind", + "CARGO_PROFILE_RELEASE_LTO": "false", + "CURL_HOME": (workspace / ".github/maturin-curl").as_posix(), + }) + return environment + + def download(self, statuses): + curl = shutil.which("curl") + self.assertIsNotNone(curl, "curl is required to exercise the container download configuration") + archive = io.BytesIO() + payload = b"fixture maturin binary\n" + with tarfile.open(fileobj=archive, mode="w:gz") as bundle: + entry = tarfile.TarInfo("maturin") + entry.size = len(payload) + bundle.addfile(entry, io.BytesIO(payload)) + requests = [] + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + status = statuses[min(len(requests), len(statuses) - 1)] + requests.append(status) + body = archive.getvalue() if status == 200 else b"upstream gateway error\n" + self.send_response(status) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + pass + + with tempfile.TemporaryDirectory(prefix="kingfisher maturin downloads ") as temporary: + workspace = Path(temporary) + config_dir = workspace / ".github/maturin-curl" + config_dir.mkdir(parents=True) + config = (ROOT / ".github/maturin-curl/.curlrc").read_bytes() + (config_dir / ".curlrc").write_bytes(config) + environment = {**os.environ, **self.container_environment(workspace)} + with ThreadingHTTPServer(("127.0.0.1", 0), Handler) as server: + thread = threading.Thread( + target=server.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True, + ) + thread.start() + try: + # Match the action's curl -L download; keep test requests local, + # even when the developer or runner has proxy variables set. + result = subprocess.run( + [curl, "-L", "--noproxy", "*", f"http://127.0.0.1:{server.server_port}/maturin.tar.gz"], + env=environment, capture_output=True, timeout=30, check=False, + ) + finally: + server.shutdown() + thread.join(timeout=5) + self.assertFalse(thread.is_alive()) + return result, requests, payload + + def test_gateway_timeout_retries_without_piping_error_page_into_tar(self): + result, requests, payload = self.download([504, 200]) + self.assertEqual(result.returncode, 0, result.stderr.decode()) + self.assertEqual(requests, [504, 200]) + with tarfile.open(fileobj=io.BytesIO(result.stdout), mode="r:gz") as bundle: + with bundle.extractfile("maturin") as binary: + self.assertEqual(binary.read(), payload) + + def test_permanent_http_error_fails_without_retry_or_error_body(self): + result, requests, _ = self.download([404]) + self.assertEqual(result.returncode, 22, result.stderr.decode()) + self.assertEqual(requests, [404]) + self.assertEqual(result.stdout, b"") + + def test_persistent_outage_exhausts_bounded_retries_and_still_fails(self): + result, requests, _ = self.download([503]) + self.assertEqual(result.returncode, 22, result.stderr.decode()) + self.assertEqual(requests, [503] * 5) + self.assertEqual(result.stdout, b"") + + def test_download_attempts_and_connection_time_are_bounded(self): + config = (ROOT / ".github/maturin-curl/.curlrc").read_text(encoding="utf-8") + for setting in ("retry-max-time", "connect-timeout", "max-time"): + with self.subTest(setting=setting): + value = re.search(rf"(?m)^{setting}\s*=\s*(\d+)\s*$", config) + self.assertIsNotNone(value) + self.assertGreater(int(value.group(1)), 0) + self.assertLessEqual(int(value.group(1)), 300) + + if __name__ == "__main__": unittest.main()