diff --git a/.github/workflows/server-kms-e2e.yaml b/.github/workflows/server-kms-e2e.yaml new file mode 100644 index 000000000..2d2698d87 --- /dev/null +++ b/.github/workflows/server-kms-e2e.yaml @@ -0,0 +1,208 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Server-side (aicrd) HashiCorp Vault-backed bundle attestation e2e. The +# companion of vault-kms-e2e.yaml, which exercises the same hashivault:// path +# through the aicr CLI; this one drives the aicrd HTTP SERVER's +# POST /v1/bundle?attest=true surface (#1150) with a non-interactive, +# operator-supplied KMS signing identity. +# +# It builds an ATTESTED aicrd binary (goreleaser + the cosign attest-blob hook, +# fed by generate-slsa-predicate which sets SLSA_PREDICATE + AICR_SIGNING_CONFIG), +# then delegates to the suite's run.sh. run.sh owns the full runtime lifecycle: +# it starts OpenBAO in dev mode (plain docker run, so it can reference the +# load-versions image pin — a services: container would start before that step), +# provisions an ecdsa-p256 Transit key, starts aicrd in the background with the +# signing identity, drives the sign + verify round-trip, and tears both down. +# Keeping OpenBAO + server ownership inside run.sh means the exact same script +# runs locally and in CI (no drift between an inlined workflow copy and the +# runner). +# +# Dev-mode OpenBAO serves plain HTTP with a known root token, so no TLS/mkcert +# dance is needed. The image is pinned in .settings.yaml +# (testing_tools.openbao_image) and resolved via load-versions, never :latest. + +name: Server KMS E2E + +on: + workflow_dispatch: {} + push: + branches: + - main + paths-ignore: + - '**.md' + - 'docs/**' + - 'LICENSE' + # Validate on PRs that touch the suite, its workflow/tooling, or the + # server-signing / verify code under test. Gated to same-repo in the job `if` + # below: fork PRs lack the OIDC token needed to attest the binary, so they + # skip rather than fail. + pull_request: + branches: + - main + paths: + - 'tests/chainsaw/signing/server-bundle-attestation-vault/**' + - '.github/workflows/server-kms-e2e.yaml' + - '.github/actions/load-versions/**' + - '.github/actions/generate-slsa-predicate/**' + - '.github/actions/setup-build-tools/**' + - '.settings.yaml' + - '.goreleaser.yaml' + # The server signing/attestation surface under test. + - 'pkg/server/**' + - 'pkg/bundler/**' + # The keyless/trust-root verification path imports pkg/trust directly. + - 'pkg/trust/**' + # `verify --key`/`--insecure-ignore-tlog` plumbing lives in pkg/cli; + # cmd/aicrd is the server entrypoint, cmd/aicr the CLI entrypoint. go.mod/ + # sum cover dependency bumps (e.g. sigstore) that change the signing path. + - 'pkg/cli/**' + - 'cmd/aicrd/**' + - 'cmd/aicr/**' + - 'go.mod' + - 'go.sum' + - 'vendor/**' + +# Least privilege at the workflow level; id-token is granted only to the job +# that needs it (below). +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + # Dev-mode OpenBAO: plain HTTP, fixed root token. Nothing here is a real secret. + VAULT_ADDR: http://127.0.0.1:8200 + VAULT_TOKEN: root + VAULT_KMS_KEY: aicr + # The aicrd binary built here is attested by THIS workflow's identity (via + # generate-slsa-predicate workflow_file: server-kms-e2e.yaml + the goreleaser + # cosign attest-blob hook), not the release workflow (on-tag.yaml) the server + # pins by default. Retarget the certificate-identity pattern to this workflow + # so the server can verify its own binary attestation at startup. Still pinned + # to NVIDIA/aicr (enforced by verifier.ValidateIdentityPattern). run.sh + # defaults to the same value; set here explicitly for clarity. + AICR_BINARY_ATTESTATION_IDENTITY_REGEXP: '^https://github\.com/NVIDIA/aicr/\.github/workflows/server-kms-e2e\.yaml@.*' + +jobs: + server-kms-e2e: + name: Server KMS E2E + runs-on: ubuntu-latest + timeout-minutes: 20 + # id-token: write mints the GitHub OIDC token for Sigstore keyless signing. + # The "Build binaries" step runs goreleaser, whose cosign attest-blob hook + # uses that identity to produce the aicrd binary's SLSA provenance (fed by + # "Generate SLSA predicate"), which the server embeds as tool provenance. + permissions: + contents: read + id-token: write + # Skip on fork PRs: they get a read-only token, so `cosign attest-blob` + # (binary attestation) cannot run. push/workflow_dispatch always run. + if: >- + github.event_name != 'pull_request' || + github.event.pull_request.head.repo.full_name == github.repository + + steps: + - name: Checkout Code + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Load versions + id: versions + uses: ./.github/actions/load-versions + + - name: Setup Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version: ${{ steps.versions.outputs.go }} + cache: true + + - name: Install GoReleaser + uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3 + with: + version: ${{ steps.versions.outputs.goreleaser }} + install-only: true + + - name: Install Cosign + uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 + with: + # Pin from .settings.yaml (>= v3.1.0) so cosign logs DSSE attestations + # to Rekor v2 as hashedrekord/PAE. The installer default (v3.0.6) writes + # the legacy dsse entry type, which sigstore-go cannot verify. See #1650. + cosign-release: ${{ steps.versions.outputs.cosign }} + + - name: Generate SLSA predicate + uses: ./.github/actions/generate-slsa-predicate + with: + workflow_file: server-kms-e2e.yaml + + - name: Build binaries + env: + GOFLAGS: -mod=vendor + run: | + set -euo pipefail + # Builds aicr + aicrd for the host target. With SLSA_PREDICATE + + # AICR_SIGNING_CONFIG set (by "Generate SLSA predicate"), the + # .goreleaser.yaml aicrd post-hook runs cosign attest-blob and writes + # aicrd-attestation.sigstore.json next to the binary — the server's + # own tool provenance the e2e embeds. + goreleaser build --clean --single-target --snapshot --timeout 10m + + - name: Locate binaries + run: | + set -euo pipefail + + AICRD_BIN="" + for pattern in dist/aicrd_linux_amd64_v1/aicrd dist/aicrd_linux_amd64/aicrd; do + if [ -x "$pattern" ]; then + AICRD_BIN="$(pwd)/$pattern" + break + fi + done + if [ -z "$AICRD_BIN" ]; then + echo "::error::Built aicrd server binary not found in dist/" + exit 1 + fi + + AICR_BIN="" + for pattern in dist/aicr_linux_amd64_v1/aicr dist/aicr_linux_amd64/aicr; do + if [ -x "$pattern" ]; then + AICR_BIN="$(pwd)/$pattern" + break + fi + done + if [ -z "$AICR_BIN" ]; then + echo "::error::Built aicr CLI binary not found in dist/" + exit 1 + fi + + echo "AICRD_BIN=${AICRD_BIN}" >> "$GITHUB_ENV" + echo "AICR_BIN=${AICR_BIN}" >> "$GITHUB_ENV" + echo "Server binary: ${AICRD_BIN}" + echo "CLI binary: ${AICR_BIN}" + + ATTEST_FILE="$(dirname "$AICRD_BIN")/aicrd-attestation.sigstore.json" + if [ -f "$ATTEST_FILE" ]; then + echo "aicrd binary attestation found: $ATTEST_FILE (full mode)" + else + echo "::warning::No aicrd binary attestation found; run.sh will fall back to smoke mode" + fi + + - name: Run Server KMS e2e (starts OpenBAO + aicrd, sign + verify) + run: | + set -euo pipefail + ./tests/chainsaw/signing/server-bundle-attestation-vault/run.sh diff --git a/.gitignore b/.gitignore index eebe30654..8c8b6da9d 100644 --- a/.gitignore +++ b/.gitignore @@ -37,6 +37,9 @@ bin/ /aicr-attestation.sigstore.json /recipe-catalog.sigstore.json +# Generated aicrd binary attestations copied into ko data at release time +cmd/aicrd/kodata/*.sigstore.json + # ============================================================================= # Build & Release # ============================================================================= diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 1026c5c7e..4a1196db6 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -109,6 +109,32 @@ builds: -X github.com/NVIDIA/aicr/pkg/server.version={{.Version}} -X github.com/NVIDIA/aicr/pkg/server.commit={{.FullCommit}} -X github.com/NVIDIA/aicr/pkg/server.date={{.Date}} + hooks: + post: + ## Binary attestation for the aicrd server binary. Same cosign attest-blob + ## flow as the aicr CLI build above (Rekor v2 via SLSA_PREDICATE + + ## AICR_SIGNING_CONFIG; skipped when SLSA_PREDICATE is unset, e.g. local + ## snapshot builds). Writes the bundle next to the binary AND copies it into + ## cmd/aicrd/kodata/ with a per-arch name so ko ships it at KO_DATA_PATH for + ## the multi-arch image. The server (pkg/server/signing.go + ## resolveBinaryAttestationPath) reads KO_DATA_PATH/aicrd--attestation.sigstore.json; + ## keep the filename in sync with defaults.BinaryAttestationKoDataNameFormat. + - cmd: >- + bash -c '[ -z "${SLSA_PREDICATE:-}" ] && exit 0; + [ -z "${AICR_SIGNING_CONFIG:-}" ] && { echo "AICR_SIGNING_CONFIG unset while signing; refusing to fall back to Rekor v1" >&2; exit 1; }; + for n in 1 2 3; do + cosign attest-blob + --predicate "${SLSA_PREDICATE}" + --type https://slsa.dev/provenance/v1 + --signing-config "${AICR_SIGNING_CONFIG}" + --bundle "$(dirname "{{ .Path }}")/aicrd-attestation.sigstore.json" + --yes "{{ .Path }}" && break; + echo "cosign attest-blob attempt $n failed; retrying" >&2; + if [ "$n" -lt 3 ]; then sleep $((n * 5)); else echo "cosign attest-blob failed after 3 attempts" >&2; exit 1; fi; + done; + mkdir -p cmd/aicrd/kodata; + cp "$(dirname "{{ .Path }}")/aicrd-attestation.sigstore.json" "cmd/aicrd/kodata/aicrd-{{ .Arch }}-attestation.sigstore.json"' + output: true goos: - linux goarch: diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index cb58c95cb..ff33b42c4 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -90,11 +90,11 @@ excluded (their license is covered by the Go distribution). | `github.com/go-openapi/swag/cmdutils` | Apache-2.0 | https://github.com/NVIDIA/aicr/blob/HEAD/vendor/github.com/go-openapi/swag/cmdutils/LICENSE | | `github.com/go-openapi/swag/conv` | Apache-2.0 | https://github.com/NVIDIA/aicr/blob/HEAD/vendor/github.com/go-openapi/swag/conv/LICENSE | | `github.com/go-openapi/swag/fileutils` | Apache-2.0 | https://github.com/NVIDIA/aicr/blob/HEAD/vendor/github.com/go-openapi/swag/fileutils/LICENSE | -| `github.com/go-openapi/swag/jsonname` | Apache-2.0 | https://github.com/NVIDIA/aicr/blob/HEAD/vendor/github.com/go-openapi/swag/jsonname/LICENSE | | `github.com/go-openapi/swag/jsonutils` | Apache-2.0 | https://github.com/NVIDIA/aicr/blob/HEAD/vendor/github.com/go-openapi/swag/jsonutils/LICENSE | | `github.com/go-openapi/swag/loading` | Apache-2.0 | https://github.com/NVIDIA/aicr/blob/HEAD/vendor/github.com/go-openapi/swag/loading/LICENSE | | `github.com/go-openapi/swag/mangling` | Apache-2.0 | https://github.com/NVIDIA/aicr/blob/HEAD/vendor/github.com/go-openapi/swag/mangling/LICENSE | | `github.com/go-openapi/swag/netutils` | Apache-2.0 | https://github.com/NVIDIA/aicr/blob/HEAD/vendor/github.com/go-openapi/swag/netutils/LICENSE | +| `github.com/go-openapi/swag/pools` | Apache-2.0 | https://github.com/NVIDIA/aicr/blob/HEAD/vendor/github.com/go-openapi/swag/pools/LICENSE | | `github.com/go-openapi/swag/stringutils` | Apache-2.0 | https://github.com/NVIDIA/aicr/blob/HEAD/vendor/github.com/go-openapi/swag/stringutils/LICENSE | | `github.com/go-openapi/swag/typeutils` | Apache-2.0 | https://github.com/NVIDIA/aicr/blob/HEAD/vendor/github.com/go-openapi/swag/typeutils/LICENSE | | `github.com/go-openapi/swag/yamlutils` | Apache-2.0 | https://github.com/NVIDIA/aicr/blob/HEAD/vendor/github.com/go-openapi/swag/yamlutils/LICENSE | @@ -11673,10 +11673,10 @@ THE SOFTWARE. ``` -### github.com/go-openapi/swag/jsonname +### github.com/go-openapi/swag/jsonutils * License: Apache-2.0 -* Source: https://github.com/NVIDIA/aicr/blob/HEAD/vendor/github.com/go-openapi/swag/jsonname/LICENSE +* Source: https://github.com/NVIDIA/aicr/blob/HEAD/vendor/github.com/go-openapi/swag/jsonutils/LICENSE #### LICENSE @@ -11887,10 +11887,10 @@ THE SOFTWARE. ``` -### github.com/go-openapi/swag/jsonutils +### github.com/go-openapi/swag/loading * License: Apache-2.0 -* Source: https://github.com/NVIDIA/aicr/blob/HEAD/vendor/github.com/go-openapi/swag/jsonutils/LICENSE +* Source: https://github.com/NVIDIA/aicr/blob/HEAD/vendor/github.com/go-openapi/swag/loading/LICENSE #### LICENSE @@ -12101,10 +12101,10 @@ THE SOFTWARE. ``` -### github.com/go-openapi/swag/loading +### github.com/go-openapi/swag/mangling * License: Apache-2.0 -* Source: https://github.com/NVIDIA/aicr/blob/HEAD/vendor/github.com/go-openapi/swag/loading/LICENSE +* Source: https://github.com/NVIDIA/aicr/blob/HEAD/vendor/github.com/go-openapi/swag/mangling/LICENSE #### LICENSE @@ -12315,10 +12315,10 @@ THE SOFTWARE. ``` -### github.com/go-openapi/swag/mangling +### github.com/go-openapi/swag/netutils * License: Apache-2.0 -* Source: https://github.com/NVIDIA/aicr/blob/HEAD/vendor/github.com/go-openapi/swag/mangling/LICENSE +* Source: https://github.com/NVIDIA/aicr/blob/HEAD/vendor/github.com/go-openapi/swag/netutils/LICENSE #### LICENSE @@ -12529,10 +12529,10 @@ THE SOFTWARE. ``` -### github.com/go-openapi/swag/netutils +### github.com/go-openapi/swag/pools * License: Apache-2.0 -* Source: https://github.com/NVIDIA/aicr/blob/HEAD/vendor/github.com/go-openapi/swag/netutils/LICENSE +* Source: https://github.com/NVIDIA/aicr/blob/HEAD/vendor/github.com/go-openapi/swag/pools/LICENSE #### LICENSE @@ -28664,7 +28664,7 @@ project { #### LICENSE ```text -Copyright (c) 2015 HashiCorp, Inc. +Copyright IBM Corp. 2016, 2025 Mozilla Public License, version 2.0 @@ -29049,7 +29049,7 @@ For a step-by-step walkthrough on using these client libraries, see the [develop #### auth.go ```text -// Copyright (c) HashiCorp, Inc. +// Copyright IBM Corp. 2016, 2025 // SPDX-License-Identifier: MPL-2.0 package api @@ -29170,7 +29170,7 @@ func (a *Auth) checkAndSetToken(s *Secret) (*Secret, error) { #### auth_token.go ```text -// Copyright (c) HashiCorp, Inc. +// Copyright IBM Corp. 2016, 2025 // SPDX-License-Identifier: MPL-2.0 package api @@ -29553,7 +29553,7 @@ type TokenCreateRequest struct { #### client.go ```text -// Copyright (c) HashiCorp, Inc. +// Copyright IBM Corp. 2016, 2025 // SPDX-License-Identifier: MPL-2.0 package api @@ -31503,7 +31503,7 @@ func validateToken(t string) error { #### hcl_dup_attr_deprecation.go ```text -// Copyright (c) HashiCorp, Inc. +// Copyright IBM Corp. 2016, 2025 // SPDX-License-Identifier: MPL-2.0 package api @@ -31556,7 +31556,7 @@ func parseAndCheckForDuplicateHclAttributes(input string) (res *ast.File, duplic #### help.go ```text -// Copyright (c) HashiCorp, Inc. +// Copyright IBM Corp. 2016, 2025 // SPDX-License-Identifier: MPL-2.0 package api @@ -31602,7 +31602,7 @@ type Help struct { #### kv.go ```text -// Copyright (c) HashiCorp, Inc. +// Copyright IBM Corp. 2016, 2025 // SPDX-License-Identifier: MPL-2.0 package api @@ -31667,7 +31667,7 @@ func (c *Client) KVv2(mountPath string) *KVv2 { #### kv_v1.go ```text -// Copyright (c) HashiCorp, Inc. +// Copyright IBM Corp. 2016, 2025 // SPDX-License-Identifier: MPL-2.0 package api @@ -31733,7 +31733,7 @@ func (kv *KVv1) Delete(ctx context.Context, secretPath string) error { #### kv_v2.go ```text -// Copyright (c) HashiCorp, Inc. +// Copyright IBM Corp. 2016, 2025 // SPDX-License-Identifier: MPL-2.0 package api @@ -32520,7 +32520,7 @@ func toMetadataMap(patchInput KVMetadataPatchInput) (map[string]interface{}, err #### lifetime_watcher.go ```text -// Copyright (c) HashiCorp, Inc. +// Copyright IBM Corp. 2016, 2025 // SPDX-License-Identifier: MPL-2.0 package api @@ -32957,7 +32957,7 @@ type ( #### logical.go ```text -// Copyright (c) HashiCorp, Inc. +// Copyright IBM Corp. 2016, 2025 // SPDX-License-Identifier: MPL-2.0 package api @@ -33259,6 +33259,18 @@ func (c *Logical) addExtraHeaders(r *Request, headers http.Header) error { return nil } +func (c *Logical) PatchRaw(path string, data []byte) (*Response, error) { + return c.PatchRawWithContext(context.Background(), path, data) +} + +func (c *Logical) PatchRawWithContext(ctx context.Context, path string, data []byte) (*Response, error) { + r := c.c.NewRequest(http.MethodPatch, "/v1/"+path) + r.Headers.Set("Content-Type", "application/scim+json") + r.BodyBytes = data + + return c.writeRaw(ctx, r) +} + // Recover recovers the data at the given Vault path from a loaded snapshot. // The snapshotID parameter is the ID of the loaded snapshot func (c *Logical) Recover(ctx context.Context, path string, snapshotID string) (*Secret, error) { @@ -33340,6 +33352,15 @@ func (c *Logical) DeleteWithContext(ctx context.Context, path string) (*Secret, return c.DeleteWithDataWithContext(ctx, path, nil) } +func (c *Logical) DeleteRaw(path string) (*Response, error) { + return c.DeleteRawWithContext(context.Background(), path) +} + +func (c *Logical) DeleteRawWithContext(ctx context.Context, path string) (*Response, error) { + r := c.c.NewRequest(http.MethodDelete, "/v1/"+path) + return c.c.RawRequestWithContext(ctx, r) +} + func (c *Logical) DeleteWithData(path string, data map[string][]string) (*Secret, error) { return c.DeleteWithDataWithContext(context.Background(), path, data) } @@ -33487,7 +33508,7 @@ func (c *Logical) UnwrapWithContext(ctx context.Context, wrappingToken string) ( #### logical_requests.go ```text -// Copyright (c) HashiCorp, Inc. +// Copyright IBM Corp. 2016, 2025 // SPDX-License-Identifier: MPL-2.0 package api @@ -33612,7 +33633,7 @@ func (r *defaultLogicalRequest) Values() url.Values { #### output_policy.go ```text -// Copyright (c) HashiCorp, Inc. +// Copyright IBM Corp. 2016, 2025 // SPDX-License-Identifier: MPL-2.0 package api @@ -33717,7 +33738,7 @@ func formatOutputPolicy(path string, capabilities []string) string { #### output_string.go ```text -// Copyright (c) HashiCorp, Inc. +// Copyright IBM Corp. 2016, 2025 // SPDX-License-Identifier: MPL-2.0 package api @@ -33825,7 +33846,7 @@ func (d *OutputStringError) buildCurlString() (string, error) { #### plugin_helpers.go ```text -// Copyright (c) HashiCorp, Inc. +// Copyright IBM Corp. 2016, 2025 // SPDX-License-Identifier: MPL-2.0 package api @@ -34050,7 +34071,7 @@ func VaultPluginTLSProviderContext(ctx context.Context, apiTLSConfig *TLSConfig) #### plugin_runtime_types.go ```text -// Copyright (c) HashiCorp, Inc. +// Copyright IBM Corp. 2016, 2025 // SPDX-License-Identifier: MPL-2.0 package api @@ -34086,7 +34107,7 @@ func ParsePluginRuntimeType(PluginRuntimeType string) (PluginRuntimeType, error) #### plugin_types.go ```text -// Copyright (c) HashiCorp, Inc. +// Copyright IBM Corp. 2016, 2025 // SPDX-License-Identifier: MPL-2.0 package api @@ -34303,7 +34324,7 @@ func (i RenewBehavior) IsARenewBehavior() bool { #### replication_status.go ```text -// Copyright (c) HashiCorp, Inc. +// Copyright IBM Corp. 2016, 2025 // SPDX-License-Identifier: MPL-2.0 package api @@ -34442,7 +34463,7 @@ func (c *Sys) ReplicationStatusWithContext(ctx context.Context, path string) (*R #### request.go ```text -// Copyright (c) HashiCorp, Inc. +// Copyright IBM Corp. 2016, 2025 // SPDX-License-Identifier: MPL-2.0 package api @@ -34603,7 +34624,7 @@ func (r *Request) toRetryableHTTP() (*retryablehttp.Request, error) { #### response.go ```text -// Copyright (c) HashiCorp, Inc. +// Copyright IBM Corp. 2016, 2025 // SPDX-License-Identifier: MPL-2.0 package api @@ -34746,7 +34767,7 @@ func (r *ResponseError) Error() string { #### secret.go ```text -// Copyright (c) HashiCorp, Inc. +// Copyright IBM Corp. 2016, 2025 // SPDX-License-Identifier: MPL-2.0 package api @@ -35151,7 +35172,7 @@ func ParseSecret(r io.Reader) (*Secret, error) { #### ssh.go ```text -// Copyright (c) HashiCorp, Inc. +// Copyright IBM Corp. 2016, 2025 // SPDX-License-Identifier: MPL-2.0 package api @@ -35235,7 +35256,7 @@ func (c *SSH) SignKeyWithContext(ctx context.Context, role string, data map[stri #### ssh_agent.go ```text -// Copyright (c) HashiCorp, Inc. +// Copyright IBM Corp. 2016, 2025 // SPDX-License-Identifier: MPL-2.0 package api @@ -35519,7 +35540,7 @@ func (c *SSHHelper) VerifyWithContext(ctx context.Context, otp string) (*SSHVeri #### sudo_paths.go ```text -// Copyright (c) HashiCorp, Inc. +// Copyright IBM Corp. 2016, 2025 // SPDX-License-Identifier: MPL-2.0 package api @@ -35577,6 +35598,7 @@ var sudoPaths = map[string]*regexp.Regexp{ "/sys/replication/reindex": regexp.MustCompile(`^/sys/replication/reindex$`), "/sys/storage/raft/snapshot-auto/config": regexp.MustCompile(`^/sys/storage/raft/snapshot-auto/config/?$`), "/sys/storage/raft/snapshot-auto/config/{name}": regexp.MustCompile(`^/sys/storage/raft/snapshot-auto/config/[^/]+$`), + "/sys/reporting/scan": regexp.MustCompile(`^/sys/reporting/scan$`), } func SudoPaths() map[string]*regexp.Regexp { @@ -35613,7 +35635,7 @@ func IsSudoPath(path string) bool { #### sys.go ```text -// Copyright (c) HashiCorp, Inc. +// Copyright IBM Corp. 2016, 2025 // SPDX-License-Identifier: MPL-2.0 package api @@ -35633,7 +35655,7 @@ func (c *Client) Sys() *Sys { #### sys_audit.go ```text -// Copyright (c) HashiCorp, Inc. +// Copyright IBM Corp. 2016, 2025 // SPDX-License-Identifier: MPL-2.0 package api @@ -35798,7 +35820,7 @@ type Audit struct { #### sys_auth.go ```text -// Copyright (c) HashiCorp, Inc. +// Copyright IBM Corp. 2016, 2025 // SPDX-License-Identifier: MPL-2.0 package api @@ -35937,10 +35959,86 @@ type ( ``` +#### sys_billing.go + +```text +// Copyright IBM Corp. 2016, 2025 +// SPDX-License-Identifier: MPL-2.0 + +package api + +import ( + "context" + "errors" + "net/http" + + "github.com/mitchellh/mapstructure" +) + +// BillingOverview returns billing metrics for the current and previous month. +// If updateCounts is true, the current month's counts will be updated before returning. +// This is an expensive operation that holds locks and should be used sparingly. +func (c *Sys) BillingOverview(updateCounts bool) (*BillingOverviewResponse, error) { + return c.BillingOverviewWithContext(context.Background(), updateCounts) +} + +// BillingOverviewWithContext returns billing metrics for the current and previous month. +func (c *Sys) BillingOverviewWithContext(ctx context.Context, updateCounts bool) (*BillingOverviewResponse, error) { + ctx, cancelFunc := c.c.withConfiguredTimeout(ctx) + defer cancelFunc() + + r := c.c.NewRequest(http.MethodGet, "/v1/sys/billing/overview") + if updateCounts { + r.Params.Set("refresh_data", "true") + } + + resp, err := c.c.rawRequestWithContext(ctx, r) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + secret, err := ParseSecret(resp.Body) + if err != nil { + return nil, err + } + if secret == nil || secret.Data == nil { + return nil, errors.New("data from server response is empty") + } + + var result BillingOverviewResponse + err = mapstructure.Decode(secret.Data, &result) + if err != nil { + return nil, err + } + + return &result, nil +} + +// BillingOverviewResponse represents the response from the billing overview endpoint. +type BillingOverviewResponse struct { + Months []BillingMonth `json:"months" mapstructure:"months"` +} + +// BillingMonth represents billing data for a single month. +type BillingMonth struct { + Month string `json:"month" mapstructure:"month"` + UpdatedAt string `json:"updated_at" mapstructure:"updated_at"` + UsageMetrics []UsageMetric `json:"usage_metrics" mapstructure:"usage_metrics"` +} + +// UsageMetric represents a single usage metric with its data. +type UsageMetric struct { + MetricName string `json:"metric_name" mapstructure:"metric_name"` + MetricData map[string]interface{} `json:"metric_data" mapstructure:"metric_data"` +} + +``` + #### sys_capabilities.go ```text -// Copyright (c) HashiCorp, Inc. +// Copyright IBM Corp. 2016, 2025 // SPDX-License-Identifier: MPL-2.0 package api @@ -36079,7 +36177,7 @@ func (c *Sys) CapabilitiesAccessorWithContext(ctx context.Context, accessor, pat #### sys_config_cors.go ```text -// Copyright (c) HashiCorp, Inc. +// Copyright IBM Corp. 2016, 2025 // SPDX-License-Identifier: MPL-2.0 package api @@ -36179,7 +36277,7 @@ type CORSResponse struct { #### sys_generate_root.go ```text -// Copyright (c) HashiCorp, Inc. +// Copyright IBM Corp. 2016, 2025 // SPDX-License-Identifier: MPL-2.0 package api @@ -36383,7 +36481,7 @@ type GenerateRootStatusResponse struct { #### sys_hastatus.go ```text -// Copyright (c) HashiCorp, Inc. +// Copyright IBM Corp. 2016, 2025 // SPDX-License-Identifier: MPL-2.0 package api @@ -36438,7 +36536,7 @@ type HANode struct { #### sys_health.go ```text -// Copyright (c) HashiCorp, Inc. +// Copyright IBM Corp. 2016, 2025 // SPDX-License-Identifier: MPL-2.0 package api @@ -36505,7 +36603,7 @@ type HealthResponse struct { #### sys_init.go ```text -// Copyright (c) HashiCorp, Inc. +// Copyright IBM Corp. 2016, 2025 // SPDX-License-Identifier: MPL-2.0 package api @@ -36588,7 +36686,7 @@ type InitResponse struct { #### sys_leader.go ```text -// Copyright (c) HashiCorp, Inc. +// Copyright IBM Corp. 2016, 2025 // SPDX-License-Identifier: MPL-2.0 package api @@ -36638,7 +36736,7 @@ type LeaderResponse struct { #### sys_leases.go ```text -// Copyright (c) HashiCorp, Inc. +// Copyright IBM Corp. 2016, 2025 // SPDX-License-Identifier: MPL-2.0 package api @@ -36810,7 +36908,7 @@ type RevokeOptions struct { #### sys_mfa.go ```text -// Copyright (c) HashiCorp, Inc. +// Copyright IBM Corp. 2016, 2025 // SPDX-License-Identifier: MPL-2.0 package api @@ -36864,7 +36962,7 @@ func (c *Sys) MFAValidateWithContext(ctx context.Context, requestID string, payl #### sys_monitor.go ```text -// Copyright (c) HashiCorp, Inc. +// Copyright IBM Corp. 2016, 2025 // SPDX-License-Identifier: MPL-2.0 package api @@ -36944,7 +37042,7 @@ func (c *Sys) Monitor(ctx context.Context, logLevel string, logFormat string) (c #### sys_mounts.go ```text -// Copyright (c) HashiCorp, Inc. +// Copyright IBM Corp. 2016, 2025 // SPDX-License-Identifier: MPL-2.0 package api @@ -37184,6 +37282,7 @@ type TuneMountConfigInput struct { TokenType *string `json:"token_type,omitempty" mapstructure:"token_type"` AllowedManagedKeys *[]string `json:"allowed_managed_keys,omitempty" mapstructure:"allowed_managed_keys"` PluginVersion *string `json:"plugin_version,omitempty"` + OverridePinnedVersion *bool `json:"override_pinned_version,omitempty" mapstructure:"override_pinned_version"` UserLockoutConfig *TuneUserLockoutConfigInput `json:"user_lockout_config,omitempty"` DelegatedAuthAccessors *[]string `json:"delegated_auth_accessors,omitempty" mapstructure:"delegated_auth_accessors"` IdentityTokenKey *string `json:"identity_token_key,omitempty" mapstructure:"identity_token_key"` @@ -37267,6 +37366,8 @@ func (c *Sys) TuneMountWithContext(ctx context.Context, path string, config Moun tuneConfig.PluginVersion = &config.PluginVersion } + tuneConfig.OverridePinnedVersion = config.OverridePinnedVersion + if config.UserLockoutConfig != nil { userLockoutConfig := TuneUserLockoutConfigInput{} if config.UserLockoutConfig.LockoutDuration != "" { @@ -37362,6 +37463,7 @@ type MountConfigInput struct { TokenType string `json:"token_type,omitempty" mapstructure:"token_type"` AllowedManagedKeys []string `json:"allowed_managed_keys,omitempty" mapstructure:"allowed_managed_keys"` PluginVersion string `json:"plugin_version,omitempty"` + OverridePinnedVersion *bool `json:"override_pinned_version,omitempty" mapstructure:"override_pinned_version"` UserLockoutConfig *UserLockoutConfigInput `json:"user_lockout_config,omitempty"` DelegatedAuthAccessors []string `json:"delegated_auth_accessors,omitempty" mapstructure:"delegated_auth_accessors"` IdentityTokenKey string `json:"identity_token_key,omitempty" mapstructure:"identity_token_key"` @@ -37401,6 +37503,7 @@ type MountConfigOutput struct { DelegatedAuthAccessors []string `json:"delegated_auth_accessors,omitempty" mapstructure:"delegated_auth_accessors"` IdentityTokenKey string `json:"identity_token_key,omitempty" mapstructure:"identity_token_key"` TrimRequestTrailingSlashes bool `json:"trim_request_trailing_slashes,omitempty" mapstructure:"trim_request_trailing_slashes"` + OverridePinnedVersion bool `json:"override_pinned_version,omitempty" mapstructure:"override_pinned_version"` // Deprecated: This field will always be blank for newer server responses. PluginName string `json:"plugin_name,omitempty" mapstructure:"plugin_name"` @@ -37440,7 +37543,7 @@ type MountMigrationStatusInfo struct { #### sys_plugins.go ```text -// Copyright (c) HashiCorp, Inc. +// Copyright IBM Corp. 2016, 2025 // SPDX-License-Identifier: MPL-2.0 package api @@ -37484,6 +37587,7 @@ type PluginDetails struct { Version string `json:"version,omitempty"` Builtin bool `json:"builtin"` DeprecationStatus string `json:"deprecation_status,omitempty" mapstructure:"deprecation_status"` + SHA256 string `json:"sha256,omitempty"` } // ListPlugins wraps ListPluginsWithContext using context.Background. @@ -37926,7 +38030,7 @@ func catalogPathByType(pluginType PluginType, name string) string { #### sys_plugins_runtimes.go ```text -// Copyright (c) HashiCorp, Inc. +// Copyright IBM Corp. 2016, 2025 // SPDX-License-Identifier: MPL-2.0 package api @@ -38122,7 +38226,7 @@ func pluginRuntimeCatalogPathByType(runtimeType PluginRuntimeType, name string) #### sys_policy.go ```text -// Copyright (c) HashiCorp, Inc. +// Copyright IBM Corp. 2016, 2025 // SPDX-License-Identifier: MPL-2.0 package api @@ -38265,7 +38369,7 @@ type listPoliciesResp struct { #### sys_raft.go ```text -// Copyright (c) HashiCorp, Inc. +// Copyright IBM Corp. 2016, 2025 // SPDX-License-Identifier: MPL-2.0 package api @@ -38806,7 +38910,7 @@ func (c *Sys) raftUnloadSnapshotWithContext(ctx context.Context, snapID string, #### sys_rekey.go ```text -// Copyright (c) HashiCorp, Inc. +// Copyright IBM Corp. 2016, 2025 // SPDX-License-Identifier: MPL-2.0 package api @@ -39325,10 +39429,75 @@ type RekeyVerificationUpdateResponse struct { ``` +#### sys_reporting_scan.go + +```text +// Copyright IBM Corp. 2016, 2025 +// SPDX-License-Identifier: MPL-2.0 + +package api + +import ( + "context" + "errors" + "net/http" + + "github.com/mitchellh/mapstructure" +) + +func (c *Sys) ReportingScan(opts *ReportingScanRequest) (*ReportingScanOutput, error) { + return c.ReportingScanWithContext(context.Background(), opts) +} + +func (c *Sys) ReportingScanWithContext(ctx context.Context, opts *ReportingScanRequest) (*ReportingScanOutput, error) { + ctx, cancelFunc := c.c.withConfiguredTimeout(ctx) + defer cancelFunc() + + r := c.c.NewRequest(http.MethodPost, "/v1/sys/reporting/scan") + + if err := r.SetJSONBody(opts); err != nil { + return nil, err + } + + resp, err := c.c.rawRequestWithContext(ctx, r) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + secret, err := ParseSecret(resp.Body) + if err != nil { + return nil, err + } + if secret == nil || secret.Data == nil { + return nil, errors.New("data from server response is empty") + } + + var result ReportingScanOutput + err = mapstructure.Decode(secret.Data, &result) + if err != nil { + return nil, err + } + + return &result, err +} + +// ReportingScanRequest represents the parameters consumed by the reporting scan API +type ReportingScanRequest struct { + Async bool `json:"async"` +} + +type ReportingScanOutput struct { + Timestamp string `json:"timestamp" structs:"timestamp" mapstructure:"timestamp"` + FullDirectoryPath string `json:"full_directory_path" structs:"full_directory_path" mapstructure:"full_directory_path"` +} + +``` + #### sys_rotate.go ```text -// Copyright (c) HashiCorp, Inc. +// Copyright IBM Corp. 2016, 2025 // SPDX-License-Identifier: MPL-2.0 package api @@ -39439,7 +39608,7 @@ type KeyStatus struct { #### sys_seal.go ```text -// Copyright (c) HashiCorp, Inc. +// Copyright IBM Corp. 2016, 2025 // SPDX-License-Identifier: MPL-2.0 package api @@ -39569,7 +39738,7 @@ type UnsealOpts struct { #### sys_stepdown.go ```text -// Copyright (c) HashiCorp, Inc. +// Copyright IBM Corp. 2016, 2025 // SPDX-License-Identifier: MPL-2.0 package api @@ -39601,7 +39770,7 @@ func (c *Sys) StepDownWithContext(ctx context.Context) error { #### sys_ui_custom_message.go ```text -// Copyright (c) HashiCorp, Inc. +// Copyright IBM Corp. 2016, 2025 // SPDX-License-Identifier: MPL-2.0 package api @@ -39888,7 +40057,7 @@ func (l *uiCustomMessageLink) UnmarshalJSON(b []byte) error { #### sys_utilization_report.go ```text -// Copyright (c) HashiCorp, Inc. +// Copyright IBM Corp. 2016, 2025 // SPDX-License-Identifier: MPL-2.0 package api diff --git a/api/aicr/v1/server.yaml b/api/aicr/v1/server.yaml index 74d9eec62..d0d823be5 100644 --- a/api/aicr/v1/server.yaml +++ b/api/aicr/v1/server.yaml @@ -37,8 +37,12 @@ info: - **Trust management** (`aicr trust update`) — updates the local Sigstore TUF root cache. - - **Bundle attestation** (`aicr bundle --attest`) — requires OIDC interactive or ambient - authentication for Sigstore signing. + - **Interactive Sigstore attestation** (`aicr bundle --attest` with browser-based OIDC): + the interactive, user-authenticated signing flow is CLI-only. The API server instead + offers non-interactive server-side attestation: `POST /v1/bundle?attest=true` returns a + signed bundle, with the server signing as itself using its operator-configured KMS key or + private-Sigstore keyless identity. No signing identity is ever taken from the request. + See the `attest` query parameter on POST /v1/bundle. - **OCI registry push** (`aicr bundle --output oci://...`) — pushes bundle artifacts to a container registry. @@ -1083,6 +1087,26 @@ paths: pattern: '^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$' maxLength: 253 example: gpu-runtime + - name: attest + in: query + required: false + description: > + Return a cryptographically signed bundle. When true, the server + signs the bundle as itself using its operator-configured signing + identity; no signing key, token, or identity is ever taken from the + request (the trust boundary). Parsed with Go strconv.ParseBool + semantics (accepts 1/t/T/TRUE/true/True and 0/f/F/FALSE/false/False); + a present-but-unparseable value is rejected with HTTP 400. Absent or + false returns an unsigned bundle, as before. If attestation is + requested but the server has no signing identity configured, the + request is rejected with HTTP 400 ("Server is not configured for + attestation"). A signed bundle additionally carries + attestation/bundle-attestation.sigstore.json and + attestation/aicr-attestation.sigstore.json inside the returned zip; + the response format is otherwise unchanged. + schema: + type: boolean + default: false requestBody: required: true description: The recipe (RecipeResult) to generate bundles from diff --git a/docs/contributor/api-server.md b/docs/contributor/api-server.md index 14c987131..bca7f518e 100644 --- a/docs/contributor/api-server.md +++ b/docs/contributor/api-server.md @@ -173,6 +173,17 @@ Environment variables read at startup: | `AICR_ALLOWED_INTENTS` | unset → unrestricted | same | | `AICR_ALLOWED_OS` | unset → unrestricted | same | | `AICR_LOG_LEVEL` | `info` | `pkg/logging` | +| `AICR_SIGNING_KEY` | unset → signing off | `parseSigningConfig` (Mode A KMS URI) | +| `AICR_FULCIO_URL` | unset → signing off | `parseSigningConfig` (Mode B private Fulcio) | +| `AICR_IDENTITY_TOKEN_FILE` | unset | `parseSigningConfig` (Mode B token source) | +| `AICR_REKOR_URL` | unset | `parseSigningConfig` (both modes) | +| `AICR_SIGNING_CONFIG_PATH` | unset | `parseSigningConfig` (both modes; Rekor v2) | +| `AICR_TLOG_UPLOAD` | `true` | `parseSigningConfig` (Mode A only) | +| `AICR_BINARY_ATTESTATION_FILE` | unset → `-attestation.sigstore.json` next to the binary | `resolveBinaryAttestationPath` (override for ko `KO_DATA_PATH` layouts) | +| `AICR_BINARY_ATTESTATION_IDENTITY_REGEXP` | unset → `verifier.TrustedRepositoryPattern` (release `on-tag.yaml`) | `resolveBinaryAttestationIdentityPattern` (must contain `NVIDIA/aicr`, validated by `verifier.ValidateIdentityPattern`; retargets the attesting NVIDIA workflow, e.g. an e2e build) | + +See [Server-Side Bundle Signing](#server-side-bundle-signing) for the identity +model and validation rules behind these variables. Compiled-time constants live in [`pkg/defaults`](https://github.com/NVIDIA/aicr/tree/main/pkg/defaults): @@ -192,6 +203,75 @@ Constraint: every per-handler `WithTimeout` must be ≤ `ServerHandlerTimeout`, and `ServerWriteTimeout` must be ≥ `ServerHandlerTimeout`, else the outer middleware silently clamps a slow request. +## Server-Side Bundle Signing + +`POST /v1/bundle?attest=true` returns a signed bundle. The signing identity is +operator configuration, parsed once at startup by `parseSigningConfig` +([`pkg/server/signing.go`](https://github.com/NVIDIA/aicr/blob/main/pkg/server/signing.go)) +into a `signingConfig`. No field on that struct ever comes from a request: the +server always signs as itself, so the request only chooses whether to sign, not +with what. The handler enforces the trust boundary in `resolveAttestRequest`, +rejecting `attest=true` with HTTP 400 when no identity is configured and +rejecting an unparseable `attest` value with HTTP 400. + +**Access control is the operator's responsibility.** The signature attests that +this aicrd deployment produced the bundle (build and tool provenance); it does not +endorse the caller-supplied recipe's contents. `/v1/bundle` accepts arbitrary +recipes and is not itself authenticated, so when signing is enabled the endpoint +must be access-controlled by the deployment (network policy, gateway, or mTLS) to +prevent an untrusted caller from obtaining server-signed bundles. Signing is +opt-in and off by default. + +**Two mutually exclusive modes.** Mode A is KMS-backed (`AICR_SIGNING_KEY`, a +cosign KMS URI validated against a scheme allowlist). Mode B is keyless against a +private Sigstore (`AICR_FULCIO_URL` plus a token source: `AICR_IDENTITY_TOKEN_FILE` +or GitHub Actions ambient OIDC). `parseSigningConfig` validates mode exclusivity +and completeness and **fails fast**: an ambiguous config (both modes set), a +malformed KMS URI, a partial keyless config, or a non-boolean `AICR_TLOG_UPLOAD` +(parsed only in KMS mode, where the toggle applies) returns an error from `Serve` +so the server does not start. When no signing variables are set, signing is simply +off and the server starts normally. + +**Per-request options are rebuilt, not cached.** `signingConfig.resolveOptions` +constructs the per-request `attestation.ResolveOptions`. For Mode B it reads the +identity-token file **fresh** on every call, because ServiceAccount tokens rotate +and Fulcio binds each certificate to a fresh token. + +**Binary attestation cached at startup.** Server signing also requires the aicrd +binary's own attestation (tool provenance): a Sigstore bundle shipped next to the +executable inside the container image, issued under the NVIDIA-CI identity and +bound to the binary's digest. `loadBinaryAttestation` verifies it **once** at +startup (`Serve` calls it after `parseSigningConfig`) and caches the raw bytes on +the `signingConfig`; each signed bundle embeds those bytes as +`attestation/aicr-attestation.sigstore.json`. This is fail-fast too: a signing +server that cannot prove its own provenance must not start. Producing that +attestation in CI/release is a separate dependency, tracked outside this feature. + +`resolveBinaryAttestationPath` chooses which file to verify: by default the +conventional path next to the executable (`FindBinaryAttestation`), or the +explicit `AICR_BINARY_ATTESTATION_FILE` override when set. The override exists for +ko-built images, whose assets live under `KO_DATA_PATH` +(`/var/run/ko/aicrd-attestation.sigstore.json`) rather than next to the binary. +Only the attestation file path changes; the digest verified against it is always +the running `os.Executable()` binary's digest. + +`resolveBinaryAttestationIdentityPattern` chooses the certificate-identity +pattern the attestation is verified against: `verifier.TrustedRepositoryPattern` +(the release `on-tag.yaml` workflow) by default, or the +`AICR_BINARY_ATTESTATION_IDENTITY_REGEXP` override when set. The override is +validated by `verifier.ValidateIdentityPattern`, which requires it to contain +`NVIDIA/aicr`, so it can only retarget which NVIDIA workflow attested the binary +(e.g. the server-kms e2e build), never widen the org. A bad override fails +startup fast. This mirrors the CLI's `--certificate-identity-regexp`, and a +custom pattern is logged because bundles the server then signs will not pass a +verifier using the default identity. + +**Injectable seams for tests.** The startup verifier +(`binaryAttestationVerifier`) and the per-request attester builder +(`attesterBuilder`) are function-typed fields so tests can inject fixtures: the +real verifier pins the NVIDIA-CI identity and the running binary's digest, which +a `go test` executable cannot satisfy. + ## OpenAPI Parity Test [`pkg/server/openapi_sync_test.go`](https://github.com/NVIDIA/aicr/blob/main/pkg/server/openapi_sync_test.go) diff --git a/docs/user/api-reference.md b/docs/user/api-reference.md index 013374f48..e3a7189c9 100644 --- a/docs/user/api-reference.md +++ b/docs/user/api-reference.md @@ -29,6 +29,7 @@ The AICR API Server provides HTTP REST access to recipe generation and bundle cr | Recipe generation | ✅ GET /v1/recipe | ✅ `aicr recipe` | | Value query | ✅ GET /v1/query | ✅ `aicr query` | | Bundle creation | ✅ POST /v1/bundle | ✅ `aicr bundle` | +| Bundle attestation | ✅ POST /v1/bundle?attest=true (server signs as itself) | ✅ `aicr bundle --attest` (interactive or ambient OIDC) | | Snapshot capture | ❌ Use CLI | ✅ `aicr snapshot` | | ConfigMap I/O | ❌ Use CLI | ✅ `cm://` URIs | | Agent deployment | ❌ Use CLI | ✅ `aicr snapshot` | @@ -411,6 +412,7 @@ Generate deployment bundles from a recipe. | `deployer` | string | helm | Deployment method: `helm`, `argocd`, `argocd-helm`, `flux`, or `helmfile` | | `repo` | string | | Git repository URL for GitOps deployments (used with `deployer=argocd` and `deployer=flux`; ignored by `deployer=argocd-helm`) | | `app-name` | string | | Parent Argo Application name (default: `aicr-stack` for `deployer=argocd-helm`, `nvidia-stack` for `deployer=argocd`). Must be a DNS-1123 subdomain. Required when deploying multiple non-overlapping AICR bundles to the same Argo CD namespace so the parent Applications do not collide. For `deployer=argocd-helm`, the value is the chart default and can still be overridden at install time via `helm install --set appName=...`. Rejected with HTTP 400 on other deployers. | +| `attest` | bool | false | Return a cryptographically signed bundle. When `true`, the server signs the bundle as **itself** using its operator-configured signing identity; no signing key, token, or identity is ever taken from the request. Parsed with Go `strconv.ParseBool` semantics; a present-but-unparseable value is rejected with HTTP 400. Absent or `false` returns an unsigned bundle. If the server has no signing identity configured, `attest=true` is rejected with HTTP 400 (`Server is not configured for attestation`). See [Server-Side Signing](#server-side-signing) for setup. | **Request Body:** @@ -566,6 +568,74 @@ verification: every manifest digest must match and every additional file or directory, symlink, or other non-regular object is rejected, except the exact allowed inventory metadata paths. +#### Server-Side Signing + +Pass `?attest=true` to `POST /v1/bundle` to receive a cryptographically signed +bundle. The server signs the bundle as **itself** using an operator-configured +signing identity. This is the trust boundary: no signing key, token, or identity +is ever taken from the request, so any client that can reach the endpoint gets +bundles signed under the server's identity, never its own. + +A signed bundle additionally carries, inside the returned zip: + +```text +attestation/bundle-attestation.sigstore.json # signature over the bundle +attestation/aicr-attestation.sigstore.json # aicrd tool-provenance attestation +``` + +`attest=true` requires a configured signing identity. If none is configured the +request is rejected with HTTP 400 (`Server is not configured for attestation`). +An unparseable `attest` value is also HTTP 400. Absent or `false` returns an +unsigned bundle, as before. + +The server supports two mutually exclusive signing modes, selected by +environment variables at startup. The configuration is validated fail-fast: a +malformed or ambiguous setting stops the server from starting. + +**Mode A: KMS key.** Set `AICR_SIGNING_KEY` to a cosign KMS URI. The bundle is +signed with a long-lived key held in the KMS; no OIDC identity is involved. + +**Mode B: keyless against a private Sigstore.** Set `AICR_FULCIO_URL` to a +private Fulcio CA plus a token source. The server obtains a short-lived signing +certificate from Fulcio using its own OIDC identity. Operator setup for Mode B: + +- Run a private Fulcio that trusts the cluster's ServiceAccount token issuer. +- Mount a projected ServiceAccount token with audience `sigstore` into the aicrd + pod, and point `AICR_IDENTITY_TOKEN_FILE` at it. The token is read fresh for + every signed request because ServiceAccount tokens rotate. +- Alternatively, when aicrd itself runs inside GitHub Actions, its ambient OIDC + environment is used as the token source. + +| Variable | Mode | Purpose | +|----------|------|---------| +| `AICR_SIGNING_KEY` | A | cosign KMS URI (`awskms://`, `gcpkms://`, `azurekms://`, `hashivault://`). Its presence selects Mode A. | +| `AICR_FULCIO_URL` | B | Private Fulcio CA endpoint. Its presence (with a token source) selects Mode B. | +| `AICR_IDENTITY_TOKEN_FILE` | B | Path to the server's OIDC token (projected ServiceAccount token, audience `sigstore`). Read fresh per request. | +| `AICR_REKOR_URL` | A, B | Rekor transparency-log endpoint override. | +| `AICR_SIGNING_CONFIG_PATH` | A, B | Sigstore SigningConfig JSON for Rekor v2 targeting. | +| `AICR_TLOG_UPLOAD` | A | Set `false` to skip the Rekor upload for air-gapped KMS signing. KMS-only; keyless always uploads. | +| `AICR_BINARY_ATTESTATION_FILE` | A, B | Absolute path to the aicrd binary attestation. Unset defaults to the conventional `-attestation.sigstore.json` next to the running binary. Set it when the attestation ships elsewhere in the image, e.g. a ko build stages assets under `KO_DATA_PATH` (`/var/run/ko/aicrd-attestation.sigstore.json`) rather than next to the binary. | +| `AICR_BINARY_ATTESTATION_IDENTITY_REGEXP` | A, B | Certificate-identity pattern the server pins its own binary attestation to. Unset uses the release-workflow default (`on-tag.yaml`). A custom value MUST still contain `NVIDIA/aicr` so it stays pinned to the NVIDIA org; it retargets which NVIDIA workflow attested the binary (e.g. an e2e workflow), not the org, and a value that is not so pinned fails startup. Mirrors the CLI's `--certificate-identity-regexp`. | + +Setting both `AICR_SIGNING_KEY` and the keyless variables is ambiguous and the +server refuses to start. + +Server signing also requires the aicrd **binary attestation** +(`aicrd-attestation.sigstore.json`, issued under the NVIDIA-CI identity and +bound to the aicrd binary digest) to be shipped inside the container image. The +server verifies it once at startup and embeds it as tool provenance +(`attestation/aicr-attestation.sigstore.json`) in every signed bundle. If +signing is enabled but that attestation is missing or invalid, the server fails +to start. Producing that attestation in the CI/release pipeline is a separate +dependency, tracked outside this feature. + +By default the server discovers that attestation next to its own executable. Set +`AICR_BINARY_ATTESTATION_FILE` to point at an explicit path when the image stages +it elsewhere: a ko-built image places assets under `KO_DATA_PATH` +(`/var/run/ko/aicrd-attestation.sigstore.json`), not next to the binary. Only the +attestation file path changes; it is still verified against the aicrd binary's +own digest. + --- ### GET /health diff --git a/pkg/bundler/bundler.go b/pkg/bundler/bundler.go index cae5091b8..14dbf07be 100644 --- a/pkg/bundler/bundler.go +++ b/pkg/bundler/bundler.go @@ -103,6 +103,15 @@ type DefaultBundler struct { // Attester signs bundle content. NoOpAttester is used when --attest is not set. Attester attestation.Attester + // verifiedBinaryAttestation, when non-empty, is a pre-verified binary + // attestation (Sigstore bundle bytes) supplied by the caller. It lets a + // long-running server verify its in-image binary attestation ONCE at + // startup and reuse it per bundle: New's fail-fast gate is satisfied + // without a file next to the binary, and attestBundle embeds these bytes + // directly instead of re-finding/re-verifying/re-reading. Empty (the CLI + // default) preserves the discover-and-verify-per-run path exactly. + verifiedBinaryAttestation []byte + // warnings stores warning messages to be added to deployment notes. warnings []string } @@ -137,6 +146,19 @@ func WithAllowLists(al *recipe.AllowLists) Option { } } +// WithVerifiedBinaryAttestation supplies a pre-verified binary attestation +// (Sigstore bundle bytes) to embed in attested bundles, bypassing the per-run +// FindBinaryAttestation + VerifyBinaryAttestation discovery. The caller is +// responsible for having verified these bytes (identity + binary-digest binding) +// before injecting them. Empty leaves the default discover-and-verify behavior. +func WithVerifiedBinaryAttestation(data []byte) Option { + return func(db *DefaultBundler) { + if len(data) > 0 { + db.verifiedBinaryAttestation = append([]byte(nil), data...) // defensive copy + } + } +} + // New creates a new DefaultBundler with the given options. // // Example: @@ -160,7 +182,7 @@ func New(opts ...Option) (*DefaultBundler, error) { // file exists before any expensive work (OIDC auth, recipe resolution, bundle // generation). Binaries installed via "go install" or manual download won't // have the attestation file that is included in release archives. - if db.Config.Attest() { + if db.Config.Attest() && len(db.verifiedBinaryAttestation) == 0 { binaryPath, err := os.Executable() if err != nil { return nil, errors.Wrap(errors.ErrCodeInternal, @@ -1841,6 +1863,21 @@ func (b *DefaultBundler) attestBundle(ctx context.Context, dir string, dataFiles // verifyAndCopyBinaryAttestation resolves the running binary's attestation, // cryptographically verifies it (REQ-6), and copies it into the bundle directory. func (b *DefaultBundler) verifyAndCopyBinaryAttestation(ctx context.Context, dir string) error { + // Injected, pre-verified attestation: write it directly. The caller + // verified identity + binary-digest binding at injection time, so we do + // not re-discover or re-verify here. + if len(b.verifiedBinaryAttestation) > 0 { + destPath, joinErr := deployer.SafeJoin(dir, attestation.BinaryAttestationFile) + if joinErr != nil { + return errors.Wrap(errors.ErrCodeInternal, "unsafe binary attestation path", joinErr) + } + if writeErr := os.WriteFile(destPath, b.verifiedBinaryAttestation, 0600); writeErr != nil { //nolint:gosec // path validated by SafeJoin + return errors.Wrap(errors.ErrCodeInternal, "failed to write injected binary attestation", writeErr) + } + slog.Info("embedded pre-verified binary attestation") + return nil + } + binaryPath, execErr := os.Executable() if execErr != nil { return errors.Wrap(errors.ErrCodeInternal, diff --git a/pkg/bundler/bundler_test.go b/pkg/bundler/bundler_test.go index d9a5ce523..0e069588c 100644 --- a/pkg/bundler/bundler_test.go +++ b/pkg/bundler/bundler_test.go @@ -514,6 +514,73 @@ func TestNew_AttestWithoutBinaryAttestation(t *testing.T) { } } +func TestNew_AttestWithInjectedBinaryAttestation(t *testing.T) { + // With a pre-verified binary attestation injected, New's fail-fast gate + // must be satisfied even though the test binary has no attestation file + // next to it (the "go install"/manual-download scenario). + cfg := config.NewConfig(config.WithAttest(true)) + b, err := New(WithConfig(cfg), WithVerifiedBinaryAttestation([]byte(`{"injected":true}`))) + if err != nil { + t.Fatalf("New() with injected attestation error = %v; gate should be bypassed", err) + } + if b == nil { + t.Fatal("New() returned nil bundler") + } +} + +// fixtureBinaryAttester returns fixed bundle JSON from Attest so tests reach +// verifyAndCopyBinaryAttestation (closedWorldTestAttester returns nil, which +// short-circuits attestBundle before the binary attestation is embedded). +type fixtureBinaryAttester struct { + bundleJSON []byte +} + +func (a *fixtureBinaryAttester) Attest(_ context.Context, _ attestation.AttestSubject) ([]byte, error) { + return a.bundleJSON, nil +} + +func (a *fixtureBinaryAttester) Identity() string { return "fixture" } + +func (a *fixtureBinaryAttester) HasRekorEntry() bool { return false } + +func TestAttestBundle_EmbedsInjectedBinaryAttestation(t *testing.T) { + dir := t.TempDir() + payloadPath := filepath.Join(dir, "payload.txt") + if err := os.WriteFile(payloadPath, []byte("payload"), 0600); err != nil { + t.Fatal(err) + } + if err := checksum.GenerateChecksums(context.Background(), dir, []string{payloadPath}); err != nil { + t.Fatal(err) + } + + fixture := []byte(`{"pre-verified":"binary-attestation"}`) + b := &DefaultBundler{ + Config: config.NewConfig( + config.WithIncludeChecksums(true), + config.WithAttest(true), + ), + Attester: &fixtureBinaryAttester{bundleJSON: []byte(`{"bundle":true}`)}, + verifiedBinaryAttestation: fixture, + } + + files, err := b.attestBundle(context.Background(), dir, nil, closedWorldRecipeResult()) + if err != nil { + t.Fatalf("attestBundle() error = %v", err) + } + if !slices.Contains(files, attestation.BinaryAttestationFile) { + t.Errorf("attestBundle() files = %v, want to contain %q", files, attestation.BinaryAttestationFile) + } + + embeddedPath := filepath.Join(dir, filepath.FromSlash(attestation.BinaryAttestationFile)) + got, err := os.ReadFile(embeddedPath) //nolint:gosec // test-controlled path + if err != nil { + t.Fatalf("reading embedded binary attestation: %v", err) + } + if !bytes.Equal(got, fixture) { + t.Errorf("embedded binary attestation = %q, want %q", got, fixture) + } +} + func TestNewWithConfig(t *testing.T) { t.Run("nil config uses default", func(t *testing.T) { bundler, err := NewWithConfig(nil) diff --git a/pkg/bundler/verifier/verifier.go b/pkg/bundler/verifier/verifier.go index f73e983da..32fbbcc6f 100644 --- a/pkg/bundler/verifier/verifier.go +++ b/pkg/bundler/verifier/verifier.go @@ -769,6 +769,19 @@ func VerifyBinaryAttestation(ctx context.Context, bundlePath string, identityPat if err != nil { return "", err // already coded by readBoundedFileContext } + return VerifyBinaryAttestationData(ctx, data, identityPattern, artifactDigest) +} + +// VerifyBinaryAttestationData verifies an already-read binary attestation +// (Sigstore bundle bytes) against the NVIDIA-CI identity pattern and the +// artifact digest. VerifyBinaryAttestation reads a file then delegates here; +// callers that already hold the bytes (e.g. the aicrd server, which caches and +// embeds them) call this directly to verify the exact content they will use, +// avoiding a verify-then-reread window. +func VerifyBinaryAttestationData(ctx context.Context, data []byte, identityPattern string, artifactDigest []byte) (string, error) { + if err := ctx.Err(); err != nil { + return "", errors.Wrap(errors.ErrCodeTimeout, "context cancelled before binary attestation verification", err) + } // Pin identity to NVIDIA CI using the provided pattern identity, err := verify.NewShortCertificateIdentity( diff --git a/pkg/client/v1/bundle.go b/pkg/client/v1/bundle.go index 35d77e1d1..0ecffd6df 100644 --- a/pkg/client/v1/bundle.go +++ b/pkg/client/v1/bundle.go @@ -67,6 +67,15 @@ type BundleOptions struct { // when --attest is passed. Attester BundleAttester + // BinaryAttestation, when non-empty, is a pre-verified binary attestation + // (Sigstore bundle bytes) embedded into attested bundles as tool provenance. + // The caller (e.g. the aicrd server, which verifies its in-image attestation + // once at startup) is responsible for having verified these bytes. Empty + // leaves the bundler's default per-run discover-and-verify path unchanged + // (the CLI passes nil and relies on the attestation shipped next to its + // install-script binary). + BinaryAttestation []byte + // OutputDir is the directory bundle files are written to. Empty // means the current directory ("."), matching Make's default. OutputDir string @@ -286,6 +295,9 @@ func (c *Client) MakeBundle(ctx context.Context, recipe *RecipeResult, opts Bund if opts.Attester != nil { bundlerOpts = append(bundlerOpts, bundler.WithAttester(opts.Attester)) } + if len(opts.BinaryAttestation) > 0 { + bundlerOpts = append(bundlerOpts, bundler.WithVerifiedBinaryAttestation(opts.BinaryAttestation)) + } b, err := bundler.New(bundlerOpts...) if err != nil { // Don't re-wrap — bundler.New returns structured errors with the diff --git a/pkg/client/v1/bundle_test.go b/pkg/client/v1/bundle_test.go index f94ba44e0..ce317830c 100644 --- a/pkg/client/v1/bundle_test.go +++ b/pkg/client/v1/bundle_test.go @@ -15,6 +15,7 @@ package aicr_test import ( + "bytes" "context" "errors" "os" @@ -24,6 +25,7 @@ import ( "time" "github.com/NVIDIA/aicr/pkg/bundler" + "github.com/NVIDIA/aicr/pkg/bundler/attestation" "github.com/NVIDIA/aicr/pkg/bundler/config" aicr "github.com/NVIDIA/aicr/pkg/client/v1" aicrerrors "github.com/NVIDIA/aicr/pkg/errors" @@ -294,6 +296,93 @@ func TestMakeBundle_TimeoutOptIn(t *testing.T) { }) } +// fixtureBundleAttester returns fixed, non-nil bundle JSON from Attest so the +// bundler reaches verifyAndCopyBinaryAttestation (a nil return short-circuits +// attestBundle before the binary attestation is embedded). It stands in for the +// real keyless attester the CLI/server build under --attest. +type fixtureBundleAttester struct { + bundleJSON []byte +} + +func (a *fixtureBundleAttester) Attest(_ context.Context, _ attestation.AttestSubject) ([]byte, error) { + return a.bundleJSON, nil +} + +func (a *fixtureBundleAttester) Identity() string { return "fixture" } + +func (a *fixtureBundleAttester) HasRekorEntry() bool { return false } + +// TestMakeBundle_InjectedBinaryAttestation proves that BundleOptions.BinaryAttestation +// bytes flow through MakeBundle into the produced bundle as the embedded tool +// provenance, and confirms the additivity contract: with no attestation enabled +// and no injected bytes, no attestation directory is emitted (the CLI/default path). +func TestMakeBundle_InjectedBinaryAttestation(t *testing.T) { + t.Parallel() + + embeddedPathOf := func(dir string) string { + return filepath.Join(dir, filepath.FromSlash(attestation.BinaryAttestationFile)) + } + + t.Run("injected bytes are embedded when attest enabled", func(t *testing.T) { + t.Parallel() + client, rec := resolveEmbeddedTrainingRecipe(t) + + fixture := []byte(`{"pre-verified":"binary-attestation"}`) + outDir := t.TempDir() + + out, err := client.MakeBundle(t.Context(), rec, aicr.BundleOptions{ + Config: config.NewConfig( + config.WithVersion("v-test"), + config.WithDeployer(config.DeployerHelm), + config.WithAttest(true), + ), + // A non-nil bundle JSON is required so attestBundle does not + // short-circuit before embedding the binary attestation. + Attester: &fixtureBundleAttester{bundleJSON: []byte(`{"bundle":true}`)}, + OutputDir: outDir, + BinaryAttestation: fixture, + }) + if err != nil { + t.Fatalf("MakeBundle: %v", err) + } + if out == nil || out.HasErrors() { + t.Fatalf("MakeBundle produced errors: %+v", out) + } + + got, err := os.ReadFile(embeddedPathOf(outDir)) + if err != nil { + t.Fatalf("reading embedded binary attestation %s: %v", attestation.BinaryAttestationFile, err) + } + if !bytes.Equal(got, fixture) { + t.Errorf("embedded binary attestation = %q, want %q", got, fixture) + } + }) + + t.Run("default path emits no attestation dir", func(t *testing.T) { + t.Parallel() + client, rec := resolveEmbeddedTrainingRecipe(t) + + outDir := t.TempDir() + out, err := client.MakeBundle(t.Context(), rec, aicr.BundleOptions{ + Config: config.NewConfig( + config.WithVersion("v-test"), + config.WithDeployer(config.DeployerHelm), + ), + OutputDir: outDir, + // No Attester, no BinaryAttestation, no --attest: the CLI default. + }) + if err != nil { + t.Fatalf("MakeBundle (default): %v", err) + } + if out == nil || out.HasErrors() { + t.Fatalf("MakeBundle (default) produced errors: %+v", out) + } + if _, statErr := os.Stat(embeddedPathOf(outDir)); !os.IsNotExist(statErr) { + t.Errorf("expected no binary attestation on default path, stat err = %v", statErr) + } + }) +} + func assertInvalidRequest(t *testing.T, err error) { t.Helper() if err == nil { diff --git a/pkg/defaults/timeouts.go b/pkg/defaults/timeouts.go index e9a734174..b758f1d40 100644 --- a/pkg/defaults/timeouts.go +++ b/pkg/defaults/timeouts.go @@ -853,6 +853,66 @@ const ( EnvServerShutdownTimeoutSeconds = "SHUTDOWN_TIMEOUT_SECONDS" ) +// Server-side bundle-signing configuration (see docs/plans/2026-07-20-server-bundle-attestation-design.md). +const ( + // EnvSigningKey selects KMS-backed (Mode A) signing. Value is a cosign + // KMS URI (awskms:// | gcpkms:// | azurekms:// | hashivault://). + EnvSigningKey = "AICR_SIGNING_KEY" + + // EnvFulcioURL is the private Fulcio CA endpoint; its presence (with a + // token source) selects keyless (Mode B) signing. + EnvFulcioURL = "AICR_FULCIO_URL" + + // EnvRekorURL overrides the Rekor transparency-log endpoint (both modes). + EnvRekorURL = "AICR_REKOR_URL" + + // EnvIdentityTokenFile is the path to the server's own OIDC token + // (projected ServiceAccount token, audience "sigstore") for Mode B. + // Read fresh per request. + EnvIdentityTokenFile = "AICR_IDENTITY_TOKEN_FILE" + + // EnvTLogUpload=false disables the Rekor upload for KMS (Mode A) signing + // (air-gapped). KMS-only; keyless always uploads. + EnvTLogUpload = "AICR_TLOG_UPLOAD" + + // EnvSigningConfigPath points signing at a Sigstore SigningConfig JSON + // (Rekor v2 targeting). Honored by both modes. + EnvSigningConfigPath = "AICR_SIGNING_CONFIG_PATH" + + // EnvGitHubActionsIDTokenRequestURL / …RequestToken are the GitHub Actions + // ambient OIDC endpoint env vars used for keyless (Mode B) signing when + // aicrd itself runs in a GitHub Actions job. + EnvGitHubActionsIDTokenRequestURL = "ACTIONS_ID_TOKEN_REQUEST_URL" + EnvGitHubActionsIDTokenRequestToken = "ACTIONS_ID_TOKEN_REQUEST_TOKEN" + + // EnvBinaryAttestationFile overrides the path to the server's own binary + // attestation (tool provenance). Unset falls back to the conventional + // -attestation.sigstore.json next to the running binary. Set it + // when the attestation ships elsewhere in the image (e.g. a ko build's + // KO_DATA_PATH: /var/run/ko/aicrd-attestation.sigstore.json). + EnvBinaryAttestationFile = "AICR_BINARY_ATTESTATION_FILE" + + // EnvKoDataPath is ko's runtime data directory env var (set automatically + // inside ko-built images). AICR ships the server's per-architecture binary + // attestation there so a multi-arch image needs no per-deployment config. + EnvKoDataPath = "KO_DATA_PATH" + + // BinaryAttestationKoDataNameFormat is the per-architecture filename for the + // aicrd binary attestation shipped in ko's KO_DATA_PATH. The %s is GOARCH + // (e.g. amd64, arm64). The .goreleaser.yaml aicrd build hook writes files + // with this exact convention into cmd/aicrd/kodata/; keep the two in sync. + BinaryAttestationKoDataNameFormat = "aicrd-%s-attestation.sigstore.json" + + // EnvBinaryAttestationIdentityRegexp overrides the certificate-identity + // pattern the server pins its own binary attestation to. Unset uses the + // release-workflow default (verifier.TrustedRepositoryPattern). A custom + // value MUST still contain "NVIDIA/aicr" (enforced by + // verifier.ValidateIdentityPattern) so it stays pinned to the NVIDIA org; + // it retargets WHICH NVIDIA workflow attested the binary (e.g. an e2e + // workflow), not the org. Mirrors the CLI's --certificate-identity-regexp. + EnvBinaryAttestationIdentityRegexp = "AICR_BINARY_ATTESTATION_IDENTITY_REGEXP" +) + // Log scanner buffer sizes. const ( // LogScannerBufferSize is the maximum line size for reading pod logs. diff --git a/pkg/server/bundle_handler.go b/pkg/server/bundle_handler.go index fa14f9f86..0a4ed0aa9 100644 --- a/pkg/server/bundle_handler.go +++ b/pkg/server/bundle_handler.go @@ -21,8 +21,11 @@ import ( "log/slog" "net/http" "os" + "strconv" "github.com/NVIDIA/aicr/pkg/bundler" + "github.com/NVIDIA/aicr/pkg/bundler/attestation" + bundlercfg "github.com/NVIDIA/aicr/pkg/bundler/config" "github.com/NVIDIA/aicr/pkg/bundler/result" aicr "github.com/NVIDIA/aicr/pkg/client/v1" "github.com/NVIDIA/aicr/pkg/defaults" @@ -30,6 +33,12 @@ import ( "github.com/NVIDIA/aicr/pkg/recipe" ) +// attesterBuilder constructs an Attester from resolve options. Injectable so +// tests can assert signing wiring without a real KMS/Fulcio backend. Defaults +// to attestation.ResolveAttester (eager: the keyless token is already in hand +// from the token-file read, so no deferred resolution is needed). +type attesterBuilder func(context.Context, attestation.ResolveOptions) (attestation.Attester, error) + var bundleZipResponseHeaders = []string{ "Content-Type", "Content-Disposition", @@ -54,15 +63,23 @@ type bundleHandler struct { // "Recipe criteria value not allowed" message) before bundling. The // Client's MakeBundle enforcement remains a backstop. allowLists *aicr.AllowLists + // signing holds the operator-configured server signing identity. When nil + // or disabled, an attest=true request is rejected with 400. No field here + // ever comes from a request. + signing *signingConfig + // newAttester builds an Attester from resolve options. Injectable for tests. + newAttester attesterBuilder } -// newBundleHandler constructs a bundleHandler bound to the given client and -// allowlists. -func newBundleHandler(client *aicr.Client, allowLists *aicr.AllowLists) *bundleHandler { +// newBundleHandler constructs a bundleHandler bound to the given client, +// allowlists, and server signing identity. +func newBundleHandler(client *aicr.Client, allowLists *aicr.AllowLists, signing *signingConfig) *bundleHandler { return &bundleHandler{ - client: client, - allowLists: allowLists, - streamZip: bundler.StreamZipResponseContext, + client: client, + allowLists: allowLists, + streamZip: bundler.StreamZipResponseContext, + signing: signing, + newAttester: attestation.ResolveAttester, } } @@ -95,6 +112,13 @@ func (h *bundleHandler) HandleBundles(w http.ResponseWriter, r *http.Request) { return } + // Opt-in signing (?attest=true), parsed and validated up front so a bad + // value or an unconfigured server is rejected before any bundle work. + attestRequested, handled := h.resolveAttestRequest(w, r) + if handled { + return + } + // Parse request body directly as RecipeResult. Bound the body to defend // against memory exhaustion (same cap as the legacy handler). bounded := http.MaxBytesReader(w, r.Body, defaults.MaxBundlePOSTBytes) @@ -164,11 +188,44 @@ func (h *bundleHandler) HandleBundles(w http.ResponseWriter, r *http.Request) { // boundary explicitly. The outer ServerHandlerTimeout middleware is // now sized ≥ BundleHandlerTimeout so this 60s deadline runs to // completion instead of being silently clamped. - output, err := h.client.MakeBundle(ctx, adopted, aicr.BundleOptions{ + bundleOpts := aicr.BundleOptions{ Config: bundleConfig, OutputDir: tempDir, Timeout: defaults.BundleHandlerTimeout, - }) + } + + // Opt-in signing via ?attest=true. The server signs as itself using the + // operator-configured identity (KMS or private-Sigstore keyless); no + // identity material comes from the request. The "not configured" rejection + // and the attest parse already ran above; here we build only the attester, + // which reads the identity token fresh (SA tokens rotate) and so cannot be + // hoisted out of the request path. + if attestRequested { + resolveOpts, buildErr := h.signing.resolveOptions() + if buildErr != nil { + logger.Error("failed to prepare signing options", "error", buildErr) + WriteError(w, r, http.StatusInternalServerError, aicrerrors.ErrCodeInternal, + "Failed to prepare signing", false, nil) + return + } + attester, attErr := h.newAttester(ctx, resolveOpts) + if attErr != nil { + logger.Error("failed to construct attester", "error", attErr) + WriteError(w, r, http.StatusInternalServerError, aicrerrors.ErrCodeInternal, + "Failed to initialize signing", false, nil) + return + } + + // Enable attestation on this request's config (mirrors the CLI's + // config.WithAttest) and embed the server's pre-verified binary + // attestation as tool provenance. The bytes were verified ONCE at + // startup and cached on the signing config. + bundlercfg.WithAttest(true)(bundleConfig) + bundleOpts.Attester = attester + bundleOpts.BinaryAttestation = h.signing.binaryAttestation + } + + output, err := h.client.MakeBundle(ctx, adopted, bundleOpts) if err != nil { WriteErrorFromErr(w, r, err, "Failed to generate bundle", nil) return @@ -197,6 +254,38 @@ func (h *bundleHandler) HandleBundles(w http.ResponseWriter, r *http.Request) { h.writeZipResponse(ctx, w, r, tempDir, output) } +// resolveAttestRequest parses the ?attest query parameter and validates it +// against the server's static signing config. It returns attestRequested and a +// handled flag: when handled is true the function has already written the HTTP +// response (a 400) and the caller must return without further processing. +// +// Absent/empty attest means no signing. A present-but-unparseable value is a +// client error (a typo must not silently ship an unsigned bundle) — mirrors +// parseTLogUpload / getSetEnabledOverride. The "not configured" rejection is +// hoisted here so it fails fast: it depends only on the query param and static +// server config, never on request-body or bundle work. +func (h *bundleHandler) resolveAttestRequest(w http.ResponseWriter, r *http.Request) (attestRequested, handled bool) { + if v := r.URL.Query().Get("attest"); v != "" { + parsed, perr := strconv.ParseBool(v) + if perr != nil { + WriteError(w, r, http.StatusBadRequest, aicrerrors.ErrCodeInvalidRequest, + "Invalid attest parameter (want true or false)", false, map[string]any{ + keyError: v, + }) + return false, true + } + attestRequested = parsed + } + + if attestRequested && (h.signing == nil || !h.signing.enabled) { + WriteError(w, r, http.StatusBadRequest, aicrerrors.ErrCodeInvalidRequest, + "Server is not configured for attestation", false, nil) + return false, true + } + + return attestRequested, false +} + func (h *bundleHandler) writeZipResponse( ctx context.Context, w http.ResponseWriter, diff --git a/pkg/server/bundle_handler_test.go b/pkg/server/bundle_handler_test.go index efe1b2daf..e80ec673b 100644 --- a/pkg/server/bundle_handler_test.go +++ b/pkg/server/bundle_handler_test.go @@ -15,18 +15,39 @@ package server import ( + "archive/zip" + "bytes" "context" "encoding/json" + "io" "net/http" "net/http/httptest" + "path/filepath" "strings" "testing" + "github.com/NVIDIA/aicr/pkg/bundler/attestation" "github.com/NVIDIA/aicr/pkg/bundler/result" aicr "github.com/NVIDIA/aicr/pkg/client/v1" aicrerrors "github.com/NVIDIA/aicr/pkg/errors" ) +// fixtureBundleAttester returns fixed, non-nil bundle JSON from Attest so the +// bundler does not short-circuit attestBundle (a nil return skips embedding the +// binary attestation). It stands in for the real keyless/KMS attester the +// server builds under ?attest=true. +type fixtureBundleAttester struct { + bundleJSON []byte +} + +func (a *fixtureBundleAttester) Attest(_ context.Context, _ attestation.AttestSubject) ([]byte, error) { + return a.bundleJSON, nil +} + +func (a *fixtureBundleAttester) Identity() string { return "fixture" } + +func (a *fixtureBundleAttester) HasRekorEntry() bool { return false } + var testBundleZipHeaders = []string{ "Content-Disposition", "X-Bundle-Files", @@ -41,7 +62,256 @@ func newTestBundleHandler(t *testing.T) *bundleHandler { t.Fatalf("NewClient: %v", err) } t.Cleanup(func() { _ = client.Close() }) - return newBundleHandler(client, nil) + return newBundleHandler(client, nil, nil) +} + +// resolveEmbeddedBundleBody resolves a known-good embedded recipe and returns +// its wire-format JSON (the pkg/recipe.RecipeResult shape the bundle handler +// decodes), so attest tests can drive a real, successful bundle end-to-end. +func resolveEmbeddedBundleBody(t *testing.T) []byte { + t.Helper() + client, err := aicr.NewClient( + aicr.WithRecipeSource(aicr.EmbeddedSource()), + aicr.WithVersion("v-test"), + ) + if err != nil { + t.Fatalf("NewClient: %v", err) + } + t.Cleanup(func() { _ = client.Close() }) + + rec, err := client.ResolveRecipe(t.Context(), aicr.RecipeRequest{ + Service: "eks", + Accelerator: "h100", + OS: "ubuntu", + Intent: "training", + }) + if err != nil { + t.Fatalf("ResolveRecipe: %v", err) + } + body, err := json.Marshal(rec.Resolved()) + if err != nil { + t.Fatalf("marshal recipe: %v", err) + } + return body +} + +// TestBundleHandler_Attest pins the ?attest=true signing seam: a configured +// server signs via the injected attesterBuilder, an unconfigured server rejects +// the request with 400, and the default (no attest) path never touches signing. +func TestBundleHandler_Attest(t *testing.T) { + const kmsKey = "awskms:///alias/aicr-signing" + + body := resolveEmbeddedBundleBody(t) + + newHandler := func(t *testing.T, signing *signingConfig, builder attesterBuilder) *bundleHandler { + t.Helper() + h := newTestBundleHandler(t) + h.signing = signing + if builder != nil { + h.newAttester = builder + } + return h + } + + post := func(h *bundleHandler, target string) *httptest.ResponseRecorder { + req := httptest.NewRequest(http.MethodPost, target, bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + h.HandleBundles(w, req) + return w + } + + t.Run("attest=true wires configured KMS signing", func(t *testing.T) { + var called bool + var gotOpts attestation.ResolveOptions + h := newHandler(t, + // binaryAttestation is required now that attest=true enables + // Config.Attest(): bundler.New's fail-fast gate is satisfied by the + // injected bytes (the startup-verified tool provenance). + &signingConfig{ + enabled: true, + signingKey: kmsKey, + tlogUpload: true, + binaryAttestation: []byte("fixture-attestation"), + }, + func(_ context.Context, opts attestation.ResolveOptions) (attestation.Attester, error) { + called = true + gotOpts = opts + return attestation.NewNoOpAttester(), nil + }) + + w := post(h, "/v1/bundle?attest=true") + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d. Body: %s", w.Code, http.StatusOK, w.Body.String()) + } + if !called { + t.Fatal("newAttester was not called for attest=true") + } + if !gotOpts.Attest { + t.Error("ResolveOptions.Attest = false, want true") + } + if gotOpts.SigningKey != kmsKey { + t.Errorf("ResolveOptions.SigningKey = %q, want %q", gotOpts.SigningKey, kmsKey) + } + }) + + t.Run("attest=true fails 500 without leaking builder error", func(t *testing.T) { + const secretErr = "super-secret-kms-internal-failure-detail" + h := newHandler(t, + &signingConfig{enabled: true, signingKey: kmsKey, tlogUpload: true}, + func(context.Context, attestation.ResolveOptions) (attestation.Attester, error) { + return nil, aicrerrors.New(aicrerrors.ErrCodeInternal, secretErr) + }) + + w := post(h, "/v1/bundle?attest=true") + + if w.Code != http.StatusInternalServerError { + t.Fatalf("status = %d, want %d. Body: %s", w.Code, http.StatusInternalServerError, w.Body.String()) + } + if strings.Contains(w.Body.String(), secretErr) { + t.Errorf("response leaked internal builder error: %s", w.Body.String()) + } + }) + + t.Run("attest=true fails 500 when identity token file is missing", func(t *testing.T) { + // Use the DEFAULT newAttester (no injected builder) so the real + // resolveOptions runs and fails reading the non-existent token file. + h := newHandler(t, &signingConfig{ + enabled: true, + keyless: true, + fulcioURL: "https://fulcio.example", + identityTokenFile: filepath.Join(t.TempDir(), "does-not-exist.token"), + }, nil) + + w := post(h, "/v1/bundle?attest=true") + + if w.Code != http.StatusInternalServerError { + t.Fatalf("status = %d, want %d. Body: %s", w.Code, http.StatusInternalServerError, w.Body.String()) + } + }) + + t.Run("attest=notabool rejected with 400", func(t *testing.T) { + h := newHandler(t, + &signingConfig{enabled: true, signingKey: kmsKey, tlogUpload: true}, + nil) + + w := post(h, "/v1/bundle?attest=notabool") + + if w.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want %d. Body: %s", w.Code, http.StatusBadRequest, w.Body.String()) + } + }) + + t.Run("attest=true rejected when not configured", func(t *testing.T) { + cases := []struct { + name string + signing *signingConfig + }{ + {"nil signing", nil}, + {"disabled signing", &signingConfig{enabled: false, signingKey: kmsKey}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var called bool + h := newHandler(t, tc.signing, + func(context.Context, attestation.ResolveOptions) (attestation.Attester, error) { + called = true + return attestation.NewNoOpAttester(), nil + }) + + w := post(h, "/v1/bundle?attest=true") + + if w.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want %d. Body: %s", w.Code, http.StatusBadRequest, w.Body.String()) + } + if called { + t.Error("newAttester was called despite signing being unconfigured") + } + }) + } + }) + + t.Run("attest absent leaves signing untouched", func(t *testing.T) { + var called bool + h := newHandler(t, + &signingConfig{enabled: true, signingKey: kmsKey, tlogUpload: true}, + func(context.Context, attestation.ResolveOptions) (attestation.Attester, error) { + called = true + return attestation.NewNoOpAttester(), nil + }) + + w := post(h, "/v1/bundle") + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d. Body: %s", w.Code, http.StatusOK, w.Body.String()) + } + if called { + t.Error("newAttester was called for a request without attest=true") + } + }) +} + +// TestBundleHandler_AttestEmbedsBinaryAttestation is the end-to-end proof that +// server signing embeds the cached tool provenance: with signing enabled, a +// non-empty binaryAttestation, and a real (non-nil JSON) attester, the streamed +// bundle zip contains attestation/aicr-attestation.sigstore.json equal to the +// cached binary attestation bytes. The zip staging step re-verifies checksums +// (not Sigstore signatures), so a fixture attestation survives the stream path. +func TestBundleHandler_AttestEmbedsBinaryAttestation(t *testing.T) { + const kmsKey = "awskms:///alias/aicr-signing" + fixtureBinary := []byte(`{"pre-verified":"server-binary-attestation"}`) + body := resolveEmbeddedBundleBody(t) + + h := newTestBundleHandler(t) + h.signing = &signingConfig{ + enabled: true, + signingKey: kmsKey, + tlogUpload: true, + binaryAttestation: fixtureBinary, + } + // A non-nil bundle JSON is required so attestBundle does not short-circuit + // before embedding the binary attestation. + h.newAttester = func(context.Context, attestation.ResolveOptions) (attestation.Attester, error) { + return &fixtureBundleAttester{bundleJSON: []byte(`{"bundle":true}`)}, nil + } + + req := httptest.NewRequest(http.MethodPost, "/v1/bundle?attest=true", bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + h.HandleBundles(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d. Body: %s", w.Code, http.StatusOK, w.Body.String()) + } + + zr, err := zip.NewReader(bytes.NewReader(w.Body.Bytes()), int64(w.Body.Len())) + if err != nil { + t.Fatalf("open bundle zip: %v", err) + } + + var got []byte + for _, f := range zr.File { + if f.Name != attestation.BinaryAttestationFile { + continue + } + rc, openErr := f.Open() + if openErr != nil { + t.Fatalf("open %s in zip: %v", f.Name, openErr) + } + got, err = io.ReadAll(rc) + _ = rc.Close() + if err != nil { + t.Fatalf("read %s in zip: %v", f.Name, err) + } + break + } + if got == nil { + t.Fatalf("bundle zip missing %s", attestation.BinaryAttestationFile) + } + if !bytes.Equal(got, fixtureBinary) { + t.Errorf("embedded binary attestation = %q, want %q", got, fixtureBinary) + } } // TestBundleHandler_MethodGate verifies only POST is accepted. diff --git a/pkg/server/serve.go b/pkg/server/serve.go index a3f443121..8b94f3a83 100644 --- a/pkg/server/serve.go +++ b/pkg/server/serve.go @@ -105,10 +105,31 @@ func Serve() error { }() h := newRecipeHandler(client, allowLists) + // Parse the operator-configured server signing identity from the + // environment. Fails fast on ambiguous/incomplete configuration so a + // misconfigured server never starts. + signing, err := parseSigningConfig() + if err != nil { + // parseSigningConfig already returns coded errors; propagate rather + // than re-wrap (avoids double-wrapping an already-classified error). + return errors.PropagateOrWrap(err, errors.ErrCodeInternal, "failed to parse signing configuration") + } + + // When signing is enabled, verify and cache the server's own binary + // attestation ONCE at startup (tool provenance embedded into every attested + // bundle). Fail-fast: a signing server that cannot prove its own provenance + // must not start. No-op when signing is disabled. + if err := signing.loadBinaryAttestation(ctx, defaultBinaryAttestationVerifier); err != nil { + return err + } + if signing.enabled { + slog.Info("server bundle signing enabled", "keyless", signing.keyless) + } + // Setup bundle handler backed by the same aicr.Client facade. server.go // no longer constructs a bundler.Bundler (or a recipe.Builder) directly — // the Client owns both, completing #1077 acceptance criterion #2. - bh := newBundleHandler(client, allowLists) + bh := newBundleHandler(client, allowLists, signing) r := map[string]http.HandlerFunc{ "/v1/recipe": h.HandleRecipes, diff --git a/pkg/server/signing.go b/pkg/server/signing.go new file mode 100644 index 000000000..8c9c8b086 --- /dev/null +++ b/pkg/server/signing.go @@ -0,0 +1,283 @@ +// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "context" + "fmt" + "io" + "log/slog" + "os" + "path/filepath" + "runtime" + "strconv" + "strings" + + "github.com/NVIDIA/aicr/pkg/bundler/attestation" + "github.com/NVIDIA/aicr/pkg/bundler/checksum" + "github.com/NVIDIA/aicr/pkg/bundler/verifier" + "github.com/NVIDIA/aicr/pkg/defaults" + aicrerrors "github.com/NVIDIA/aicr/pkg/errors" +) + +// kmsURISchemes are the cosign KMS URI prefixes accepted for Mode A signing. +var kmsURISchemes = []string{"awskms://", "gcpkms://", "azurekms://", "hashivault://"} + +// signingConfig holds the validated server signing identity parsed from the +// environment. The server signs as itself: no field here ever comes from a +// request. enabled reports whether any signing mode is configured; when false +// an attest=true request is rejected. keyless reports Mode B (token file read +// fresh per request in resolveOptions). +type signingConfig struct { + enabled bool + keyless bool + + signingKey string // Mode A KMS URI + fulcioURL string // Mode B + rekorURL string // both + identityTokenFile string // Mode B token source + ambientURL string // Mode B GHA ambient (optional) + ambientToken string + tlogUpload bool // Mode A only; default true + signingConfigPath string + + // binaryAttestation is the server's own pre-verified binary attestation + // (tool provenance), loaded and cryptographically verified ONCE at startup + // and embedded into every attested bundle. Populated by loadBinaryAttestation + // when signing is enabled. + binaryAttestation []byte +} + +// parseSigningConfig reads the AICR_* signing env vars, validates mode +// exclusivity and completeness, and fails fast on ambiguous or malformed +// configuration so a misconfigured server does not start. +func parseSigningConfig() (*signingConfig, error) { + cfg := &signingConfig{ + signingKey: os.Getenv(defaults.EnvSigningKey), + fulcioURL: os.Getenv(defaults.EnvFulcioURL), + rekorURL: os.Getenv(defaults.EnvRekorURL), + identityTokenFile: os.Getenv(defaults.EnvIdentityTokenFile), + ambientURL: os.Getenv(defaults.EnvGitHubActionsIDTokenRequestURL), + ambientToken: os.Getenv(defaults.EnvGitHubActionsIDTokenRequestToken), + signingConfigPath: os.Getenv(defaults.EnvSigningConfigPath), + // tlogUpload is KMS-only; parsed in the KMS branch below so a stray + // malformed AICR_TLOG_UPLOAD does not fail startup for keyless or + // unconfigured servers where it has no effect. + } + + hasKMS := cfg.signingKey != "" + hasKeyless := cfg.fulcioURL != "" || cfg.identityTokenFile != "" + + switch { + case !hasKMS && !hasKeyless: + return cfg, nil // capability off; server starts unsigned + case hasKMS && hasKeyless: + return nil, aicrerrors.New(aicrerrors.ErrCodeInternal, + "ambiguous signing identity: set either "+defaults.EnvSigningKey+ + " (KMS) or "+defaults.EnvFulcioURL+"/"+defaults.EnvIdentityTokenFile+ + " (keyless), not both") + case hasKMS: + if !hasValidKMSScheme(cfg.signingKey) { + return nil, aicrerrors.New(aicrerrors.ErrCodeInternal, + "invalid "+defaults.EnvSigningKey+": must be a cosign KMS URI "+ + "(awskms:// | gcpkms:// | azurekms:// | hashivault://)") + } + tlogUpload, err := parseTLogUpload() + if err != nil { + return nil, err + } + cfg.tlogUpload = tlogUpload + cfg.enabled = true + default: // keyless + if cfg.fulcioURL == "" { + return nil, aicrerrors.New(aicrerrors.ErrCodeInternal, + "keyless signing requires "+defaults.EnvFulcioURL) + } + if cfg.identityTokenFile == "" && (cfg.ambientURL == "" || cfg.ambientToken == "") { + return nil, aicrerrors.New(aicrerrors.ErrCodeInternal, + "keyless signing requires a token source: "+defaults.EnvIdentityTokenFile+ + " or GitHub Actions ambient OIDC env") + } + cfg.enabled = true + cfg.keyless = true + } + return cfg, nil +} + +// parseTLogUpload interprets AICR_TLOG_UPLOAD (Mode A Rekor upload toggle). +// Unset or empty defaults to true; otherwise the value is parsed with +// strconv.ParseBool (accepts 1/t/T/TRUE/true/True and 0/f/F/FALSE/false/False). +// A non-empty, unparseable value is a server-side operator/config fault and +// fails startup fast rather than silently defaulting. +func parseTLogUpload() (bool, error) { + v := os.Getenv(defaults.EnvTLogUpload) + if v == "" { + return true, nil + } + b, err := strconv.ParseBool(v) + if err != nil { + return false, aicrerrors.New(aicrerrors.ErrCodeInternal, + "invalid "+defaults.EnvTLogUpload+": must be a boolean "+ + "(true/false), got "+strconv.Quote(v)) + } + return b, nil +} + +// hasValidKMSScheme reports whether uri carries one of the accepted cosign KMS +// URI prefixes. +func hasValidKMSScheme(uri string) bool { + for _, s := range kmsURISchemes { + if strings.HasPrefix(uri, s) { + return true + } + } + return false +} + +// resolveOptions builds a per-request attestation.ResolveOptions with Attest +// set. For keyless it reads the identity-token file FRESH each call: SA tokens +// rotate and Fulcio binds the cert to a fresh token, so the attester must be +// rebuilt per request rather than cached. +func (c *signingConfig) resolveOptions() (attestation.ResolveOptions, error) { + o := attestation.ResolveOptions{ + Attest: true, + RekorURL: c.rekorURL, + SigningConfigPath: c.signingConfigPath, + } + if c.keyless { + o.FulcioURL = c.fulcioURL + o.AmbientURL = c.ambientURL + o.AmbientToken = c.ambientToken + if c.identityTokenFile != "" { + tok, err := os.ReadFile(c.identityTokenFile) //nolint:gosec // operator-configured path + if err != nil { + return attestation.ResolveOptions{}, aicrerrors.Wrap(aicrerrors.ErrCodeInternal, + "failed to read identity token file", err) + } + o.IdentityToken = strings.TrimSpace(string(tok)) + } + return o, nil + } + o.SigningKey = c.signingKey + o.DisableTLogUpload = !c.tlogUpload + return o, nil +} + +// binaryAttestationVerifier resolves and cryptographically verifies the running +// server binary's attestation (tool provenance), returning the raw Sigstore +// bundle bytes to embed in signed bundles. Injectable so tests can supply a +// fixture: the real verification pins NVIDIA-CI identity + the running binary's +// digest, which cannot be satisfied by a `go test` executable. +type binaryAttestationVerifier func(ctx context.Context) ([]byte, error) + +// resolveBinaryAttestationPath returns the path to the server's binary +// attestation, in precedence order: the AICR_BINARY_ATTESTATION_FILE operator +// override, then ko's per-architecture attestation in KO_DATA_PATH (auto-set +// inside ko-built images), then the conventional file next to the running +// executable (direct-binary / CLI-style deployments). +func resolveBinaryAttestationPath(binPath string) (string, error) { + // 1. Explicit operator override. + if p := os.Getenv(defaults.EnvBinaryAttestationFile); p != "" { + return p, nil + } + // 2. ko image: per-architecture attestation shipped in KO_DATA_PATH. ko sets + // this env automatically inside the image, so no deployment config is needed. + if koData := os.Getenv(defaults.EnvKoDataPath); koData != "" { + name := fmt.Sprintf(defaults.BinaryAttestationKoDataNameFormat, runtime.GOARCH) + return filepath.Join(koData, name), nil + } + // 3. Conventional file next to the running binary (direct-binary / CLI-style). + return attestation.FindBinaryAttestation(binPath) +} + +// resolveBinaryAttestationIdentityPattern returns the certificate-identity +// pattern used to verify the server's own binary attestation: the +// AICR_BINARY_ATTESTATION_IDENTITY_REGEXP override when set (validated to stay +// pinned to the NVIDIA org), else the release-workflow default. A custom pattern +// means bundles this server signs will not pass a verifier using the default +// identity, so it is logged. +func resolveBinaryAttestationIdentityPattern() (string, error) { + p := os.Getenv(defaults.EnvBinaryAttestationIdentityRegexp) + if p == "" { + return verifier.TrustedRepositoryPattern, nil + } + if err := verifier.ValidateIdentityPattern(p); err != nil { + return "", err // already coded (ErrCodeInvalidRequest) + } + slog.Warn("using custom binary-attestation identity pattern; bundles this server signs "+ + "will not pass verification under the default identity", "pattern", p) + return p, nil +} + +// defaultBinaryAttestationVerifier locates the attestation next to the running +// executable (the in-image convention), verifies it against the NVIDIA-CI +// identity pattern and the binary's own digest, and returns its bytes. +func defaultBinaryAttestationVerifier(ctx context.Context) ([]byte, error) { + binPath, err := os.Executable() + if err != nil { + return nil, aicrerrors.Wrap(aicrerrors.ErrCodeInternal, "could not resolve executable path", err) + } + attestPath, err := resolveBinaryAttestationPath(binPath) + if err != nil { + return nil, err // already coded (ErrCodeNotFound when absent) + } + identityPattern, err := resolveBinaryAttestationIdentityPattern() + if err != nil { + return nil, err // already coded (ErrCodeInvalidRequest); a bad override fails startup fast + } + digest, err := checksum.SHA256RawContext(ctx, binPath) + if err != nil { + return nil, aicrerrors.PropagateOrWrap(err, aicrerrors.ErrCodeInternal, "failed to compute binary digest") + } + // Read the attestation once (bounded: os.Open + LimitReader so an oversized + // or symlink-swapped path cannot allocate the whole file before the size + // check), then verify the EXACT bytes we return. Verifying the in-hand bytes + // (rather than re-reading after a path-based verify) removes any + // verify-then-reread window. + f, err := os.Open(attestPath) //nolint:gosec // in-image, operator-trusted path already resolved by resolveBinaryAttestationPath + if err != nil { + return nil, aicrerrors.Wrap(aicrerrors.ErrCodeInternal, "failed to open binary attestation", err) + } + defer func() { _ = f.Close() }() + data, err := io.ReadAll(io.LimitReader(f, defaults.MaxAttestationFileBytes+1)) + if err != nil { + return nil, aicrerrors.Wrap(aicrerrors.ErrCodeInternal, "failed to read binary attestation", err) + } + if int64(len(data)) > defaults.MaxAttestationFileBytes { + return nil, aicrerrors.New(aicrerrors.ErrCodeInvalidRequest, "binary attestation exceeds size limit") + } + if _, verifyErr := verifier.VerifyBinaryAttestationData(ctx, data, identityPattern, digest); verifyErr != nil { + return nil, verifyErr // already coded + } + return data, nil +} + +// loadBinaryAttestation verifies and caches the server's binary attestation onto +// the signingConfig. Fail-fast: an enabled signing server that cannot prove its +// own provenance must not start. No-op when signing is disabled. +func (c *signingConfig) loadBinaryAttestation(ctx context.Context, verify binaryAttestationVerifier) error { + if !c.enabled { + return nil + } + data, err := verify(ctx) + if err != nil { + // PropagateOrWrap preserves an already-coded verify error (e.g. + // ErrCodeNotFound when the attestation is absent) instead of + // reclassifying it to ErrCodeInternal. + return aicrerrors.PropagateOrWrap(err, aicrerrors.ErrCodeInternal, "server signing enabled but binary attestation verification failed") + } + c.binaryAttestation = data + return nil +} diff --git a/pkg/server/signing_integration_test.go b/pkg/server/signing_integration_test.go new file mode 100644 index 000000000..3b02e2cf7 --- /dev/null +++ b/pkg/server/signing_integration_test.go @@ -0,0 +1,227 @@ +// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build integration + +// This build-tagged integration test proves the aicrd server produces a REAL +// KMS-signed bundle end-to-end. It is excluded from the default build (and thus +// from `make test`) by the `integration` tag; run it explicitly with +// `go test -tags integration ...` against a live OpenBAO Transit KMS. +// +// Unlike the unit tests in bundle_handler_test.go (which inject a stub +// attesterBuilder), this test leaves h.newAttester at its DEFAULT +// (attestation.ResolveAttester), so the server performs REAL hashivault signing +// via the sigstore KMS provider. Only bundler.New's binary-attestation gate is +// bypassed with fixture bytes: the real NVIDIA-CI binary attestation cannot +// exist inside a `go test` executable, but the KMS signing over the bundle +// checksums is genuine and is verified back with the same key. +// +// Provision the backing infra the way tests/chainsaw/signing/bundle-attestation-vault/run.sh +// does (openbao/openbao:2.6.0 dev mode + Transit ecdsa-p256 key), then: +// +// export VAULT_ADDR=http://127.0.0.1:8200 VAULT_TOKEN=root VAULT_KMS_KEY=aicr +// go test -tags integration -run TestServerSigning ./pkg/server/ -v +package server + +import ( + "archive/zip" + "bytes" + "context" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/NVIDIA/aicr/pkg/bundler/attestation" + "github.com/NVIDIA/aicr/pkg/bundler/verifier" +) + +// serverSigningIntegrationTimeout bounds the whole flow: real KMS round-trips +// (sign on the POST, GetPublicKey + verify on the verifier) plus bundle +// generation. Generous so a slow local OpenBAO does not flake the test. +const serverSigningIntegrationTimeout = 2 * time.Minute + +// TestServerSigning_KMSBundleEndToEnd drives POST /v1/bundle?attest=true against +// a REAL OpenBAO Transit KMS and proves the returned zip carries a genuine +// KMS-signed bundle attestation: +// +// 1. The server builds the KMS attester itself (default newAttester = +// attestation.ResolveAttester) from AICR_SIGNING_KEY=hashivault://. +// 2. The streamed zip contains a structurally valid Sigstore bundle at +// attestation/bundle-attestation.sigstore.json. +// 3. That bundle is a REAL signature over the bundle's checksums.txt: the Go +// verifier (verifier.Verify with the same hashivault Key) re-derives the +// checksums digest and cryptographically verifies the signature against the +// KMS public key, yielding BundleAttested=true. +// 4. attestation/aicr-attestation.sigstore.json equals the injected binary +// attestation fixture bytes (the startup-verified tool provenance the server +// embeds verbatim). +func TestServerSigning_KMSBundleEndToEnd(t *testing.T) { + vaultAddr := os.Getenv("VAULT_ADDR") + kmsKeyName := os.Getenv("VAULT_KMS_KEY") + if vaultAddr == "" || kmsKeyName == "" { + t.Skip("integration: set VAULT_ADDR + VAULT_KMS_KEY (run " + + "tests/chainsaw/signing/bundle-attestation-vault/run.sh or a local OpenBAO) to run") + } + + // The sigstore hashivault provider reads VAULT_ADDR / VAULT_TOKEN from the + // process environment; nothing else in this test performs network I/O. + kmsURI := "hashivault://" + kmsKeyName + + // Fixture stand-in for the server's pre-verified binary attestation. The real + // NVIDIA-CI provenance cannot exist in a `go test` binary, and bundler.New's + // Config.Attest() gate only requires the bytes to be present (verified once at + // startup in production), so arbitrary JSON satisfies the gate. This ONLY + // bypasses the binary-attestation gate; the bundle-attestation KMS signing is + // real. These exact bytes must round-trip into the zip unchanged. + fixtureBinaryAttestation := []byte(`{"pre-verified":"server-binary-attestation"}`) + + ctx, cancel := context.WithTimeout(context.Background(), serverSigningIntegrationTimeout) + defer cancel() + + body := resolveEmbeddedBundleBody(t) + + // Build the REAL bundle handler with an embedded-recipe Client, then attach + // an operator-style KMS signing identity. h.newAttester stays at its default + // (attestation.ResolveAttester), so the server signs via the live KMS. + h := newTestBundleHandler(t) + h.signing = &signingConfig{ + enabled: true, + signingKey: kmsURI, + tlogUpload: false, // no transparency log against a local OpenBAO + binaryAttestation: fixtureBinaryAttestation, + } + + req := httptest.NewRequestWithContext(ctx, http.MethodPost, "/v1/bundle?attest=true", bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + h.HandleBundles(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d. Body: %s", w.Code, http.StatusOK, w.Body.String()) + } + + // Extract the returned zip to disk so the file-based verifier can re-stage and + // verify it exactly as an operator's `aicr verify` would. + bundleDir := t.TempDir() + extractZip(t, w.Body.Bytes(), bundleDir) + + bundleAttestPath := filepath.Join(bundleDir, filepath.FromSlash(attestation.BundleAttestationFile)) + bundleAttestBytes := readFile(t, bundleAttestPath) + + // (2) Structural proof: the emitted file is a valid Sigstore bundle. + if err := attestation.ValidateSigstoreBundleData(bundleAttestBytes); err != nil { + t.Fatalf("bundle attestation is not a structurally valid sigstore bundle: %v", err) + } + // Belt-and-suspenders: a real signature is non-empty. A no-op/empty bundle + // would still parse, so guard against a hollow artifact even before the + // cryptographic check below. + if !bytes.Contains(bundleAttestBytes, []byte("messageSignature")) && + !bytes.Contains(bundleAttestBytes, []byte("dsseEnvelope")) { + + t.Errorf("bundle attestation carries neither messageSignature nor dsseEnvelope; got: %s", bundleAttestBytes) + } + + // (3) Cryptographic proof: verify the KMS signature over the bundle checksums + // with the SAME hashivault key. IgnoreTLog=true because the server signed with + // tlogUpload=false (no transparency-log entry); IgnoreTLog is only valid with + // Key set, which it is. A KMS Key URI still makes a live GetPublicKey call, so + // this exercises the real key material, not a cached PEM. + res, err := verifier.Verify(ctx, bundleDir, &verifier.VerifyOptions{ + Key: kmsURI, + IgnoreTLog: true, + }) + if err != nil { + t.Fatalf("verifier.Verify returned error: %v", err) + } + if !res.BundleAttested { + t.Fatalf("bundle attestation did not verify against the KMS key (BundleAttested=false); "+ + "trust=%s errors=%v", res.TrustLevel, res.Errors) + } + if res.ChecksumFiles == 0 { + t.Errorf("expected checksum-verified payload files, got ChecksumFiles=0") + } + // The binary attestation is a fixture (not NVIDIA-CI signed), so the FULL + // chain caps at TrustUnknown after the binary step fails. That does not + // weaken the KMS proof above: BundleAttested is set only after the real + // signature-over-checksums verification succeeds. Log for visibility. + t.Logf("verify result: trust=%s bundleAttested=%t bundleCreator=%q checksumFiles=%d", + res.TrustLevel, res.BundleAttested, res.BundleCreator, res.ChecksumFiles) + + // (4) The embedded binary attestation is the injected fixture, byte-for-byte. + binaryAttestPath := filepath.Join(bundleDir, filepath.FromSlash(attestation.BinaryAttestationFile)) + gotBinary := readFile(t, binaryAttestPath) + if !bytes.Equal(gotBinary, fixtureBinaryAttestation) { + t.Errorf("embedded binary attestation = %q, want %q", gotBinary, fixtureBinaryAttestation) + } +} + +// extractZip writes every entry of the in-memory zip archive under destDir, +// recreating the directory structure and rejecting zip-slip paths that would +// escape destDir. +func extractZip(t *testing.T, archive []byte, destDir string) { + t.Helper() + zr, err := zip.NewReader(bytes.NewReader(archive), int64(len(archive))) + if err != nil { + t.Fatalf("open bundle zip: %v", err) + } + for _, f := range zr.File { + target := filepath.Join(destDir, filepath.FromSlash(f.Name)) + // Guard against zip-slip: the cleaned target must stay within destDir. + if !strings.HasPrefix(target, filepath.Clean(destDir)+string(os.PathSeparator)) { + t.Fatalf("zip entry %q escapes destination directory", f.Name) + } + if f.FileInfo().IsDir() { + if mkErr := os.MkdirAll(target, 0o750); mkErr != nil { + t.Fatalf("mkdir %s: %v", target, mkErr) + } + continue + } + if mkErr := os.MkdirAll(filepath.Dir(target), 0o750); mkErr != nil { + t.Fatalf("mkdir parent of %s: %v", target, mkErr) + } + rc, openErr := f.Open() + if openErr != nil { + t.Fatalf("open zip entry %s: %v", f.Name, openErr) + } + out, createErr := os.Create(target) //nolint:gosec // target validated against destDir above + if createErr != nil { + _ = rc.Close() + t.Fatalf("create %s: %v", target, createErr) + } + if _, copyErr := io.Copy(out, rc); copyErr != nil { //nolint:gosec // trusted test-produced archive + _ = rc.Close() + _ = out.Close() + t.Fatalf("write %s: %v", target, copyErr) + } + _ = rc.Close() + if closeErr := out.Close(); closeErr != nil { + t.Fatalf("close %s: %v", target, closeErr) + } + } +} + +// readFile reads path or fails the test. +func readFile(t *testing.T, path string) []byte { + t.Helper() + data, err := os.ReadFile(path) //nolint:gosec // test-controlled path under t.TempDir() + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + return data +} diff --git a/pkg/server/signing_test.go b/pkg/server/signing_test.go new file mode 100644 index 000000000..68f2e8dda --- /dev/null +++ b/pkg/server/signing_test.go @@ -0,0 +1,411 @@ +// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "context" + "fmt" + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/NVIDIA/aicr/pkg/bundler/attestation" + "github.com/NVIDIA/aicr/pkg/bundler/verifier" + "github.com/NVIDIA/aicr/pkg/defaults" + aicrerrors "github.com/NVIDIA/aicr/pkg/errors" +) + +// TestLoadBinaryAttestation exercises the startup fail-fast loader with an +// injected verifier seam (the real verify pins NVIDIA-CI identity + the running +// binary's digest, unsatisfiable in a `go test` executable). +func TestLoadBinaryAttestation(t *testing.T) { + t.Run("disabled is a no-op", func(t *testing.T) { + cfg := &signingConfig{enabled: false} + called := false + err := cfg.loadBinaryAttestation(context.Background(), func(context.Context) ([]byte, error) { + called = true + return []byte("unexpected"), nil + }) + if err != nil { + t.Fatalf("err = %v, want nil", err) + } + if called { + t.Error("verifier was called while signing disabled") + } + if cfg.binaryAttestation != nil { + t.Errorf("binaryAttestation = %q, want nil", cfg.binaryAttestation) + } + }) + + t.Run("enabled caches verified bytes", func(t *testing.T) { + cfg := &signingConfig{enabled: true} + fixture := []byte("fixture") + err := cfg.loadBinaryAttestation(context.Background(), func(context.Context) ([]byte, error) { + return fixture, nil + }) + if err != nil { + t.Fatalf("err = %v, want nil", err) + } + if string(cfg.binaryAttestation) != "fixture" { + t.Errorf("binaryAttestation = %q, want %q", cfg.binaryAttestation, fixture) + } + }) + + t.Run("enabled fails fast when verify errors", func(t *testing.T) { + cfg := &signingConfig{enabled: true} + verifyErr := aicrerrors.New(aicrerrors.ErrCodeNotFound, "attestation absent") + err := cfg.loadBinaryAttestation(context.Background(), func(context.Context) ([]byte, error) { + return nil, verifyErr + }) + if err == nil { + t.Fatal("err = nil, want fail-fast error") + } + if cfg.binaryAttestation != nil { + t.Errorf("binaryAttestation = %q, want nil on failure", cfg.binaryAttestation) + } + }) +} + +// TestResolveBinaryAttestationPath verifies the override/discovery branch: +// AICR_BINARY_ATTESTATION_FILE wins when set, otherwise the path falls back to +// the conventional file next to the running executable via FindBinaryAttestation. +func TestResolveBinaryAttestationPath(t *testing.T) { + // Build a temp binPath with a sibling attestation so the fallback discovery + // (suffix "-attestation.sigstore.json" next to the binary) succeeds. + dir := t.TempDir() + binPath := dir + "/aicrd" + sibling := binPath + attestation.AttestationFileSuffix + if err := os.WriteFile(binPath, []byte("binary"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(sibling, []byte("{}"), 0o600); err != nil { + t.Fatal(err) + } + + t.Run("both env unset falls back to FindBinaryAttestation", func(t *testing.T) { + t.Setenv(defaults.EnvBinaryAttestationFile, "") + t.Setenv(defaults.EnvKoDataPath, "") + want, err := attestation.FindBinaryAttestation(binPath) + if err != nil { + t.Fatalf("FindBinaryAttestation setup = %v, want nil", err) + } + got, err := resolveBinaryAttestationPath(binPath) + if err != nil { + t.Fatalf("err = %v, want nil", err) + } + if got != want { + t.Errorf("path = %q, want %q (sibling of binary)", got, want) + } + }) + + t.Run("explicit file env overrides regardless of binPath", func(t *testing.T) { + t.Setenv(defaults.EnvKoDataPath, "") + t.Setenv(defaults.EnvBinaryAttestationFile, "/custom/att.json") + got, err := resolveBinaryAttestationPath(binPath) + if err != nil { + t.Fatalf("err = %v, want nil", err) + } + if got != "/custom/att.json" { + t.Errorf("path = %q, want /custom/att.json", got) + } + }) + + t.Run("explicit file env wins over KO_DATA_PATH", func(t *testing.T) { + t.Setenv(defaults.EnvKoDataPath, "/ko/data") + t.Setenv(defaults.EnvBinaryAttestationFile, "/custom/att.json") + got, err := resolveBinaryAttestationPath(binPath) + if err != nil { + t.Fatalf("err = %v, want nil", err) + } + if got != "/custom/att.json" { + t.Errorf("path = %q, want /custom/att.json (explicit override wins)", got) + } + }) + + t.Run("KO_DATA_PATH resolves per-architecture attestation", func(t *testing.T) { + t.Setenv(defaults.EnvBinaryAttestationFile, "") + t.Setenv(defaults.EnvKoDataPath, "/ko/data") + name := fmt.Sprintf(defaults.BinaryAttestationKoDataNameFormat, runtime.GOARCH) + want := filepath.Join("/ko/data", name) + got, err := resolveBinaryAttestationPath(binPath) + if err != nil { + t.Fatalf("err = %v, want nil", err) + } + if got != want { + t.Errorf("path = %q, want %q", got, want) + } + }) +} + +// TestResolveBinaryAttestationIdentityPattern verifies the +// AICR_BINARY_ATTESTATION_IDENTITY_REGEXP override: unset falls back to the +// release-workflow default; a value still pinned to the NVIDIA repo is accepted; +// a value pointing elsewhere is rejected (fails startup fast). +func TestResolveBinaryAttestationIdentityPattern(t *testing.T) { + tests := []struct { + name string + env string // "" means unset (uses default) + setEnv bool + want string + wantErr bool + }{ + { + name: "unset returns release-workflow default", + setEnv: false, + want: verifier.TrustedRepositoryPattern, + }, + { + name: "valid override pinned to NVIDIA repo is returned", + setEnv: true, + env: `^https://github\.com/NVIDIA/aicr/\.github/workflows/server-kms-e2e\.yaml@.*`, + want: `^https://github\.com/NVIDIA/aicr/\.github/workflows/server-kms-e2e\.yaml@.*`, + }, + { + name: "override not pinned to NVIDIA repo is rejected", + setEnv: true, + env: `^https://github\.com/evil/repo/.*`, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.setEnv { + t.Setenv(defaults.EnvBinaryAttestationIdentityRegexp, tt.env) + } else { + t.Setenv(defaults.EnvBinaryAttestationIdentityRegexp, "") + } + got, err := resolveBinaryAttestationIdentityPattern() + if (err != nil) != tt.wantErr { + t.Fatalf("err = %v, wantErr %v", err, tt.wantErr) + } + if tt.wantErr { + return + } + if got != tt.want { + t.Errorf("pattern = %q, want %q", got, tt.want) + } + }) + } +} + +func TestParseSigningConfig(t *testing.T) { + tests := []struct { + name string + env map[string]string + wantEnabled bool + wantKeyless bool + wantErr bool + }{ + {"none configured", nil, false, false, false}, + {"valid KMS", map[string]string{defaults.EnvSigningKey: "awskms:///alias/aicr"}, true, false, false}, + {"malformed KMS scheme", map[string]string{defaults.EnvSigningKey: "vault:/x"}, false, false, true}, + {"valid keyless (file)", map[string]string{ + defaults.EnvFulcioURL: "https://fulcio.internal", + defaults.EnvIdentityTokenFile: "/var/run/sigstore/token", + }, true, true, false}, + {"valid keyless (ambient)", map[string]string{ + defaults.EnvFulcioURL: "https://fulcio.internal", + defaults.EnvGitHubActionsIDTokenRequestURL: "https://token.actions.example/req", + defaults.EnvGitHubActionsIDTokenRequestToken: "ambient-tok", + }, true, true, false}, + {"keyless missing token source", map[string]string{ + defaults.EnvFulcioURL: "https://fulcio.internal", + }, false, false, true}, + {"both modes ambiguous", map[string]string{ + defaults.EnvSigningKey: "awskms:///alias/aicr", + defaults.EnvFulcioURL: "https://fulcio.internal", + }, false, false, true}, + {"KMS with invalid tlog upload", map[string]string{ + defaults.EnvSigningKey: "awskms:///alias/aicr", + defaults.EnvTLogUpload: "garbage", + }, false, false, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Ensure GHA ambient env doesn't leak into keyless cases. + t.Setenv(defaults.EnvGitHubActionsIDTokenRequestURL, "") + t.Setenv(defaults.EnvGitHubActionsIDTokenRequestToken, "") + for k, v := range tt.env { + t.Setenv(k, v) + } + cfg, err := parseSigningConfig() + if (err != nil) != tt.wantErr { + t.Fatalf("err = %v, wantErr %v", err, tt.wantErr) + } + if err != nil { + return + } + if cfg.enabled != tt.wantEnabled { + t.Errorf("enabled = %v, want %v", cfg.enabled, tt.wantEnabled) + } + if cfg.keyless != tt.wantKeyless { + t.Errorf("keyless = %v, want %v", cfg.keyless, tt.wantKeyless) + } + }) + } +} + +func TestSigningConfigResolveOptions_KeylessReadsTokenFresh(t *testing.T) { + dir := t.TempDir() + tokenPath := dir + "/token" + if err := os.WriteFile(tokenPath, []byte("tok-v1"), 0o600); err != nil { + t.Fatal(err) + } + t.Setenv(defaults.EnvGitHubActionsIDTokenRequestURL, "") + t.Setenv(defaults.EnvGitHubActionsIDTokenRequestToken, "") + t.Setenv(defaults.EnvFulcioURL, "https://fulcio.internal") + t.Setenv(defaults.EnvIdentityTokenFile, tokenPath) + cfg, err := parseSigningConfig() + if err != nil { + t.Fatal(err) + } + o1, err := cfg.resolveOptions() + if err != nil || o1.IdentityToken != "tok-v1" { + t.Fatalf("o1 token = %q err %v", o1.IdentityToken, err) + } + if err := os.WriteFile(tokenPath, []byte("tok-v2"), 0o600); err != nil { + t.Fatal(err) + } + o2, _ := cfg.resolveOptions() + if o2.IdentityToken != "tok-v2" { + t.Errorf("token not re-read: got %q, want tok-v2", o2.IdentityToken) + } + if !o2.Attest { + t.Error("Attest should be true") + } +} + +func TestSigningConfigResolveOptions_KMS(t *testing.T) { + t.Setenv(defaults.EnvSigningKey, "gcpkms://projects/p/locations/l/keyRings/r/cryptoKeys/k") + cfg, err := parseSigningConfig() + if err != nil { + t.Fatal(err) + } + o, err := cfg.resolveOptions() + if err != nil { + t.Fatal(err) + } + if !o.Attest || o.SigningKey != "gcpkms://projects/p/locations/l/keyRings/r/cryptoKeys/k" { + t.Errorf("resolve options = %+v", o) + } +} + +// TestSigningConfigResolveOptions_KeylessAmbient verifies that a keyless config +// backed by the GitHub Actions ambient OIDC endpoint (no token file) surfaces +// the ambient URL/token and leaves IdentityToken empty. +func TestSigningConfigResolveOptions_KeylessAmbient(t *testing.T) { + t.Setenv(defaults.EnvGitHubActionsIDTokenRequestURL, "https://token.actions.example/req") + t.Setenv(defaults.EnvGitHubActionsIDTokenRequestToken, "ambient-tok") + t.Setenv(defaults.EnvFulcioURL, "https://fulcio.internal") + cfg, err := parseSigningConfig() + if err != nil { + t.Fatal(err) + } + if !cfg.enabled || !cfg.keyless { + t.Fatalf("enabled=%v keyless=%v, want both true", cfg.enabled, cfg.keyless) + } + o, err := cfg.resolveOptions() + if err != nil { + t.Fatal(err) + } + if o.AmbientURL != "https://token.actions.example/req" || o.AmbientToken != "ambient-tok" { + t.Errorf("ambient = %q/%q, want request URL/token", o.AmbientURL, o.AmbientToken) + } + if o.IdentityToken != "" { + t.Errorf("IdentityToken = %q, want empty (ambient source)", o.IdentityToken) + } + if o.FulcioURL != "https://fulcio.internal" || !o.Attest { + t.Errorf("resolve options = %+v", o) + } +} + +// TestSigningConfigResolveOptions_TLogUpload verifies the AICR_TLOG_UPLOAD +// toggle maps to DisableTLogUpload for KMS (Mode A) signing. +func TestSigningConfigResolveOptions_TLogUpload(t *testing.T) { + tests := []struct { + name string + tlog string // "" means unset + wantDisable bool + }{ + {"default unset uploads", "", false}, + {"explicit false disables", "false", true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Setenv(defaults.EnvGitHubActionsIDTokenRequestURL, "") + t.Setenv(defaults.EnvGitHubActionsIDTokenRequestToken, "") + t.Setenv(defaults.EnvSigningKey, "awskms:///alias/aicr") + if tt.tlog != "" { + t.Setenv(defaults.EnvTLogUpload, tt.tlog) + } + cfg, err := parseSigningConfig() + if err != nil { + t.Fatal(err) + } + o, err := cfg.resolveOptions() + if err != nil { + t.Fatal(err) + } + if o.DisableTLogUpload != tt.wantDisable { + t.Errorf("DisableTLogUpload = %v, want %v", o.DisableTLogUpload, tt.wantDisable) + } + }) + } +} + +// TestParseSigningConfig_InvalidTLogUpload asserts a non-boolean +// AICR_TLOG_UPLOAD fails startup fast rather than silently defaulting. +func TestParseSigningConfig_InvalidTLogUpload(t *testing.T) { + t.Setenv(defaults.EnvGitHubActionsIDTokenRequestURL, "") + t.Setenv(defaults.EnvGitHubActionsIDTokenRequestToken, "") + t.Setenv(defaults.EnvSigningKey, "awskms:///alias/aicr") + t.Setenv(defaults.EnvTLogUpload, "garbage") + if _, err := parseSigningConfig(); err == nil { + t.Fatal("expected error for non-boolean AICR_TLOG_UPLOAD, got nil") + } +} + +// TestParseSigningConfig_InvalidTLogUploadIgnoredWhenUnconfigured asserts a +// malformed AICR_TLOG_UPLOAD does NOT fail startup when no signing mode is +// configured (the toggle is KMS-only and has no effect otherwise). +func TestParseSigningConfig_InvalidTLogUploadIgnoredWhenUnconfigured(t *testing.T) { + t.Setenv(defaults.EnvGitHubActionsIDTokenRequestURL, "") + t.Setenv(defaults.EnvGitHubActionsIDTokenRequestToken, "") + t.Setenv(defaults.EnvTLogUpload, "garbage") + cfg, err := parseSigningConfig() + if err != nil { + t.Fatalf("unconfigured server must ignore AICR_TLOG_UPLOAD, got err %v", err) + } + if cfg.enabled { + t.Error("expected signing disabled when no mode configured") + } +} + +// TestSigningConfigResolveOptions_MissingTokenFile verifies resolveOptions +// surfaces the read error when the keyless identity-token file is absent. +func TestSigningConfigResolveOptions_MissingTokenFile(t *testing.T) { + t.Setenv(defaults.EnvGitHubActionsIDTokenRequestURL, "") + t.Setenv(defaults.EnvGitHubActionsIDTokenRequestToken, "") + t.Setenv(defaults.EnvFulcioURL, "https://fulcio.internal") + t.Setenv(defaults.EnvIdentityTokenFile, t.TempDir()+"/does-not-exist") + cfg, err := parseSigningConfig() + if err != nil { + t.Fatal(err) + } + if _, err := cfg.resolveOptions(); err == nil { + t.Fatal("expected error for missing identity token file, got nil") + } +} diff --git a/tests/chainsaw/signing/server-bundle-attestation-vault/README.md b/tests/chainsaw/signing/server-bundle-attestation-vault/README.md new file mode 100644 index 000000000..6d67a837b --- /dev/null +++ b/tests/chainsaw/signing/server-bundle-attestation-vault/README.md @@ -0,0 +1,170 @@ +# Server (aicrd) Vault (OpenBAO) KMS bundle-attestation E2E + +End-to-end (E2E) test suite for **server-side** bundle signing over HTTP: the +aicrd `POST /v1/bundle?attest=true` surface (#1150) driven against +[OpenBAO](https://openbao.org) with a HashiCorp Vault Transit signing key. It is +the server companion of +[`bundle-attestation-vault`](../bundle-attestation-vault/), which exercises the +same `hashivault://` path through the `aicr` CLI. + +## Why this exists + +Issue [#1150](https://github.com/NVIDIA/aicr/issues/1150) added server-side +attestation to `POST /v1/bundle`: with a signing identity configured entirely +from the server's own environment (a KMS key or a private-Sigstore keyless +identity), a request opts into signing with `?attest=true`, and aicrd signs the +bundle **as itself** — no human at a browser, no request-supplied identity +material. Unit tests +(`pkg/server/bundle_handler_test.go`, `pkg/server/signing_test.go`) cover the +mode selection, the `attest=true` wiring, and the "not configured" rejection, +but they inject a fake attester and never sign or verify through a real Vault +server over HTTP. This suite closes that gap: it starts the real aicrd server +with `AICR_SIGNING_KEY=hashivault://aicr` and drives the +sign (`POST /v1/bundle?attest=true`) → verify (`aicr verify --key`) round-trip +against a live (dev-mode) Transit engine. + +## Structure: `run.sh` only (no `chainsaw-test.yaml`) + +Unlike the CLI suite (which uses chainsaw to invoke `aicr bundle`/`aicr verify` +as discrete steps), this suite is a single `run.sh`. The subject under test is a +**long-running background process**, and its lifecycle — start aicrd, poll +`/health`, fail fast (dumping logs) if it dies at startup, `curl` the endpoint, +`aicr verify` the result, then kill the server — does not map cleanly onto +chainsaw's per-step script model, which has no notion of a process that must +outlive one step and be torn down after another. Keeping process ownership, the +startup fail-fast log dump, and the cleanup trap (which kills the server **and** +removes the OpenBAO container) in one script is clearer and is the single +artifact both local runs and CI invoke. The design note in the task brief +explicitly allows this: "a well-structured `run.sh` that does +start-server/curl/verify is acceptable and likely cleaner." + +## What is OpenBAO + +[OpenBAO](https://openbao.org) is the Linux Foundation Apache-2.0 fork of +HashiCorp Vault, shipped as a single Docker image (`openbao/openbao`). Its +Transit secrets engine is API-identical to Vault's, so the sigstore +`hashivault://` signer/verifier drives it unchanged — the same code path serves +a production HashiCorp Vault. We use OpenBAO for the test container because its +Apache-2.0 license is consistent with this project's dependency policy. + +Dev mode (`server -dev`) starts a single in-memory node that auto-initializes +and unseals, serves plain **HTTP** on port `8200`, and sets a known root token. +No TLS setup is needed. The image is pinned in `.settings.yaml` +(`testing_tools.openbao_image`) and resolved via the `load-versions` action, +never floated to `:latest`. + +## What is tested + +### Full mode (attested aicrd binary present) + +| Check | What it verifies | +|-------|-----------------| +| server startup | aicrd starts with `AICR_SIGNING_KEY=hashivault://aicr`, verifying its own embedded binary attestation, and reports `/health` ready | +| `POST /v1/bundle?attest=true` | Server signs the bundle as itself and returns a zip (HTTP 200) | +| bundle contents | `attestation/bundle-attestation.sigstore.json` **and** the embedded `attestation/aicr-attestation.sigstore.json` (tool provenance) are present | +| `aicr verify --key hashivault://aicr --insecure-ignore-tlog` | `checksumsPassed: true` **and** `bundleAttested: true` | + +`--insecure-ignore-tlog` is used because the server signs with +`AICR_TLOG_UPLOAD=false` (KMS / Mode A, no Rekor upload), so verification is the +offline/air-gapped key-based path. + +### Smoke mode (no attested aicrd binary — e.g. a local snapshot) + +The server-side `--attest` path requires the server to verify its **own** binary +attestation at startup, which a plain `goreleaser --snapshot` build does not +carry. So without an attested binary the suite validates the plumbing and exits +`0`: + +| Check | What it verifies | +|-------|-----------------| +| OpenBAO + Transit | container up, ECDSA P-256 key `aicr` provisioned, PEM exported | +| server startup | aicrd starts with signing **disabled** and reports `/health` ready | +| `POST /v1/bundle?attest=true` | rejected with **HTTP 400** "not configured for attestation" | +| `POST /v1/bundle` (unsigned) | returns a valid zip (HTTP 200) carrying `checksums.txt` | + +## KMS URI format + +```text +hashivault:// +``` + +For this suite: `hashivault://aicr`. The sigstore hashivault provider (inside +aicrd) reads the server address and token from the `VAULT_ADDR` / `VAULT_TOKEN` +environment variables (`BAO_ADDR` / `BAO_TOKEN` are honored too); the URI carries +only the Transit key name. The signing key is `ecdsa-p256`, which signs over the +SHA-256 digest the bundler uses. + +> **`VAULT_NAMESPACE` gotcha.** A stray `VAULT_NAMESPACE` (inherited from a shell +> or CI secret) makes the hashivault client target an Enterprise namespace path +> that dev-mode OpenBAO does not serve, breaking every Transit call. `run.sh` +> `unset`s it unconditionally, and the CI workflow does not set it. + +## Prerequisites + +| Tool | Install | When | +|------|---------|------| +| Docker | https://docs.docker.com/get-docker/ | always (runs OpenBAO) | +| `curl` | System package (usually pre-installed) | always (Vault HTTP API + aicrd) | +| `python3` | System package | always (Transit PEM, verify-JSON parse, unzip, free-port pick) | +| `yq` | `make tools-setup` | always (reads the pinned OpenBAO image from `.settings.yaml`) | +| `goreleaser` | `make tools-setup` | only when building binaries (skipped if `AICRD_BIN` + `AICR_BIN` are provided) | + +## Running locally + +```bash +./tests/chainsaw/signing/server-bundle-attestation-vault/run.sh +``` + +The script starts OpenBAO in dev mode, enables Transit, provisions an +`ecdsa-p256` key, builds (or locates) the `aicrd` server and `aicr` CLI binaries, +starts aicrd in the background, runs the full sign/verify round-trip (or the +smoke checks; see below), and on exit kills the server and removes the container. +Override the image, port, token, or key with `OPENBAO_IMAGE` / `OPENBAO_PORT` / +`VAULT_TOKEN` / `VAULT_KMS_KEY`, and the server port with `PORT`; pass +`DEBUG=true` for verbose server logs. + +`aicrd` is a **linux-only** goreleaser target, so a local `goreleaser --snapshot` +build produces it only on a Linux host. On macOS, build the two binaries with +`go build` and point the runner at them: + +```bash +go build -o /tmp/aicrd ./cmd/aicrd +go build -o /tmp/aicr ./cmd/aicr +AICRD_BIN=/tmp/aicrd AICR_BIN=/tmp/aicr \ + ./tests/chainsaw/signing/server-bundle-attestation-vault/run.sh # smoke mode +``` + +> **The `--attest` path needs an attested aicrd binary.** The server verifies its +> own binary attestation at startup (embedded into every attested bundle as tool +> provenance), and a plain snapshot build carries none. When no attested binary +> is available, `run.sh` enters **smoke mode** and exits `0` after validating the +> plumbing. For a full run, run the `server-kms-e2e.yaml` workflow (which attests +> the aicrd binary with its own workflow identity), or point `AICRD_BIN` at a +> CI-attested binary with its sibling `aicrd-attestation.sigstore.json`. + +## CI + +Workflow: `.github/workflows/server-kms-e2e.yaml` + +Triggers: push to `main`, `pull_request` against `main` (same-repo only; fork PRs +are skipped because they lack the OIDC needed to attest the binary), and +`workflow_dispatch`. The workflow builds the **attested** `aicr` + `aicrd` +binaries (goreleaser + the cosign `attest-blob` hook, fed by +`generate-slsa-predicate` which sets both `SLSA_PREDICATE` and +`AICR_SIGNING_CONFIG`), exports `AICRD_BIN`/`AICR_BIN`, then runs the suite's +`run.sh`. `run.sh` owns the OpenBAO container and the aicrd server lifecycle, so +the same script runs locally and in CI with no inlined copy to drift. + +## Environment variables + +| Variable | Set by | Purpose | +|----------|--------|---------| +| `VAULT_ADDR` | CI / `run.sh` | Vault/OpenBAO base URL (`http://127.0.0.1:8200`) | +| `VAULT_TOKEN` | CI / `run.sh` | Auth token (dev-mode root token) | +| `VAULT_KMS_KEY` | CI / `run.sh` | Transit key name (`aicr`); the URI is `hashivault://` | +| `AICRD_BIN` | CI / `run.sh` | Path to the built `aicrd` server binary | +| `AICR_BIN` | CI / `run.sh` | Path to the built `aicr` CLI binary (recipe generation + verify) | +| `PORT` | `run.sh` | aicrd listen port (a free ephemeral port when unset) | +| `AICR_SIGNING_KEY` | `run.sh` (full) | `hashivault://aicr`; selects server KMS signing | +| `AICR_TLOG_UPLOAD` | `run.sh` (full) | `false`; KMS signing with no Rekor upload (verify uses `--insecure-ignore-tlog`) | +| `AICR_BINARY_ATTESTATION_FILE` | `run.sh` (full) | Path to the sibling `aicrd-attestation.sigstore.json` the server embeds | diff --git a/tests/chainsaw/signing/server-bundle-attestation-vault/run.sh b/tests/chainsaw/signing/server-bundle-attestation-vault/run.sh new file mode 100755 index 000000000..2501c6226 --- /dev/null +++ b/tests/chainsaw/signing/server-bundle-attestation-vault/run.sh @@ -0,0 +1,576 @@ +#!/bin/bash +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ============================================================================= +# Server (aicrd) Vault/OpenBAO KMS bundle-attestation E2E local runner +# ============================================================================= +# +# PURPOSE: +# Drives the aicrd HTTP SERVER (not the aicr CLI) end to end against a +# HashiCorp Vault-compatible Transit engine (OpenBAO, https://openbao.org). +# It starts OpenBAO in dev mode, enables the Transit secrets engine, provisions +# an ECDSA P-256 signing key `aicr`, starts aicrd in the background with the +# operator-configured KMS signing identity, and exercises the server-side +# bundle-attestation surface added for #1150: +# +# POST /v1/bundle?attest=true -> server signs the bundle as itself +# aicr verify --key hashivault://aicr --insecure-ignore-tlog +# +# The companion of tests/chainsaw/signing/bundle-attestation-vault/, which +# exercises the same hashivault:// path through the aicr CLI. This suite proves +# the server reproduces that behavior over HTTP with a non-interactive, +# operator-supplied signing identity (no human at a browser, no request-supplied +# identity material). +# +# Because the server is a long-running process, lifecycle management is the key +# difference from the CLI suite: the runner starts aicrd in the background, waits +# for /health, fails fast (dumping logs) if the process dies at startup, and the +# cleanup trap kills the server in addition to removing the OpenBAO container. +# +# STRUCTURE (why run.sh, not chainsaw-test.yaml): the server is a background +# process whose lifecycle (start -> poll /health -> curl -> verify -> kill) does +# not map cleanly onto chainsaw's per-step script model. A single well-structured +# run.sh keeps the process ownership, the fail-fast log dump, and the cleanup +# trap in one place, and is the primary artifact the CI workflow invokes. See the +# README for the rationale. +# +# MODES: +# SMOKE MODE (no aicrd binary attestation present, e.g. a local snapshot): +# The server-side --attest path requires the server to verify its OWN binary +# attestation at startup, which a plain `goreleaser --snapshot` build does not +# carry. So without an attested binary this runner validates the plumbing and +# exits 0: +# - OpenBAO up, Transit ECDSA P-256 key provisioned, PEM exported +# - aicrd starts WITHOUT signing configured +# - POST /v1/bundle?attest=true -> 400 "not configured for attestation" +# - POST /v1/bundle (unsigned) -> 200, a zip archive +# +# FULL MODE (attested aicrd binary present, e.g. built by server-kms-e2e.yaml): +# Starts aicrd with AICR_SIGNING_KEY=hashivault://aicr, AICR_TLOG_UPLOAD=false, +# and AICR_BINARY_ATTESTATION_FILE pointing at the sibling +# aicrd-attestation.sigstore.json, then: +# - POST /v1/bundle?attest=true -> signed bundle zip +# - aicr verify --key hashivault://aicr --insecure-ignore-tlog --format json +# asserts bundleAttested:true and checksumsPassed:true +# - asserts attestation/aicr-attestation.sigstore.json is in the bundle +# For a full run, run the server-kms-e2e.yaml workflow (which attests the +# aicrd binary with its own workflow identity), or point AICRD_BIN/AICR_BIN at +# CI-attested binaries. +# +# PREREQUISITES: +# Always required (checked upfront): +# - docker (runs the OpenBAO container) +# - curl (talks to the Vault HTTP API and the aicrd server) +# - python3 (extracts the Transit PEM, parses verify JSON, unzips the bundle, +# picks a free port) +# - yq (reads the pinned OpenBAO image from .settings.yaml before startup) +# Checked lazily, only on the path that uses them: +# - goreleaser (builds unattested snapshot binaries when AICRD_BIN/AICR_BIN unset) +# +# USAGE: +# ./tests/chainsaw/signing/server-bundle-attestation-vault/run.sh +# +# The script kills the aicrd server and removes the OpenBAO container on exit +# (success or failure). Pass DEBUG=true for verbose server logs. Override the +# image, port, token, or key with OPENBAO_IMAGE / OPENBAO_PORT / VAULT_TOKEN / +# VAULT_KMS_KEY; override the server port with PORT. +# +# ============================================================================= + +set -euo pipefail + +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd "${DIR}/../../../.." && pwd)" +# shellcheck source=tools/common +. "${ROOT}/tools/common" + +# ============================================================================= +# Prerequisites +# ============================================================================= + +# Tools every path needs. goreleaser (build) is checked lazily in +# build_or_locate_binaries so the pre-built-binary and smoke paths do not +# require it. +has_tools docker curl python3 yq + +# ============================================================================= +# Configuration +# ============================================================================= + +OPENBAO_CONTAINER_NAME="aicr-server-kms-e2e-openbao" +OPENBAO_PORT="${OPENBAO_PORT:-8200}" +# Image from .settings.yaml (testing_tools.openbao_image) — the single source of +# truth, shared with CI via the load-versions action. Read it rather than +# duplicating a literal tag here (which could silently drift from the pin). +# Never :latest. +OPENBAO_IMAGE="${OPENBAO_IMAGE:-$(yq -r '.testing_tools.openbao_image' "${ROOT}/.settings.yaml" 2>/dev/null)}" +if [ -z "${OPENBAO_IMAGE}" ] || [ "${OPENBAO_IMAGE}" = "null" ]; then + err "Could not read testing_tools.openbao_image from ${ROOT}/.settings.yaml" +fi + +# Dev-mode root token. Dev mode auto-initializes and unseals, mounts Transit-able +# storage in memory, and serves plain HTTP — nothing here is a real secret. +VAULT_TOKEN="${VAULT_TOKEN:-root}" +VAULT_ADDR="http://127.0.0.1:${OPENBAO_PORT}" +# The sigstore hashivault provider (inside aicrd) reads VAULT_ADDR / VAULT_TOKEN +# (BAO_* also work). Exported so the provider in the server process reaches the +# same OpenBAO instance the runner provisioned. +export VAULT_ADDR VAULT_TOKEN +# Transit key name; the KMS URI is hashivault://. +VAULT_KMS_KEY="${VAULT_KMS_KEY:-aicr}" +export VAULT_KMS_KEY + +# A stray VAULT_NAMESPACE (e.g. inherited from an operator's shell or CI secret) +# makes the sigstore hashivault client target an Enterprise namespace path that +# dev-mode OpenBAO does not serve, breaking every Transit call with a confusing +# 404. Dev-mode OpenBAO has no namespaces, so clear it unconditionally. +unset VAULT_NAMESPACE || true + +WORK_DIR="${TMPDIR:-/tmp}/server-kms-e2e-$$" +SERVER_LOG="${WORK_DIR}/aicrd.log" + +# Optional pre-built binaries. Provide CI-attested binaries (e.g. from the +# server-kms-e2e.yaml workflow) to run the full sign/verify chain locally; +# otherwise the runner builds unattested snapshots and enters smoke mode. +# aicrd is the server under test; aicr is used for recipe generation and verify. +AICRD_BIN="${AICRD_BIN:-}" +AICR_BIN="${AICR_BIN:-}" + +# Server listen port. Explicit PORT wins; otherwise pick a free ephemeral port so +# concurrent runs (and a stray local aicrd on 8080) do not collide. +SERVER_PORT="${PORT:-}" + +# Populated by later stages. +AICR_ATTESTED="false" +BINARY_ATTESTATION_FILE="" +SERVER_PID="" + +# ============================================================================= +# Cleanup +# ============================================================================= + +stop_server() { + if [ -n "${SERVER_PID}" ] && kill -0 "${SERVER_PID}" 2>/dev/null; then + kill "${SERVER_PID}" 2>/dev/null || true + wait "${SERVER_PID}" 2>/dev/null || true + fi + SERVER_PID="" +} + +cleanup() { + local rc=$? + msg "Cleaning up" + stop_server + docker rm -f "${OPENBAO_CONTAINER_NAME}" &>/dev/null || true + # WORK_DIR is an internal, per-PID path under TMPDIR, so a full rm is safe. + rm -rf "${WORK_DIR}" + if [ $rc -eq 0 ]; then + msg "Server KMS E2E: PASSED" + else + log_error "Server KMS E2E: FAILED (exit ${rc})" + fi + exit $rc +} +trap cleanup EXIT + +# ============================================================================= +# OpenBAO +# ============================================================================= + +start_openbao() { + msg "Starting OpenBAO (${OPENBAO_IMAGE})" + + # Remove any leftover container from a prior interrupted run + docker rm -f "${OPENBAO_CONTAINER_NAME}" &>/dev/null || true + + # Dev mode: single in-memory node, auto-unsealed, root token fixed to + # ${VAULT_TOKEN}, HTTP listener on 0.0.0.0:8200. IPC_LOCK lets the server mlock + # memory (Vault/OpenBAO warn without it; harmless to grant). + docker run -d \ + --name "${OPENBAO_CONTAINER_NAME}" \ + --cap-add IPC_LOCK \ + -p "${OPENBAO_PORT}:8200" \ + "${OPENBAO_IMAGE}" \ + server -dev \ + -dev-root-token-id="${VAULT_TOKEN}" \ + -dev-listen-address=0.0.0.0:8200 >/dev/null + + msg "Waiting for OpenBAO to become available" + local retries=45 + until [ $retries -eq 0 ]; do + # Fail fast if the container died (e.g. bad image tag) instead of polling a + # dead endpoint for the full retry budget. + if [ "$(docker inspect -f '{{.State.Running}}' "${OPENBAO_CONTAINER_NAME}" 2>/dev/null)" != "true" ]; then + docker logs "${OPENBAO_CONTAINER_NAME}" 2>&1 | tail -15 + err "OpenBAO container exited unexpectedly" + fi + + # sys/health returns 200 only when initialized, unsealed, and active. + if curl -sf --max-time 3 "${VAULT_ADDR}/v1/sys/health" >/dev/null 2>&1; then + msg "OpenBAO is ready" + return 0 + fi + retries=$((retries - 1)) + sleep 2 + done + err "OpenBAO did not become ready in time (check: docker logs ${OPENBAO_CONTAINER_NAME})" +} + +# ============================================================================= +# Transit provisioning +# ============================================================================= + +provision_transit_key() { + msg "Enabling Transit secrets engine and provisioning ECDSA P-256 key '${VAULT_KMS_KEY}'" + + # Enable transit at the default mount path. Ignore "path is already in use" + # (idempotent across reruns against a persistent server). + curl -sf -H "X-Vault-Token: ${VAULT_TOKEN}" \ + -X POST -d '{"type":"transit"}' \ + "${VAULT_ADDR}/v1/sys/mounts/transit" >/dev/null 2>&1 || true + + # Create the signing key. Transit treats create-of-existing as a no-op. + curl -sf -H "X-Vault-Token: ${VAULT_TOKEN}" \ + -X POST -d '{"type":"ecdsa-p256"}' \ + "${VAULT_ADDR}/v1/transit/keys/${VAULT_KMS_KEY}" >/dev/null + + KMS_URI="hashivault://${VAULT_KMS_KEY}" + msg "KMS URI: ${KMS_URI}" + + # Export the public key (PEM) for the smoke-mode plumbing assertion. Transit + # nests it under data.keys..public_key; pick the latest version. + mkdir -p "${WORK_DIR}" + curl -sf -H "X-Vault-Token: ${VAULT_TOKEN}" \ + "${VAULT_ADDR}/v1/transit/keys/${VAULT_KMS_KEY}" | \ + python3 -c " +import json, sys +keys = json.load(sys.stdin)['data']['keys'] +latest = max(keys, key=lambda k: int(k)) +sys.stdout.write(keys[latest]['public_key']) +" > "${WORK_DIR}/signing-key.pem" + + test -s "${WORK_DIR}/signing-key.pem" + msg "Public key exported to ${WORK_DIR}/signing-key.pem" + + export KMS_URI +} + +# ============================================================================= +# Binary build / location +# ============================================================================= + +# find_dist_binary echoes the first matching goreleaser dist +# path for (aicr | aicrd), covering the amd64 microarch suffixes. +find_dist_binary() { + local name="$1" os="$2" arch="$3" pattern + for pattern in \ + "${ROOT}/dist/${name}_${os}_${arch}/${name}" \ + "${ROOT}/dist/${name}_${os}_${arch}_v1/${name}" \ + "${ROOT}/dist/${name}_${os}_${arch}_v8.0/${name}"; do + if [ -x "${pattern}" ]; then + echo "${pattern}" + return 0 + fi + done + return 1 +} + +build_or_locate_binaries() { + local os_name arch_name + os_name=$(uname -s | tr '[:upper:]' '[:lower:]') + arch_name=$(uname -m) + case "${arch_name}" in + x86_64) arch_name="amd64" ;; + aarch64 | arm64) arch_name="arm64" ;; + esac + + # Validate explicitly-provided overrides up front (fail fast on a typo'd path + # rather than silently falling through to a snapshot build). + if [ -n "${AICRD_BIN}" ] && [ ! -x "${AICRD_BIN}" ]; then + err "AICRD_BIN is set to '${AICRD_BIN}' but is not an executable file" + fi + if [ -n "${AICR_BIN}" ] && [ ! -x "${AICR_BIN}" ]; then + err "AICR_BIN is set to '${AICR_BIN}' but is not an executable file" + fi + + # Build unattested snapshots only when a binary is missing. aicrd is a + # linux-only build target, so `--single-target` produces it only on a linux + # host; elsewhere provide AICRD_BIN (e.g. `go build -o /tmp/aicrd ./cmd/aicrd`). + if [ -z "${AICRD_BIN}" ] || [ -z "${AICR_BIN}" ]; then + has_tools goreleaser + msg "Building aicr + aicrd (unattested snapshot; smoke mode)" + cd "${ROOT}" + if ! GOFLAGS=-mod=vendor goreleaser build --clean --single-target --snapshot --timeout 10m 2>&1; then + err "Failed to build binaries with goreleaser" + fi + [ -n "${AICRD_BIN}" ] || AICRD_BIN="$(find_dist_binary aicrd "${os_name}" "${arch_name}" || true)" + [ -n "${AICR_BIN}" ] || AICR_BIN="$(find_dist_binary aicr "${os_name}" "${arch_name}" || true)" + fi + + if [ -z "${AICRD_BIN}" ] || [ ! -x "${AICRD_BIN}" ]; then + err "aicrd server binary not found (build it, or set AICRD_BIN; note aicrd is a linux-only goreleaser target)" + fi + if [ -z "${AICR_BIN}" ] || [ ! -x "${AICR_BIN}" ]; then + err "aicr CLI binary not found (needed for recipe generation + verify; build it, or set AICR_BIN)" + fi + + msg "Server binary: ${AICRD_BIN}" + msg "CLI binary: ${AICR_BIN}" + export AICRD_BIN AICR_BIN + + # Detect the aicrd binary attestation (FindBinaryAttestation convention: + # -attestation.sigstore.json) to choose full vs smoke mode. The server + # embeds this as tool provenance and verifies it against its own digest at + # startup, so full mode only runs when the binary actually carries one. + BINARY_ATTESTATION_FILE="$(dirname "${AICRD_BIN}")/aicrd-attestation.sigstore.json" + if [ -f "${BINARY_ATTESTATION_FILE}" ]; then + AICR_ATTESTED="true" + msg "aicrd binary attestation found: ${BINARY_ATTESTATION_FILE}" + else + AICR_ATTESTED="false" + BINARY_ATTESTATION_FILE="" + msg "No aicrd binary attestation next to ${AICRD_BIN}; running smoke mode" + fi +} + +# ============================================================================= +# Recipe body +# ============================================================================= + +# The server's POST /v1/bundle body is a resolved RecipeResult JSON. Generate it +# with the aicr CLI (the same shape /v1/recipe would return), matching the CLI +# suite's criteria (eks / h100 / ubuntu / training). +generate_recipe_body() { + msg "Generating RecipeResult body (eks / h100 / ubuntu / training)" + "${AICR_BIN}" recipe --service eks --accelerator h100 --os ubuntu \ + --intent training --format json -o "${WORK_DIR}/recipe.json" + test -s "${WORK_DIR}/recipe.json" +} + +# ============================================================================= +# Server lifecycle +# ============================================================================= + +pick_server_port() { + if [ -z "${SERVER_PORT}" ]; then + SERVER_PORT="$(python3 -c 'import socket +s = socket.socket() +s.bind(("127.0.0.1", 0)) +print(s.getsockname()[1]) +s.close()')" + fi + msg "Server port: ${SERVER_PORT}" +} + +# start_server starts aicrd in the background with the given extra environment +# (name=value pairs) and waits for /health. Fails fast (dumping logs) if the +# process exits during startup — e.g. a signing server that cannot verify its own +# binary attestation must not be polled for the full retry budget. +start_server() { + local desc="$1" + shift + msg "Starting aicrd server (${desc}) on port ${SERVER_PORT}" + + local log_level="info" + [ "${DEBUG:-false}" = "true" ] && log_level="debug" + + # PORT selects the listen port; AICR_LOG_LEVEL controls verbosity. Any extra + # name=value args carry the signing identity for full mode. + PORT="${SERVER_PORT}" AICR_LOG_LEVEL="${log_level}" \ + env "$@" "${AICRD_BIN}" >"${SERVER_LOG}" 2>&1 & + SERVER_PID=$! + + local retries=30 + until [ ${retries} -eq 0 ]; do + if ! kill -0 "${SERVER_PID}" 2>/dev/null; then + log_error "aicrd exited during startup; server log:" + tail -30 "${SERVER_LOG}" >&2 || true + SERVER_PID="" + err "aicrd server failed to start (${desc})" + fi + if curl -sf --max-time 3 "http://127.0.0.1:${SERVER_PORT}/health" >/dev/null 2>&1; then + msg "aicrd server is ready" + return 0 + fi + retries=$((retries - 1)) + sleep 1 + done + log_error "aicrd did not become ready in time; server log:" + tail -30 "${SERVER_LOG}" >&2 || true + err "aicrd server /health never became ready (${desc})" +} + +# ============================================================================= +# Smoke test +# ============================================================================= + +# smoke_test starts aicrd WITHOUT a signing identity and validates the server +# plumbing: attest=true is rejected (400, not configured) and an unsigned bundle +# is still produced. Used when no attested aicrd binary is available. +smoke_test() { + msg "Smoke mode (no attested aicrd binary): validating server plumbing only" + + # Start without any AICR_SIGNING_* env: the server comes up with signing + # disabled. VAULT_ADDR/VAULT_TOKEN are harmless without AICR_SIGNING_KEY. + start_server "signing disabled" + + local base="http://127.0.0.1:${SERVER_PORT}" + + # 1. attest=true must be rejected 400 "not configured" (signing disabled). + local code + code=$(curl -s -o "${WORK_DIR}/smoke-attest.json" -w '%{http_code}' \ + -X POST "${base}/v1/bundle?attest=true" \ + -H 'Content-Type: application/json' \ + --data-binary @"${WORK_DIR}/recipe.json") + if [ "${code}" != "400" ]; then + log_error "attest=true response body:" + cat "${WORK_DIR}/smoke-attest.json" >&2 || true + err "expected HTTP 400 for attest=true on an unconfigured server, got ${code}" + fi + if ! grep -qi "not configured" "${WORK_DIR}/smoke-attest.json"; then + log_error "attest=true response body:" + cat "${WORK_DIR}/smoke-attest.json" >&2 || true + err "expected 'not configured' message in the attest=true rejection" + fi + msg "attest=true correctly rejected with 400 (server not configured for attestation)" + + # 2. An unsigned bundle POST returns a zip archive. + code=$(curl -s -o "${WORK_DIR}/smoke-bundle.zip" -w '%{http_code}' \ + -X POST "${base}/v1/bundle" \ + -H 'Content-Type: application/json' \ + --data-binary @"${WORK_DIR}/recipe.json") + if [ "${code}" != "200" ]; then + log_error "unsigned bundle response body:" + cat "${WORK_DIR}/smoke-bundle.zip" >&2 || true + err "expected HTTP 200 for an unsigned bundle POST, got ${code}" + fi + # Confirm the response is a real zip carrying the standard bundle manifest. + python3 -c " +import sys, zipfile +z = zipfile.ZipFile('${WORK_DIR}/smoke-bundle.zip') +names = z.namelist() +assert any(n.endswith('checksums.txt') for n in names), 'checksums.txt missing from bundle: %r' % names +print('unsigned bundle contains %d entries (checksums.txt present)' % len(names)) +" + msg "Unsigned bundle POST returned a valid zip" + + stop_server + + msg "Smoke checks passed: OpenBAO up, Transit ECDSA P-256 key provisioned, PEM exported, server up, attest=true rejected, unsigned bundle produced" + msg "To run the full server sign/verify suite, provide an attested aicrd binary (AICRD_BIN + sibling aicrd-attestation.sigstore.json), or run the server-kms-e2e.yaml workflow." +} + +# ============================================================================= +# Full test +# ============================================================================= + +# full_test starts aicrd with the Vault KMS signing identity and drives the full +# server-side attestation + verify round-trip. +full_test() { + msg "Full mode (attested aicrd binary): server sign + verify round-trip" + + # AICR_TLOG_UPLOAD=false: KMS (Mode A) signing with no Rekor upload, so verify + # uses --insecure-ignore-tlog + --key (the offline/air-gapped path). + # AICR_BINARY_ATTESTATION_FILE: the sibling attestation the server embeds as + # tool provenance (and verifies against its own digest at startup). + # AICR_BINARY_ATTESTATION_IDENTITY_REGEXP: the aicrd binary here is attested by + # THIS e2e workflow (server-kms-e2e.yaml via generate-slsa-predicate), not the + # release workflow (on-tag.yaml) the server pins by default, so retarget the + # certificate-identity pattern to the e2e workflow. Still pinned to NVIDIA/aicr + # (enforced by verifier.ValidateIdentityPattern). Overridable so a different + # attesting workflow can supply its own identity. + local identity_regexp + identity_regexp="${AICR_BINARY_ATTESTATION_IDENTITY_REGEXP:-^https://github\.com/NVIDIA/aicr/\.github/workflows/server-kms-e2e\.yaml@.*}" + start_server "hashivault KMS signing" \ + "AICR_SIGNING_KEY=hashivault://${VAULT_KMS_KEY}" \ + "AICR_TLOG_UPLOAD=false" \ + "AICR_BINARY_ATTESTATION_FILE=${BINARY_ATTESTATION_FILE}" \ + "AICR_BINARY_ATTESTATION_IDENTITY_REGEXP=${identity_regexp}" \ + "VAULT_ADDR=${VAULT_ADDR}" \ + "VAULT_TOKEN=${VAULT_TOKEN}" + + local base="http://127.0.0.1:${SERVER_PORT}" + + # 1. Request a signed bundle. + local code + code=$(curl -s -o "${WORK_DIR}/bundle.zip" -w '%{http_code}' \ + -X POST "${base}/v1/bundle?attest=true" \ + -H 'Content-Type: application/json' \ + --data-binary @"${WORK_DIR}/recipe.json") + if [ "${code}" != "200" ]; then + log_error "signed bundle response body:" + cat "${WORK_DIR}/bundle.zip" >&2 || true + err "expected HTTP 200 for POST /v1/bundle?attest=true, got ${code}" + fi + msg "Server returned a signed bundle zip" + + # 2. Unzip it. + local unzipped="${WORK_DIR}/unzipped" + rm -rf "${unzipped}" + python3 -c " +import zipfile +zipfile.ZipFile('${WORK_DIR}/bundle.zip').extractall('${unzipped}') +" + + # 3. The bundle carries both the bundle attestation and the embedded binary + # (tool provenance) attestation the server injected. + test -f "${unzipped}/attestation/bundle-attestation.sigstore.json" + test -f "${unzipped}/attestation/aicr-attestation.sigstore.json" + msg "Bundle contains attestation/{bundle,aicr}-attestation.sigstore.json" + + # 4. Verify against the Transit key. --insecure-ignore-tlog because the server + # signed with AICR_TLOG_UPLOAD=false (no Rekor entry to prove trusted time). + # --certificate-identity-regexp pins the *binary* attestation identity: the + # aicrd binary here was attested by this e2e workflow, not on-tag.yaml, so + # verify must accept that identity (same override the server used to embed it) + # to reach the "verified" trust level instead of "unknown". + local out + out=$("${AICR_BIN}" verify "${unzipped}" \ + --key "hashivault://${VAULT_KMS_KEY}" \ + --certificate-identity-regexp "${identity_regexp}" \ + --insecure-ignore-tlog \ + --format json) + echo "${out}" + echo "${out}" | python3 -c " +import json, sys +d = json.load(sys.stdin) +assert d.get('checksumsPassed') is True, 'checksumsPassed != true: %r' % d.get('checksumsPassed') +assert d.get('bundleAttested') is True, 'bundleAttested != true: %r' % d.get('bundleAttested') +print('verify: checksumsPassed=true, bundleAttested=true') +" + + stop_server + + msg "Full checks passed: server signed the bundle with hashivault://${VAULT_KMS_KEY} and it verified (checksumsPassed + bundleAttested)" +} + +# ============================================================================= +# Main +# ============================================================================= + +msg "Starting Server (aicrd) Vault (OpenBAO) KMS bundle-attestation E2E" + +mkdir -p "${WORK_DIR}" +start_openbao +provision_transit_key +build_or_locate_binaries +generate_recipe_body +pick_server_port + +if [ "${AICR_ATTESTED}" = "true" ]; then + full_test +else + smoke_test +fi