From 435b33cb93b65ec25ddfa0cd9a730b367475a310 Mon Sep 17 00:00:00 2001 From: circle-github-action-bot Date: Sat, 5 Sep 2026 01:57:52 -0700 Subject: [PATCH] chore: sync v1.7.8 to release/public (#432) Included changes: - fix: backport audit remediations to v1.7 - fix: align GenerateKey validation - fix: preserve expired-credential errors acro... GitOrigin-RevId: 5f90a044f57f9e7f36d874c552c43a77713fcb8c --- .dockerignore | 1 + .github/actions/cloudsmith-login/action.yml | 66 + .github/workflows/build-docker.yaml | 34 + .github/workflows/build-enclave-image.yaml | 114 ++ .github/workflows/nitro-enclave-build.yaml | 130 ++ .github/workflows/pipeline-common-ci.yaml | 6 +- Makefile | 8 +- README.md | 151 ++- cmd/cmd.go | 10 +- cmd/cmd_test.go | 32 + cmd/enclave/main.go | 54 + cmd/enclave/main_test.go | 110 ++ cmd/enclave/run_enclave_dev.go | 22 + cmd/enclave/run_enclave_prod.go | 22 + .../run_enclave_prod_stub.go} | 25 +- cmd/enclave/run_enclave_prod_stub_test.go | 29 + cmd/run_vsockproxy.go | 39 + configs/.env.dev | 17 +- configs/app.production.yaml | 52 + configs/app.yaml | 32 +- configs/enclave.yaml | 5 + deployments/docker-compose.yaml | 44 +- docker-bake.hcl | 18 + docker/Dockerfile | 79 +- docker/Dockerfile.enclave | 32 +- docker/run.sh | 152 ++- docker/run_enclave.dev.sh | 10 +- docker/run_enclave.sh | 62 +- docker/run_proxy.production.sh | 7 + docs/architecture.md | 1067 +++++++++++++---- docs/development.md | 44 +- docs/testing.md | 42 +- go.mod | 35 +- go.sum | 66 +- internal/app/app.go | 71 +- internal/app/app_test.go | 52 + internal/app/config.go | 74 +- internal/app/config_test.go | 33 +- internal/app/enclaveinit.go | 137 +++ internal/app/enclaveinit_test.go | 434 +++++++ internal/app/provider/awskms/awskms_test.go | 160 --- internal/app/provider/awskms/config.go | 41 - internal/app/provider/enclave/config.go | 51 +- internal/app/provider/enclave/config_test.go | 23 +- internal/app/provider/enclave/enclave.go | 58 +- internal/app/provider/enclave/enclave_mock.go | 28 +- internal/app/provider/enclave/enclave_test.go | 136 ++- .../app/provider/enclave/integration_test.go | 101 +- internal/app/public/public.go | 4 +- internal/app/public/public_test.go | 88 ++ internal/app/runtime_dev.go | 97 ++ internal/app/runtime_dev_test.go | 40 + internal/app/runtime_prod.go | 83 ++ internal/app/runtime_prod_test.go | 178 +++ internal/app/service/signer/cache_test.go | 164 --- internal/app/service/signer/crypto.go | 43 - internal/app/service/signer/signer.go | 275 ----- internal/app/service/signer/signer_mock.go | 5 - internal/app/service/signer/signer_test.go | 522 -------- internal/app/service/signer/v1/attestation.go | 125 ++ .../app/service/signer/v1/attestation_test.go | 128 ++ .../app/service/signer/{ => v1}/config.go | 2 +- internal/app/service/signer/v1/crypto.go | 78 ++ .../service/signer/{ => v1}/crypto_test.go | 2 +- internal/app/service/signer/v1/signer.go | 384 ++++++ internal/app/service/signer/v1/signer_test.go | 467 ++++++++ internal/common/byteproxy/awsproxy.go | 56 + internal/common/byteproxy/awsproxy_test.go | 111 ++ internal/common/byteproxy/awsroute.go | 89 ++ internal/common/byteproxy/awsroute_test.go | 120 ++ internal/common/byteproxy/byteproxy.go | 387 ++++++ internal/common/byteproxy/byteproxy_test.go | 617 ++++++++++ internal/common/byteproxy/services.go | 29 + internal/common/byteproxy/transport.go | 32 + internal/common/config/loader.go | 9 +- internal/common/config/loader_test.go | 58 +- internal/common/grpc/client/client.go | 95 +- internal/common/grpc/client/client_test.go | 143 ++- internal/common/grpc/client/config.go | 20 +- internal/common/grpc/client/config_test.go | 20 + internal/common/grpc/server/config.go | 23 +- .../grpc/server/interceptor/recovery.go | 2 +- .../grpc/server/interceptor/recovery_test.go | 34 +- internal/common/grpc/server/option.go | 82 +- internal/common/grpc/server/option_test.go | 413 ++++++- internal/common/telemetry/tracer.go | 2 +- internal/enclave/common/crypto/bls/const.go | 2 +- .../enclave/common/crypto/ed25519/ed25519.go | 16 +- .../common/crypto/ed25519/ed25519_test.go | 49 + internal/enclave/config.go | 49 +- internal/enclave/config_test.go | 196 +-- internal/enclave/enclave.go | 101 +- internal/enclave/enclave_test.go | 105 ++ internal/enclave/enclave_vsock.go | 26 + internal/enclave/enclave_vsock_stub.go | 33 + .../enclave_vsock_stub_test.go} | 36 +- internal/enclave/enclave_vsock_test.go | 36 + .../enclave/provider/awskms/amazonroots.go | 137 +++ .../provider/awskms/amazonroots_test.go | 342 ++++++ internal/enclave/provider/awskms/awsconfig.go | 172 +++ .../enclave/provider/awskms/awsconfig_test.go | 232 ++++ .../provider/awskms/awskms.go | 212 +++- .../awskms/awskms_integration_test.go} | 62 +- .../provider/awskms/awskms_mock.go | 0 .../enclave/provider/awskms/awskms_test.go | 716 +++++++++++ .../enclave/provider/awskms/certs/README.md | 76 ++ .../awskms/certs/amazon-trust-roots.pem | 100 ++ internal/enclave/provider/awskms/config.go | 21 + internal/enclave/provider/awskms/factory.go | 26 + internal/enclave/provider/awskms/kmserror.go | 75 ++ .../enclave/provider/awskms/kmserror_test.go | 104 ++ .../awskms/testdata/kms-us-east-1-chain.pem | 75 ++ .../enclave/provider/awsproxy/awsproxy.go | 83 ++ .../provider/awsproxy/awsproxy_stub.go | 63 + .../provider/awsproxy/awsproxy_stub_test.go | 52 + .../provider/awsproxy/awsproxy_test.go | 190 +++ internal/enclave/provider/awsproxy/config.go | 52 + .../enclave/provider/awsproxy/config_test.go | 37 + internal/enclave/provider/awsproxy/options.go | 53 + .../provider/awsproxy/transport_tcp.go | 63 + .../provider/awsproxy/transport_tcp_test.go | 47 + .../provider/awsproxy/transport_vsock.go | 113 ++ internal/enclave/provider/enclave/enclave.go | 95 +- .../enclave/provider/enclave/enclave_mock.go | 84 +- .../enclave/provider/enclave/enclave_test.go | 228 ++++ .../provider/enclave/kmsrecipient/LICENSE-MIT | 21 + .../provider/enclave/kmsrecipient/ber.go | 223 ++++ .../provider/enclave/kmsrecipient/decrypt.go | 291 +++++ .../enclave/kmsrecipient/decrypt_test.go | 578 +++++++++ .../enclave/provider/keystore/keystore.go | 64 - .../provider/keystore/keystore_mock.go | 63 - .../provider/keystore/keystore_test.go | 111 -- internal/enclave/public/public.go | 3 - internal/enclave/public/public_test.go | 95 +- internal/enclave/runtime_dev.go | 44 + internal/enclave/runtime_prod_linux_cgo.go | 21 + internal/enclave/service/enclave/enclave.go | 338 +++--- .../enclave/service/enclave/enclave_test.go | 723 ++++++----- internal/enclave/service/enclave/envelope.go | 217 ++++ .../enclave/service/enclave/envelope_test.go | 125 ++ internal/enclave/service/enclave/initgate.go | 133 ++ .../service/enclave/initialization_test.go | 458 +++++++ .../enclave/service/enclave/initrunner.go | 81 ++ .../enclave/initrunner_integration_test.go | 133 ++ .../service/enclave/initrunner_test.go | 227 ++++ internal/smoke/public/attestation_dev_test.go | 18 + .../smoke/public/attestation_nitro_test.go | 18 + internal/smoke/public/public_test.go | 5 + internal/vsockproxy/config.go | 116 ++ internal/vsockproxy/config_test.go | 81 ++ internal/vsockproxy/options.go | 56 + internal/vsockproxy/ready.go | 68 ++ internal/vsockproxy/ready_test.go | 75 ++ internal/vsockproxy/routing.go | 156 +++ internal/vsockproxy/routing_test.go | 231 ++++ internal/vsockproxy/run.go | 69 ++ internal/vsockproxy/run_signal_test.go | 42 + internal/vsockproxy/run_stub.go | 77 ++ internal/vsockproxy/run_stub_test.go | 31 + internal/vsockproxy/run_test.go | 80 ++ internal/vsockproxy/runtime_dev_linux.go | 31 + internal/vsockproxy/runtime_prod_linux.go | 27 + internal/vsockproxy/runtime_prod_test.go | 40 + internal/vsockproxy/transport_tcp.go | 66 + internal/vsockproxy/transport_tcp_test.go | 57 + internal/vsockproxy/transport_vsock.go | 64 + internal/vsockproxy/transport_vsock_test.go | 40 + internal/vsockproxy/vsock_listener.go | 26 + internal/vsockproxy/vsockproxy.go | 81 ++ internal/vsockproxy/vsockproxy_stub.go | 72 ++ internal/vsockproxy/vsockproxy_stub_test.go | 50 + internal/vsockproxy/vsockproxy_test.go | 317 +++++ proto/arc/enclave/v1/enclave.proto | 101 +- proto/arc/signer/v1/signer.proto | 3 + scripts/common.sh | 5 + scripts/merge-production-coverage.sh | 190 +++ scripts/smoke-eif.sh | 277 ++++- scripts/test-build-enclave.sh | 17 + scripts/test-ci-shell.sh | 225 ++++ scripts/test-dev-image.sh | 77 ++ scripts/test-harness.sh | 23 + scripts/test-production-coverage.sh | 371 ++++++ scripts/test-production-image.sh | 131 ++ scripts/test-secure-eif.sh | 49 + scripts/test.sh | 32 + 185 files changed, 17598 insertions(+), 3048 deletions(-) create mode 100644 .github/actions/cloudsmith-login/action.yml create mode 100644 .github/workflows/build-docker.yaml create mode 100644 .github/workflows/build-enclave-image.yaml create mode 100644 .github/workflows/nitro-enclave-build.yaml create mode 100644 cmd/cmd_test.go create mode 100644 cmd/enclave/main.go create mode 100644 cmd/enclave/main_test.go create mode 100644 cmd/enclave/run_enclave_dev.go create mode 100644 cmd/enclave/run_enclave_prod.go rename cmd/{run_enclave.go => enclave/run_enclave_prod_stub.go} (61%) create mode 100644 cmd/enclave/run_enclave_prod_stub_test.go create mode 100644 cmd/run_vsockproxy.go create mode 100644 configs/app.production.yaml create mode 100755 docker/run_proxy.production.sh create mode 100644 internal/app/app_test.go create mode 100644 internal/app/enclaveinit.go create mode 100644 internal/app/enclaveinit_test.go delete mode 100644 internal/app/provider/awskms/awskms_test.go delete mode 100644 internal/app/provider/awskms/config.go create mode 100644 internal/app/runtime_dev.go create mode 100644 internal/app/runtime_dev_test.go create mode 100644 internal/app/runtime_prod.go create mode 100644 internal/app/runtime_prod_test.go delete mode 100644 internal/app/service/signer/cache_test.go delete mode 100644 internal/app/service/signer/crypto.go delete mode 100644 internal/app/service/signer/signer.go delete mode 100644 internal/app/service/signer/signer_mock.go delete mode 100644 internal/app/service/signer/signer_test.go create mode 100644 internal/app/service/signer/v1/attestation.go create mode 100644 internal/app/service/signer/v1/attestation_test.go rename internal/app/service/signer/{ => v1}/config.go (98%) create mode 100644 internal/app/service/signer/v1/crypto.go rename internal/app/service/signer/{ => v1}/crypto_test.go (99%) create mode 100644 internal/app/service/signer/v1/signer.go create mode 100644 internal/app/service/signer/v1/signer_test.go create mode 100644 internal/common/byteproxy/awsproxy.go create mode 100644 internal/common/byteproxy/awsproxy_test.go create mode 100644 internal/common/byteproxy/awsroute.go create mode 100644 internal/common/byteproxy/awsroute_test.go create mode 100644 internal/common/byteproxy/byteproxy.go create mode 100644 internal/common/byteproxy/byteproxy_test.go create mode 100644 internal/common/byteproxy/services.go create mode 100644 internal/common/byteproxy/transport.go create mode 100644 internal/enclave/enclave_test.go create mode 100644 internal/enclave/enclave_vsock.go create mode 100644 internal/enclave/enclave_vsock_stub.go rename internal/{app/service/signer/cache.go => enclave/enclave_vsock_stub_test.go} (59%) create mode 100644 internal/enclave/enclave_vsock_test.go create mode 100644 internal/enclave/provider/awskms/amazonroots.go create mode 100644 internal/enclave/provider/awskms/amazonroots_test.go create mode 100644 internal/enclave/provider/awskms/awsconfig.go create mode 100644 internal/enclave/provider/awskms/awsconfig_test.go rename internal/{app => enclave}/provider/awskms/awskms.go (52%) rename internal/{app/provider/awskms/integration_test.go => enclave/provider/awskms/awskms_integration_test.go} (77%) rename internal/{app => enclave}/provider/awskms/awskms_mock.go (100%) create mode 100644 internal/enclave/provider/awskms/awskms_test.go create mode 100644 internal/enclave/provider/awskms/certs/README.md create mode 100644 internal/enclave/provider/awskms/certs/amazon-trust-roots.pem create mode 100644 internal/enclave/provider/awskms/config.go create mode 100644 internal/enclave/provider/awskms/factory.go create mode 100644 internal/enclave/provider/awskms/kmserror.go create mode 100644 internal/enclave/provider/awskms/kmserror_test.go create mode 100644 internal/enclave/provider/awskms/testdata/kms-us-east-1-chain.pem create mode 100644 internal/enclave/provider/awsproxy/awsproxy.go create mode 100644 internal/enclave/provider/awsproxy/awsproxy_stub.go create mode 100644 internal/enclave/provider/awsproxy/awsproxy_stub_test.go create mode 100644 internal/enclave/provider/awsproxy/awsproxy_test.go create mode 100644 internal/enclave/provider/awsproxy/config.go create mode 100644 internal/enclave/provider/awsproxy/config_test.go create mode 100644 internal/enclave/provider/awsproxy/options.go create mode 100644 internal/enclave/provider/awsproxy/transport_tcp.go create mode 100644 internal/enclave/provider/awsproxy/transport_tcp_test.go create mode 100644 internal/enclave/provider/awsproxy/transport_vsock.go create mode 100644 internal/enclave/provider/enclave/enclave_test.go create mode 100644 internal/enclave/provider/enclave/kmsrecipient/LICENSE-MIT create mode 100644 internal/enclave/provider/enclave/kmsrecipient/ber.go create mode 100644 internal/enclave/provider/enclave/kmsrecipient/decrypt.go create mode 100644 internal/enclave/provider/enclave/kmsrecipient/decrypt_test.go delete mode 100644 internal/enclave/provider/keystore/keystore.go delete mode 100644 internal/enclave/provider/keystore/keystore_mock.go delete mode 100644 internal/enclave/provider/keystore/keystore_test.go create mode 100644 internal/enclave/runtime_dev.go create mode 100644 internal/enclave/runtime_prod_linux_cgo.go create mode 100644 internal/enclave/service/enclave/envelope.go create mode 100644 internal/enclave/service/enclave/envelope_test.go create mode 100644 internal/enclave/service/enclave/initgate.go create mode 100644 internal/enclave/service/enclave/initialization_test.go create mode 100644 internal/enclave/service/enclave/initrunner.go create mode 100644 internal/enclave/service/enclave/initrunner_integration_test.go create mode 100644 internal/enclave/service/enclave/initrunner_test.go create mode 100644 internal/smoke/public/attestation_dev_test.go create mode 100644 internal/smoke/public/attestation_nitro_test.go create mode 100644 internal/vsockproxy/config.go create mode 100644 internal/vsockproxy/config_test.go create mode 100644 internal/vsockproxy/options.go create mode 100644 internal/vsockproxy/ready.go create mode 100644 internal/vsockproxy/ready_test.go create mode 100644 internal/vsockproxy/routing.go create mode 100644 internal/vsockproxy/routing_test.go create mode 100644 internal/vsockproxy/run.go create mode 100644 internal/vsockproxy/run_signal_test.go create mode 100644 internal/vsockproxy/run_stub.go create mode 100644 internal/vsockproxy/run_stub_test.go create mode 100644 internal/vsockproxy/run_test.go create mode 100644 internal/vsockproxy/runtime_dev_linux.go create mode 100644 internal/vsockproxy/runtime_prod_linux.go create mode 100644 internal/vsockproxy/runtime_prod_test.go create mode 100644 internal/vsockproxy/transport_tcp.go create mode 100644 internal/vsockproxy/transport_tcp_test.go create mode 100644 internal/vsockproxy/transport_vsock.go create mode 100644 internal/vsockproxy/transport_vsock_test.go create mode 100644 internal/vsockproxy/vsock_listener.go create mode 100644 internal/vsockproxy/vsockproxy.go create mode 100644 internal/vsockproxy/vsockproxy_stub.go create mode 100644 internal/vsockproxy/vsockproxy_stub_test.go create mode 100644 internal/vsockproxy/vsockproxy_test.go create mode 100755 scripts/merge-production-coverage.sh create mode 100755 scripts/test-ci-shell.sh create mode 100755 scripts/test-dev-image.sh create mode 100644 scripts/test-harness.sh create mode 100755 scripts/test-production-coverage.sh create mode 100755 scripts/test-production-image.sh create mode 100755 scripts/test-secure-eif.sh diff --git a/.dockerignore b/.dockerignore index df41dca..0a77930 100644 --- a/.dockerignore +++ b/.dockerignore @@ -6,6 +6,7 @@ coverage/ test_results/ logs/ +bin/ artifacts/ !artifacts/nitro-enclave-image/ deploy/ diff --git a/.github/actions/cloudsmith-login/action.yml b/.github/actions/cloudsmith-login/action.yml new file mode 100644 index 0000000..71adc29 --- /dev/null +++ b/.github/actions/cloudsmith-login/action.yml @@ -0,0 +1,66 @@ +name: Cloudsmith Docker Login +description: Exchange GitHub OIDC token for Cloudsmith credentials and log in to the Docker registry. + +inputs: + registry: + description: Cloudsmith Docker registry hostname + required: true + org: + description: Cloudsmith organization + required: false + default: circle + service-slug: + description: Cloudsmith OIDC service account slug + required: false + default: arc-publisher-gha-service + +runs: + using: composite + steps: + - name: Get Cloudsmith token + id: get-token + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + env: + CLOUDSMITH_ORG: ${{ inputs.org }} + CLOUDSMITH_SERVICE_SLUG: ${{ inputs.service-slug }} + with: + script: |- + let oidc_token; + try { + oidc_token = await core.getIDToken(); + core.setSecret(oidc_token); + } catch (error) { + core.setFailed(`Failed to get OIDC token: ${error}`); + return; + } + try { + const response = await fetch(`https://api.cloudsmith.io/openid/${process.env.CLOUDSMITH_ORG}/`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + oidc_token: oidc_token, + service_slug: process.env.CLOUDSMITH_SERVICE_SLUG + }), + signal: AbortSignal.timeout(30000), + }); + if (!response.ok) { + core.setFailed(`Cloudsmith OIDC exchange failed: ${response.status} ${response.statusText}`); + return; + } + const data = await response.json(); + core.setSecret(data.token); + core.setOutput('cloudsmith_user', process.env.CLOUDSMITH_SERVICE_SLUG); + core.setOutput('cloudsmith_token', data.token); + } catch (error) { + core.setFailed(`Failed to login to Cloudsmith: ${error}`); + return; + } + + - name: Login to Cloudsmith + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 + with: + registry: ${{ inputs.registry }} + username: ${{ steps.get-token.outputs.cloudsmith_user }} + password: ${{ steps.get-token.outputs.cloudsmith_token }} diff --git a/.github/workflows/build-docker.yaml b/.github/workflows/build-docker.yaml new file mode 100644 index 0000000..3951c0d --- /dev/null +++ b/.github/workflows/build-docker.yaml @@ -0,0 +1,34 @@ +name: Build Docker + +on: + pull_request: + types: [labeled, synchronize] + branches: + - main + - release/* + push: + tags: + - "v*" + +concurrency: + group: docker-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + enclave: + if: >- + github.event_name == 'push' || + contains(github.event.pull_request.labels.*.name, 'build-docker') + uses: ./.github/workflows/nitro-enclave-build.yaml + + docker: + needs: [enclave] + uses: ./.github/workflows/build-enclave-image.yaml + with: + push: ${{ github.event_name == 'push' }} + version: ${{ github.ref_name }} + enclave_artifact_name: ${{ needs.enclave.outputs.image_artifact_name }} + enclave_file: ${{ needs.enclave.outputs.output_file }} + enclave_pcr0: ${{ needs.enclave.outputs.pcr0 }} + enclave_pcr1: ${{ needs.enclave.outputs.pcr1 }} + enclave_pcr2: ${{ needs.enclave.outputs.pcr2 }} diff --git a/.github/workflows/build-enclave-image.yaml b/.github/workflows/build-enclave-image.yaml new file mode 100644 index 0000000..b68e51a --- /dev/null +++ b/.github/workflows/build-enclave-image.yaml @@ -0,0 +1,114 @@ +name: Build Docker + +on: + workflow_call: + outputs: + docker_archive_artifact_name: + description: Docker archive artifact name for the built signer-with-enclave image + value: ${{ jobs.build.outputs.docker_archive_artifact_name }} + inputs: + runner: + type: string + default: ubuntu-latest + docker_archive_artifact_name: + description: Artifact name for an optional signer-with-enclave docker archive upload + type: string + default: "" + enclave_artifact_name: + type: string + required: true + enclave_file: + type: string + required: true + enclave_pcr0: + type: string + required: true + enclave_pcr1: + type: string + required: true + enclave_pcr2: + type: string + required: true + +jobs: + build: + name: Build signer-with-enclave + permissions: + contents: read + runs-on: ${{ inputs.runner }} + outputs: + docker_archive_artifact_name: ${{ steps.archive_meta.outputs.name }} + steps: + - name: Checkout + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Download EIF artifact + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4 + with: + name: ${{ inputs.enclave_artifact_name }} + path: artifacts/${{ inputs.enclave_artifact_name }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 + + - name: Build image + id: build + uses: docker/bake-action@82490499d2e5613fcead7e128237ef0b0ea210f7 # v7.0.0 + with: + source: . + files: docker-bake.hcl + targets: signer-with-enclave + set: | + signer-with-enclave.platform=linux/amd64 + signer-with-enclave.tags= + signer-with-enclave.output=type=oci,tar=false,dest=./oci-image + env: + BUILDX_NO_DEFAULT_ATTESTATIONS: 1 + CI: true + ENCLAVE_EIF: artifacts/${{ inputs.enclave_artifact_name }}/${{ inputs.enclave_file }} + ENCLAVE_PCR0: ${{ inputs.enclave_pcr0 }} + ENCLAVE_PCR1: ${{ inputs.enclave_pcr1 }} + ENCLAVE_PCR2: ${{ inputs.enclave_pcr2 }} + + - name: Trivy vulnerability scan + env: + TRIVY_IMAGE: aquasec/trivy:0.71.2@sha256:f5d0e600ecda7449e2a9b272805aef698631d3bb3f3a739a750de2c6819acdc9 + run: | + docker run --rm \ + -v "${GITHUB_WORKSPACE}:/workspace" \ + "${TRIVY_IMAGE}" image \ + --input /workspace/oci-image \ + --format sarif \ + --output /workspace/trivy-results.sarif \ + --severity CRITICAL \ + --ignore-unfixed \ + --exit-code 1 + + - name: Upload Trivy scan results + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: trivy-signer-with-enclave + path: trivy-results.sarif + + - name: Record docker archive artifact name + id: archive_meta + run: echo "name=${ARTIFACT_NAME}" >> "$GITHUB_OUTPUT" + env: + ARTIFACT_NAME: ${{ inputs.docker_archive_artifact_name }} + + - name: Export docker archive for smoke validation + if: ${{ inputs.docker_archive_artifact_name != '' }} + run: | + skopeo copy \ + "oci:./oci-image" \ + "docker-archive:./signer-with-enclave.tar:nitro-enclave-signer/signer-with-enclave:smoke" + + - name: Upload docker archive artifact + if: ${{ inputs.docker_archive_artifact_name != '' }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: ${{ inputs.docker_archive_artifact_name }} + path: signer-with-enclave.tar + compression-level: 0 + retention-days: 1 diff --git a/.github/workflows/nitro-enclave-build.yaml b/.github/workflows/nitro-enclave-build.yaml new file mode 100644 index 0000000..28a6684 --- /dev/null +++ b/.github/workflows/nitro-enclave-build.yaml @@ -0,0 +1,130 @@ +name: Nitro Enclave Build + +on: + workflow_call: + inputs: + runner: + description: "Runner label for the build job." + type: string + default: "ubuntu-latest" + artifact_name: + description: "Name for the uploaded EIF artifact." + type: string + default: "nitro-enclave-image" + retention_days: + description: "Artifact retention in days." + type: number + default: 1 + cli_commit: + description: >- + Commit SHA of aws-nitro-enclaves-cli to build from. + WARNING: changing this value can alter PCR0 and PCR2 measurements + because nitro-cli controls the kernel and init ramdisk bundled + into the EIF. Coordinate with downstream attestation policies. + type: string + default: "18a5f6f35f110c0f235f193ae3caff9434d64ee1" # v1.4.5 + outputs: + image_artifact_name: + description: "Name of the artifact containing the EIF." + value: ${{ jobs.build.outputs.image_artifact_name }} + output_file: + description: "Path to the EIF file within the artifact." + value: ${{ jobs.build.outputs.output_file }} + pcr0: + description: "PCR0 measurement (enclave image hash)." + value: ${{ jobs.build.outputs.pcr0 }} + pcr1: + description: "PCR1 measurement (Linux kernel hash)." + value: ${{ jobs.build.outputs.pcr1 }} + pcr2: + description: "PCR2 measurement (application hash)." + value: ${{ jobs.build.outputs.pcr2 }} + +permissions: + contents: read + +jobs: + build: + runs-on: ${{ inputs.runner }} + timeout-minutes: 30 + outputs: + image_artifact_name: ${{ steps.eif.outputs.image_artifact_name }} + output_file: ${{ steps.eif.outputs.output_file }} + pcr0: ${{ steps.eif.outputs.pcr0 }} + pcr1: ${{ steps.eif.outputs.pcr1 }} + pcr2: ${{ steps.eif.outputs.pcr2 }} + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 + + - name: Build enclave image + run: docker buildx bake --provenance=false --allow fs.read=./docker/certs enclave + + - name: Restore nitro-cli from cache + id: cli-cache + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: /tmp/nitro-install + key: nitro-cli-${{ inputs.cli_commit }}-${{ runner.os }}-${{ runner.arch }} + + - name: Install Rust toolchain + if: steps.cli-cache.outputs.cache-hit != 'true' + uses: actions-rust-lang/setup-rust-toolchain@46268bd060767258de96ed93c1251119784f2ab6 # v1 + with: + toolchain: stable + + - name: Build nitro-cli from source + if: steps.cli-cache.outputs.cache-hit != 'true' + env: + COMMIT: ${{ inputs.cli_commit }} + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + curl tar build-essential clang gcc \ + libssl-dev pkg-config musl-tools + + mkdir -p /tmp/nitro-src + curl -sSfL "https://github.com/aws/aws-nitro-enclaves-cli/archive/${COMMIT}.tar.gz" | \ + tar -C /tmp/nitro-src -xz --strip-components=1 + + cd /tmp/nitro-src + make vsock-proxy-native nitro-cli-native + mkdir -p /tmp/nitro-install + NITRO_CLI_INSTALL_DIR=/tmp/nitro-install make install-tools + + - name: Install nitro-cli + run: sudo cp -a /tmp/nitro-install/. / + + - name: Build EIF + id: eif + run: | + sudo mkdir -p /var/log/nitro_enclaves + sudo chmod 777 /var/log/nitro_enclaves + + nitro-cli build-enclave \ + --docker-uri nitro-enclave-signer/enclave:latest \ + --output-file enclave.eif + + nitro-cli describe-eif \ + --eif-path enclave.eif \ + > describe_eif.json + + echo "image_artifact_name=${{ inputs.artifact_name }}" >> "$GITHUB_OUTPUT" + echo "output_file=enclave.eif" >> "$GITHUB_OUTPUT" + pcr0=$(jq -er '.Measurements.PCR0' describe_eif.json) + pcr1=$(jq -er '.Measurements.PCR1' describe_eif.json) + pcr2=$(jq -er '.Measurements.PCR2' describe_eif.json) + echo "pcr0=${pcr0}" >> "$GITHUB_OUTPUT" + echo "pcr1=${pcr1}" >> "$GITHUB_OUTPUT" + echo "pcr2=${pcr2}" >> "$GITHUB_OUTPUT" + + - name: Upload EIF and metadata + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ${{ inputs.artifact_name }} + retention-days: ${{ inputs.retention_days }} + path: | + enclave.eif + describe_eif.json diff --git a/.github/workflows/pipeline-common-ci.yaml b/.github/workflows/pipeline-common-ci.yaml index 4925691..65d7cc1 100644 --- a/.github/workflows/pipeline-common-ci.yaml +++ b/.github/workflows/pipeline-common-ci.yaml @@ -13,15 +13,15 @@ jobs: - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 with: - go-version: "1.24" + go-version-file: go.mod - name: Install dependencies run: | sudo apt-get update -y sudo apt-get install -y protobuf-compiler libprotobuf-dev gcc-aarch64-linux-gnu sudo ln -sf /usr/libexec/docker/cli-plugins/docker-compose /usr/local/bin/docker-compose - go install google.golang.org/protobuf/cmd/protoc-gen-go@latest - go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest + go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.36.6 + go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.5.1 - name: Run all tests run: make test-all diff --git a/Makefile b/Makefile index 227d0e5..6ffc35f 100644 --- a/Makefile +++ b/Makefile @@ -15,8 +15,8 @@ proto: ## Generate protocol buffers build: proto ## Build application binary @$(SCRIPTS)/build.sh -local-enclave-docker: build ## Build local enclave docker image - @docker buildx bake --set "*.tags=$(ENCLAVE_NAME):local" signer +local-enclave-docker: ## Build local enclave docker image + @docker buildx bake --set "*.tags=$(ENCLAVE_NAME):local" signer-dev #------------------------------------------------------------------------------ # Test @@ -25,7 +25,7 @@ local-enclave-docker: build ## Build local enclave docker image test: build ## Run unit tests and linting @$(SCRIPTS)/test.sh -test-it: up ## Run integration tests +test-it: build up ## Run integration tests @RUN_IT_TESTS=true $(SCRIPTS)/test.sh test-all: test-it smoke ## Run all tests @@ -40,7 +40,7 @@ test-reproducibility: ## Verify enclave build reproducibility # Development #------------------------------------------------------------------------------ -dev: up ## Start local development +dev: build up ## Start local development @$(SCRIPTS)/dev.sh up: local-enclave-docker ## Start dependencies diff --git a/README.md b/README.md index c816260..59f36a1 100644 --- a/README.md +++ b/README.md @@ -8,20 +8,20 @@ Secure cryptographic signing service for Arc Chain validators, powered by AWS Ni ## Overview -Arc Remote Signer is a gRPC microservice designed specifically for Arc Chain validators, providing cryptographic key generation and signing operations with hardware-level security guarantees from AWS Nitro Enclaves. The service implements a dual-process architecture where a Proxy (outside the enclave) handles incoming requests while the Enclave (inside Nitro Enclave) performs all cryptographic operations. +Arc Remote Signer is a gRPC microservice designed specifically for Arc Chain validators, providing cryptographic key generation and signing operations with hardware-level security guarantees from AWS Nitro Enclaves. The service runs as three runtime processes across two trust domains: a host application and standalone KMS traffic proxy outside the enclave, plus the signing application inside the Nitro Enclave. -Traditional key management systems store private keys in memory, making them vulnerable to compromise from privileged users, malware, or system vulnerabilities. Arc Remote Signer addresses these risks by isolating all cryptographic operations inside an AWS Nitro Enclaveβ€”a hardware-backed trusted execution environment with cryptographic attestation capabilities. +Host-based signing services expose private keys to the host process's memory, where privileged access, malware, or system vulnerabilities can compromise them. Arc Remote Signer instead isolates private-key operations inside an AWS Nitro Enclaveβ€”a hardware-backed trusted execution environment with cryptographic attestation capabilities. -Deployed as a 1-to-1 sidecar alongside Arc Chain validator nodes, Arc Remote Signer provides verifiable proof that signing operations occur within a secure enclave, enabling zero-trust architectures where even the host system cannot access private key material. +Deployed as a 1-to-1 sidecar alongside Arc Chain validator nodes, Arc Remote Signer uses enclave attestation to bind KMS access to an approved enclave image, so the host system cannot access plaintext private keys or data keys. ## Features - πŸ” **Hardware-isolated operations** - Validator key operations isolated in AWS Nitro Enclaves - πŸ”‘ **Ed25519 signatures** - Primary signing algorithm for Arc Chain validators with BLS support -- πŸ“œ **Cryptographic attestation** - Verifiable proof of enclave execution for validator operations -- πŸ”„ **Envelope encryption** - Four-layer key protection (validator key β†’ data key β†’ KMS key β†’ enclave key) +- πŸ“œ **Cryptographic attestation** - Attestation-bound KMS access for approved enclave images +- πŸ”„ **Envelope encryption** - Private keys wrapped by KMS-protected data keys, with attestation-bound KMS responses decryptable only inside the enclave - πŸš€ **High-performance gRPC** - Efficient binary protocol for low-latency validator signing -- πŸ“Š **Full observability** - OpenTelemetry tracing, Datadog metrics, and an optional Prometheus endpoint +- πŸ“Š **Host observability** - OpenTelemetry tracing, Datadog metrics, and an optional Prometheus endpoint for the host application - πŸ—οΈ **Sidecar architecture** - 1-to-1 deployment alongside Arc Chain validator nodes - βœ… **Comprehensive testing** - Unit, integration, and smoke test coverage @@ -63,13 +63,13 @@ If all tests pass, the service is ready to use! ## Documentation -- [`docs/architecture.md`](./docs/architecture.md) β€” dual-process architecture, runtime flows, security model, deployment, API reference, troubleshooting +- [`docs/architecture.md`](./docs/architecture.md) β€” runtime processes and trust domains, runtime flows, security model, deployment, API reference, troubleshooting - [`docs/development.md`](./docs/development.md) β€” prerequisites, build/run commands, coding conventions, pre-commit hooks - [`docs/testing.md`](./docs/testing.md) β€” test matrix and conventions ## Architecture -Arc Remote Signer uses a **dual-process architecture** with clear security boundaries between the Proxy (outside enclave) and Enclave (inside Nitro Enclave): +Arc Remote Signer uses **three runtime processes across two trust domains**: the host application and standalone `app run-vsockproxy` process run on the EC2 host, while the enclave application runs inside the Nitro Enclave: ``` β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” @@ -78,16 +78,16 @@ Arc Remote Signer uses a **dual-process architecture** with clear security bound β”‚ gRPC (sidecar) β–Ό β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ Proxy (Host) β”‚ +β”‚ EC2 Host β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ -β”‚ β”‚ gRPC API β”‚ β”‚ Signer β”‚ β”‚ Providers β”‚ β”‚ -β”‚ β”‚ Handlers │─▢│ Service │─▢│ (KMS, Secretsβ”‚ β”‚ -β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ Manager) β”‚ β”‚ -β”‚ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ -β”‚ β”‚ vsock/gRPC β”‚ -β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - β”‚ - β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β–Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ β”‚ gRPC API β”‚ β”‚ Signer β”‚ β”‚ Secrets β”‚ β”‚ +β”‚ β”‚ Handlers │─▢│ Service │─▢│ Manager β”‚ β”‚ +β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ +β”‚ Host app (`app run`) β”‚ β”‚ +β”‚ β”‚ gRPC: TCP (dev/CI), VSOCK (prod)β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β–Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ Nitro Enclave β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ β”‚ β”‚ Enclave gRPC β”‚ β”‚ @@ -95,23 +95,30 @@ Arc Remote Signer uses a **dual-process architecture** with clear security bound β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ β”‚ β”‚ β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β–Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ - β”‚ β”‚ Key Decryption β”‚ β”‚ - β”‚ β”‚ & Signing β”‚ β”‚ + β”‚ β”‚ KMS Client, Key β”‚ β”‚ + β”‚ β”‚ Decrypt & Signingβ”‚ β”‚ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ β”‚ β”‚ β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β–Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ - β”‚ β”‚ NSM Attestation β”‚ β”‚ + β”‚ β”‚ NSM + awsproxy β”‚ β”‚ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ KMS traffic + β–Ό +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ EC2 Host: standalone `app run-vsockproxy` process ───▢ KMS β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ ``` ### Key Components -1. **Proxy** (`internal/app/`) - Runs outside the enclave, handles validator signing requests, manages AWS integrations (KMS for key decryption, Secrets Manager for configuration), caches attestation documents, and proxies signing operations to the enclave over vsock. +1. **Host application** (`app run`, `internal/app/`) - Runs outside the enclave, handles validator requests, and persists the wrapped key in AWS Secrets Manager. During startup, it passes temporary AWS credentials, KMS key ARNs, the LocalStack selector, and either the stored envelope or a key-generation request to enclave `Initialize`. It does not construct KMS requests or handle plaintext data keys. -2. **Enclave** (`internal/enclave/`) - Runs inside an AWS Nitro Enclave, performs all cryptographic operations (key decryption, validator signing), maintains secure key material in memory, and generates attestation documents proving execution within the secure environment. +2. **KMS traffic proxy** (`app run-vsockproxy`, `internal/vsockproxy/`) - Runs as a separate host-side process. It routes opaque KMS traffic to LocalStack or to the regional AWS endpoint selected by the enclave. It restricts AWS routes to the regions in the configured KMS key ARNs. -3. **Communication Layer** - Proxy and enclave communicate via gRPC over vsock (virtual socket), ensuring type-safe, versioned contracts between trust boundaries. Validator nodes communicate with the Proxy via standard gRPC. +3. **Enclave application** (`internal/enclave/`) - Runs inside an AWS Nitro Enclave. Each `Initialize` attempt creates an ephemeral AWS KMS client, attaches NSM-signed attestation to KMS requests, and routes them through the enclave-side `awsproxy`. A successful attempt installs one signing key in enclave memory. Later public-key and signing requests use that key without KMS. + +4. **Communication layer** - The host and enclave applications use gRPC over TCP on localhost in dev/CI and over VSOCK in Nitro production. Validator nodes communicate with the host application via standard gRPC. For detailed layer architecture and shared infrastructure, see [`docs/architecture.md`](./docs/architecture.md). @@ -176,26 +183,104 @@ func main() { } ``` +The example uses the default plaintext listener for local development. Production can enable server-authenticated TLS with the `APP_PUBLIC_SERVER_TLS_ENABLED`, `APP_PUBLIC_SERVER_TLS_CERT`, and `APP_PUBLIC_SERVER_TLS_KEY` settings documented in [`docs/architecture.md`](./docs/architecture.md#common-production-settings). It can optionally require client certificates by enabling `APP_PUBLIC_SERVER_TLS_CLIENTAUTH_ENABLED` and configuring `APP_PUBLIC_SERVER_TLS_CLIENTAUTH_CA` with a dedicated client CA bundle. The application authenticates membership in that CA; restricting access to Malachite depends on provisioning its client certificates exclusively to that workload. + ### EnclaveService API (Internal) -**EnclaveService** is an **internal API** inside the Nitro Enclave, **not accessible to external clients**: +**EnclaveService** is an **internal API** served inside the Nitro Enclave. It is not exposed as a validator-facing network API: -- **Port**: 10350 (internal vsock only, accessible only by Proxy) +- **Port**: 10350 (TCP on localhost in dev/CI; VSOCK in Nitro production) - **Proto**: `proto/arc/enclave/v1/enclave.proto` -- **Methods**: `GenerateKey()`, `GetPublicKey()`, `SignMessage()`, `GetAttestation()` -- **Use case**: Internal communication between Proxy and Enclave over vsock +- **Methods**: `Initialize()`, `GenerateKey()`, `GetPublicKey()`, `SignMessage()` +- **Use case**: Internal communication between the host and enclave applications -This full-featured API is used exclusively by the Proxy to communicate with the Enclave for cryptographic operations. Arc Chain validators do not interact with this API directly. The EnclaveService handles key generation, encrypted key material management, signing operations, and attestation document generationβ€”all within the secure enclave environment. +The host application calls this API over TCP in dev/CI or VSOCK in Nitro +production. Arc Chain validators do not interact with this API directly. +`Initialize` generates or recovers the signing key and returns its public key +and attestation. `GetPublicKey` refreshes the public-key attestation. +`SignMessage` receives only the message. `GenerateKey` is retained for protocol +compatibility and rejects direct key generation. ## Deployment Arc Remote Signer is deployed as a 1-to-1 sidecar with Arc Chain validator nodes on Nitro-enabled AWS EC2 instances. +### Release and rollback operations + +The internal `Initialize` protocol changed in v1.7.7. Upgrade or roll back the +host and enclave as one unit. A mixed-version host and enclave are not +compatible. + +If a restarted host receives a stale replayed attestation from a long-lived +enclave, restart the enclave with the host. Use this procedure until upstream +re-attestation behavior is fixed. + ## Reproducible Enclave Builds The enclave Docker image is built reproducibly β€” identical source produces bit-for-bit identical images across builds. This ensures the PCR0 hash (derived from the enclave image) is stable and predictable, which is critical for attestation policy. -The enclave build uses a dedicated `docker/Dockerfile.enclave`, separate from the main `docker/Dockerfile` used by the signer targets. Key techniques: +Circle publishes the tested Arc Remote Signer image to +`docker.cloudsmith.io/circle/arc/arc-remote-signer`. Circle sends the Cloudsmith +entitlement token to each authorized partner in a secure email. Circle sends +the versioned image reference in the partner Slack channel. The operator must +pull the supplied image. The operator must record the image digest. The image +labels contain PCR0, PCR1, and PCR2. The KMS policy in this repository requires +PCR0, PCR1, PCR2, and PCR4 (derived from the parent EC2 instance ID). The +operator must use the PCR values from the pulled image and parent instance. The +operator does not have to build the Circle-published image. + +Circle syncs the source code, but not the Circle-published image, to the public +`circlefin/arc-remote-signer` repository. + +An operator can build a custom image from the public source code instead of +using the Circle-published image. The operator controls the build environment. +The operator configures the KMS key policy with the measured PCR0, PCR1, and +PCR2 values and the parent instance PCR4 value. A reproducible build permits +independent verification, but it does not establish trust in the build +environment. + +A custom image is not the Circle-published Arc Remote Signer artifact. The +operator is responsible for the integrity of the custom build environment. + +The enclave build uses `docker/Dockerfile.enclave`. This build uses the +`prod` tag. The tag selects the Nitro runner and the VSOCK transport. The final +image contains only the enclave executable, `configs/enclave.yaml`, and the +startup script. + +A `go list -deps` test examines the dependencies of `cmd/enclave`. The test fails if `cmd/enclave` imports the generic command package or a host-only dependency that is in the test list. + +The signer targets use `docker/Dockerfile`. The production targets build the +host with the `prod` tag. This tag sets the host transport to +VSOCK and disables LocalStack. The production image contains only the +production host executable and production configuration. The production +launcher always starts the VSOCK proxy, the EIF, and the host application. + +Local development and CI use the `signer-dev` target. This target uses the +development host build and the development enclave build. It supports TCP and +LocalStack. The production targets do not contain its executable, configuration, +or startup script. + +Production uses one enclave target and one EIF. The build configuration has no +debug EIF target. +The EIF does not read `APP_ENV` or Datadog variables from the host. It does not +send logs, traces, or metrics to the host. The enclave launcher sends standard +output and standard error to `/dev/null`. + +The `signer-dev` image is not an EIF. It can show local enclave process logs for +development. It does not have the security boundary of the production EIF. + +The release pipeline does a test of the `signer-with-enclave` Docker archive on +a Nitro runner. The test starts the EIF with Nitro debug mode. Debug mode is a +runtime option. It does not create or change EIF bytes. The pipeline publishes +the tested image to Cloudsmith and ECR. Operators must pull the supplied image +reference. Operators must then record the image digest. Operators must use the +recorded image digest in the deployment configuration. Staging and production +must use the recorded digest. + +Nitro debug mode does not validate a production KMS policy that uses PCR values. +A production-mode test is necessary for each PCR policy change. + +Key reproducibility techniques: - **Digest-pinned base images** (Go toolchain + Debian) - **Snapshot-pinned apt** via `snapshot.debian.org` for deterministic packages - **In-Docker Go build** with `-trimpath -buildvcs=false -ldflags=-buildid=` @@ -223,12 +308,14 @@ This project uses `make` targets as the standard workflow entry points. For full β”‚ β”œβ”€β”€ app/ # Proxy (outside enclave) β”‚ β”‚ β”œβ”€β”€ public/ # gRPC handlers β”‚ β”‚ β”œβ”€β”€ service/ # Business logic -β”‚ β”‚ └── provider/ # AWS integrations (KMS, Secrets Manager) +β”‚ β”‚ └── provider/ # Secrets Manager and enclave client β”‚ β”œβ”€β”€ enclave/ # Enclave (inside Nitro Enclave) β”‚ β”‚ β”œβ”€β”€ public/ # Enclave gRPC handlers β”‚ β”‚ β”œβ”€β”€ service/ # Enclave business logic -β”‚ β”‚ └── provider/ # Key storage, attestation +β”‚ β”‚ └── provider/ # KMS, key storage, attestation +β”‚ β”œβ”€β”€ vsockproxy/ # Host-side vsock↔AWS KMS bridge (run-vsockproxy) β”‚ β”œβ”€β”€ common/ # Shared infrastructure +β”‚ β”‚ β”œβ”€β”€ byteproxy/ # vsock byte proxy + AWS route framing β”‚ β”‚ β”œβ”€β”€ crypto/ # AES, random generation β”‚ β”‚ β”œβ”€β”€ grpc/ # gRPC client/server/interceptors β”‚ β”‚ β”œβ”€β”€ logging/ # Structured logging @@ -297,7 +384,7 @@ This project was developed by the Circle engineering team. **AWS Nitro Enclaves Documentation**: - [AWS Nitro Enclaves](https://docs.aws.amazon.com/enclaves/latest/user/nitro-enclave.html) -- [Nitro System Manager Documentation](https://docs.aws.amazon.com/enclaves/latest/user/set-up-nitro-enclave-dev-environment.html) +- [Nitro Enclaves development environment](https://docs.aws.amazon.com/enclaves/latest/user/set-up-nitro-enclave-dev-environment.html) - [Attestation Document Format](https://docs.aws.amazon.com/enclaves/latest/user/verify-root.html) **Cryptography References**: diff --git a/cmd/cmd.go b/cmd/cmd.go index 111ce2d..947e902 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -19,15 +19,12 @@ import ( "os" "github.com/circlefin/arc-remote-signer/internal/app" - "github.com/circlefin/arc-remote-signer/internal/enclave" "github.com/spf13/cobra" ) var ( - cfgFile string - neCfgFile string - cfg *app.Config - neCfg *enclave.Config + cfgFile string + cfg *app.Config rootCmd = &cobra.Command{ Use: "app", @@ -40,8 +37,7 @@ var ( ) func init() { - rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (by default looks for ./app.yaml, ./configs/app.yaml, or /etc/app/app.yaml)") - rootCmd.PersistentFlags().StringVar(&neCfgFile, "enclave-config", "", "enclave config file (by default looks for ./enclave.yaml, ./configs/enclave.yaml, or /etc/app/enclave.yaml)") + rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default: config.yaml in . or ./configs)") } // Execute is the primary entrypoint for the app. diff --git a/cmd/cmd_test.go b/cmd/cmd_test.go new file mode 100644 index 0000000..eeeeb7e --- /dev/null +++ b/cmd/cmd_test.go @@ -0,0 +1,32 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestRootCommandExcludesEnclaveRuntime(t *testing.T) { + commandNames := make([]string, 0, len(rootCmd.Commands())) + for _, command := range rootCmd.Commands() { + commandNames = append(commandNames, command.Name()) + } + + require.Contains(t, commandNames, "run") + require.Contains(t, commandNames, "run-vsockproxy") + require.NotContains(t, commandNames, "run-enclave") + require.Nil(t, rootCmd.PersistentFlags().Lookup("enclave-config")) +} diff --git a/cmd/enclave/main.go b/cmd/enclave/main.go new file mode 100644 index 0000000..dffc276 --- /dev/null +++ b/cmd/enclave/main.go @@ -0,0 +1,54 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package main starts only the enclave service. +package main + +import ( + "errors" + "flag" + "fmt" + "os" + "strings" + + commonConfig "github.com/circlefin/arc-remote-signer/internal/common/config" + "github.com/circlefin/arc-remote-signer/internal/enclave" +) + +type loadConfigFunc func(commonConfig.ApplicationConfig, string) +type runEnclaveFunc func(*enclave.Config) error + +func main() { + if err := run(os.Args[1:], commonConfig.LoadConfig, runConfiguredEnclave); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func run(args []string, loadConfig loadConfigFunc, runEnclave runEnclaveFunc) error { + flags := flag.NewFlagSet("enclave", flag.ContinueOnError) + configFile := flags.String("enclave-config", "", "enclave config file") + if err := flags.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + return nil + } + return err + } + if flags.NArg() != 0 { + return fmt.Errorf("unexpected arguments: %s", strings.Join(flags.Args(), " ")) + } + + cfg := enclave.NewConfig() + loadConfig(cfg, *configFile) + return runEnclave(cfg) +} diff --git a/cmd/enclave/main_test.go b/cmd/enclave/main_test.go new file mode 100644 index 0000000..6428c3f --- /dev/null +++ b/cmd/enclave/main_test.go @@ -0,0 +1,110 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "os/exec" + "strings" + "testing" + + commonConfig "github.com/circlefin/arc-remote-signer/internal/common/config" + "github.com/circlefin/arc-remote-signer/internal/enclave" + "github.com/stretchr/testify/require" +) + +func TestRun(t *testing.T) { + t.Run("loads selected config and starts enclave", func(t *testing.T) { + var loadedConfig commonConfig.ApplicationConfig + var loadedPath string + var startedConfig *enclave.Config + + err := run( + []string{"--enclave-config", "/tmp/enclave.yaml"}, + func(cfg commonConfig.ApplicationConfig, path string) { + loadedConfig = cfg + loadedPath = path + }, + func(cfg *enclave.Config) error { + startedConfig = cfg + return nil + }, + ) + + require.NoError(t, err) + require.Equal(t, "/tmp/enclave.yaml", loadedPath) + require.Same(t, loadedConfig, startedConfig) + }) + + t.Run("rejects legacy cobra subcommand", func(t *testing.T) { + called := false + err := run( + []string{"run-enclave"}, + func(commonConfig.ApplicationConfig, string) { called = true }, + func(*enclave.Config) error { + called = true + return nil + }, + ) + + require.ErrorContains(t, err, "unexpected arguments") + require.False(t, called) + }) + + t.Run("prints help without starting enclave", func(t *testing.T) { + called := false + err := run( + []string{"--help"}, + func(commonConfig.ApplicationConfig, string) { called = true }, + func(*enclave.Config) error { + called = true + return nil + }, + ) + + require.NoError(t, err) + require.False(t, called) + }) +} + +func TestEnclaveDependencyBoundary(t *testing.T) { + output, err := exec.Command("go", "list", "-deps", ".").CombinedOutput() + require.NoError(t, err, string(output)) + + for _, dependency := range strings.Fields(string(output)) { + require.Falsef(t, isForbiddenDependency(dependency), "enclave entrypoint imports forbidden dependency %s", dependency) + } +} + +func isForbiddenDependency(dependency string) bool { + for _, exact := range []string{ + "github.com/circlefin/arc-remote-signer/cmd", + "github.com/spf13/cobra", + } { + if dependency == exact { + return true + } + } + + for _, subtree := range []string{ + "github.com/circlefin/arc-remote-signer/internal/app", + "github.com/circlefin/arc-remote-signer/internal/vsockproxy", + "github.com/aws/aws-sdk-go-v2/service/secretsmanager", + } { + if dependency == subtree || strings.HasPrefix(dependency, subtree+"/") { + return true + } + } + + return false +} diff --git a/cmd/enclave/run_enclave_dev.go b/cmd/enclave/run_enclave_dev.go new file mode 100644 index 0000000..0804e02 --- /dev/null +++ b/cmd/enclave/run_enclave_dev.go @@ -0,0 +1,22 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !prod + +package main + +import "github.com/circlefin/arc-remote-signer/internal/enclave" + +func runConfiguredEnclave(cfg *enclave.Config) error { + return enclave.Run(cfg) +} diff --git a/cmd/enclave/run_enclave_prod.go b/cmd/enclave/run_enclave_prod.go new file mode 100644 index 0000000..d8caa63 --- /dev/null +++ b/cmd/enclave/run_enclave_prod.go @@ -0,0 +1,22 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build prod && linux && cgo + +package main + +import "github.com/circlefin/arc-remote-signer/internal/enclave" + +func runConfiguredEnclave(cfg *enclave.Config) error { + return enclave.RunNitro(cfg) +} diff --git a/cmd/run_enclave.go b/cmd/enclave/run_enclave_prod_stub.go similarity index 61% rename from cmd/run_enclave.go rename to cmd/enclave/run_enclave_prod_stub.go index e47c9cb..9db8e99 100644 --- a/cmd/run_enclave.go +++ b/cmd/enclave/run_enclave_prod_stub.go @@ -11,27 +11,18 @@ // See the License for the specific language governing permissions and // limitations under the License. -package cmd +//go:build prod && (!linux || !cgo) + +package main import ( - "github.com/circlefin/arc-remote-signer/internal/common/config" + "errors" + "github.com/circlefin/arc-remote-signer/internal/enclave" - "github.com/spf13/cobra" ) -func init() { - rootCmd.AddCommand(runEnclaveCmd) -} +var errProductionEnclaveUnsupported = errors.New("production enclave requires Linux and cgo") -var runEnclaveCmd = &cobra.Command{ - Use: "run-enclave", - Short: "Run the enclave service.", - Run: func(_ *cobra.Command, _ []string) { - neCfg = enclave.NewConfig() - config.LoadConfig(neCfg, neCfgFile) - err := enclave.Run(neCfg) - if err != nil { - panic(err) - } - }, +func runConfiguredEnclave(*enclave.Config) error { + return errProductionEnclaveUnsupported } diff --git a/cmd/enclave/run_enclave_prod_stub_test.go b/cmd/enclave/run_enclave_prod_stub_test.go new file mode 100644 index 0000000..c38ae90 --- /dev/null +++ b/cmd/enclave/run_enclave_prod_stub_test.go @@ -0,0 +1,29 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build prod && (!linux || !cgo) + +package main + +import ( + "testing" + + "github.com/circlefin/arc-remote-signer/internal/enclave" + "github.com/stretchr/testify/require" +) + +func TestRunConfiguredEnclave_RejectsUnsupportedProductionBuild(t *testing.T) { + err := runConfiguredEnclave(enclave.NewConfig()) + + require.ErrorIs(t, err, errProductionEnclaveUnsupported) +} diff --git a/cmd/run_vsockproxy.go b/cmd/run_vsockproxy.go new file mode 100644 index 0000000..1c8b863 --- /dev/null +++ b/cmd/run_vsockproxy.go @@ -0,0 +1,39 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "github.com/circlefin/arc-remote-signer/internal/common/config" + "github.com/circlefin/arc-remote-signer/internal/vsockproxy" + "github.com/spf13/cobra" +) + +func init() { + rootCmd.AddCommand(runVsockproxyCmd) +} + +var runVsockproxyCmd = &cobra.Command{ + Use: "run-vsockproxy", + Short: "Run the standalone vsockproxy bridge process.", + // Startup failures are reported once by Execute() as a single stderr + // line with exit code 1; silence cobra's default usage dump and + // duplicate error print so they match the lifecycle.Manager path. + SilenceUsage: true, + SilenceErrors: true, + RunE: func(_ *cobra.Command, _ []string) error { + vsockproxyCfg := vsockproxy.NewConfig() + config.LoadConfig(vsockproxyCfg, cfgFile) + return vsockproxy.Run(vsockproxyCfg) + }, +} diff --git a/configs/.env.dev b/configs/.env.dev index 8f790a0..a56780c 100644 --- a/configs/.env.dev +++ b/configs/.env.dev @@ -1,4 +1,13 @@ -USE_LOCALSTACK=true -APP_PROVIDER_SECRETS_LOCALSTACK_ENABLED=true -APP_PROVIDER_AWSKMS_LOCALSTACK_ENABLED=true -APP_PROVIDER_ENCLAVE_NITROENCLAVE_ENABLED=false +USE_LOCALSTACK=${USE_LOCALSTACK:-true} +APP_PROVIDER_SECRETS_LOCALSTACK_ENABLED=${APP_PROVIDER_SECRETS_LOCALSTACK_ENABLED:-true} +APP_PROVIDER_AWSKMS_LOCALSTACK_ENABLED=${APP_PROVIDER_AWSKMS_LOCALSTACK_ENABLED:-true} +# Single switch for Nitro enclave mode in local dev/CI. common.sh derives the +# host app flag from it and docker-compose feeds it into both the vsockproxy and +# enclave services, so nothing can drift. The ${:-} form lets an ambient +# ENABLE_ENCLAVE override the default without editing this file. +# +# Keep false: plain Docker has no Nitro device / NSM, so docker-compose only +# exercises the non-Nitro (TCP + LocalStack) path β€” setting this true here would +# not produce a working Nitro setup. Real Nitro mode is exercised by the EIF +# smoke test on a Nitro host (scripts/smoke-eif.sh), not via compose. +ENABLE_ENCLAVE=${ENABLE_ENCLAVE:-false} diff --git a/configs/app.production.yaml b/configs/app.production.yaml new file mode 100644 index 0000000..324b03f --- /dev/null +++ b/configs/app.production.yaml @@ -0,0 +1,52 @@ +env: prod +public: + server: + host: 0.0.0.0 + port: 10340 + tls: + enabled: false + cert: "" + key: "" + clientAuth: + enabled: false + ca: "" + reflection: + enabled: false +service: + signer: + keyId: "00000000-0000-0000-0000-000000000000" + algorithm: "ed25519" +provider: + enclave: + startupTimeoutMS: 30000 + client: + requestTimeoutMS: 500 + methods: + initialize: + timeoutMS: 30000 + maxAttempts: 1 + generatekey: + timeoutMS: 30000 + maxAttempts: 1 + getpublickey: + timeoutMS: 30000 + signmessage: + timeoutMS: 500 + maxAttempts: 1 + awskms: + arns: [] +profiler: + enabled: false +metrics: + statsd: + host: 127.0.0.1 + port: 8125 + telemetryDisabled: true + prometheus: + enabled: false + host: 0.0.0.0 + port: 9090 + path: /metrics +vsockproxy: + basePort: 10316 + maxConns: 50 diff --git a/configs/app.yaml b/configs/app.yaml index 65afcee..f82df13 100644 --- a/configs/app.yaml +++ b/configs/app.yaml @@ -8,6 +8,13 @@ public: enabled: false cert: "" key: "" + # clientAuth upgrades server-authenticated TLS to mutual TLS. Use a + # dedicated CA that issues certificates only to authorized clients. + clientAuth: + enabled: false + ca: "" + reflection: + enabled: false service: signer: keyId: "00000000-0000-0000-0000-000000000000" @@ -19,6 +26,7 @@ provider: # config for external services endpoint: http://localhost:4566 region: us-east-1 enclave: + startupTimeoutMS: 30000 nitroEnclave: enabled: false # Set to true in production with Nitro Enclave (uses VSOCK) cid: 16 # Enclave CID (typically 16 for first enclave) @@ -29,13 +37,22 @@ provider: # config for external services retry: maxAttempts: 1 requestTimeoutMS: 500 + methods: + initialize: + timeoutMS: 30000 + maxAttempts: 1 + generatekey: + timeoutMS: 30000 + maxAttempts: 1 + getpublickey: + timeoutMS: 30000 + signmessage: + timeoutMS: 500 + maxAttempts: 1 awskms: localstack: enabled: false - endpoint: http://localhost:4566 - region: us-east-1 arns: - connectTimeout: 500 profiler: # These are the dev values. Even if we didn't want to specify dev values, we would still have to specify # the structure here so Viper knows to look for them in environment variables @@ -53,3 +70,12 @@ metrics: host: 0.0.0.0 port: 9090 path: /metrics +# vsockproxy configures the standalone host-side bridge process started via +# `app run-vsockproxy`. `app run` ignores this block. The transport is +# selected by provider.enclave.nitroEnclave.enabled above (shared with the +# app process). Override fields with APP_VSOCKPROXY_* environment variables. +vsockproxy: + basePort: 10316 + maxConns: 50 + # Standard AWS KMS requests use the ARN regions as an allowlist. + # LocalStack requests use the fixed compose service endpoint. diff --git a/configs/enclave.yaml b/configs/enclave.yaml index 4ae2015..b0b0f37 100644 --- a/configs/enclave.yaml +++ b/configs/enclave.yaml @@ -4,3 +4,8 @@ public: port: 10350 nitroEnclave: enabled: true + awsproxyEndpoint: http://127.0.0.1:10316 + kmsConnectTimeoutMs: 3000 +awsproxy: + base_port: 10316 + max_conns: 50 diff --git a/deployments/docker-compose.yaml b/deployments/docker-compose.yaml index 3879f4e..7a73851 100644 --- a/deployments/docker-compose.yaml +++ b/deployments/docker-compose.yaml @@ -5,18 +5,42 @@ x-health-retry: retries: 30 services: + vsockproxy: + image: "nitro-enclave-signer-internal:local" + environment: + # Single switch: ENABLE_ENCLAVE (from configs/.env.dev) drives both this + # host-side flag and the enclave service's APP_NITROENCLAVE_ENABLED below, + # so the two cannot drift. + APP_PROVIDER_ENCLAVE_NITROENCLAVE_ENABLED: "${ENABLE_ENCLAVE:-false}" + APP_PROVIDER_AWSKMS_LOCALSTACK_ENABLED: "${APP_PROVIDER_AWSKMS_LOCALSTACK_ENABLED:-true}" + APP_PROVIDER_AWSKMS_ARNS: "${APP_PROVIDER_AWSKMS_ARNS:-arn:aws:kms:us-east-1:000000000000:alias/dev-multi-region-crypto,arn:aws:kms:us-west-2:000000000000:alias/dev-multi-region-crypto}" + APP_VSOCKPROXY_BASEPORT: "10316" + APP_VSOCKPROXY_MAXCONNS: "50" + # Arm the readiness sentinel: run-vsockproxy writes this file once it + # has bound its listener, so the healthcheck below can gate the enclave + # on the proxy actually accepting connections. Same mechanism prod uses + # via docker/run.sh. + VSOCKPROXY_READY_FILE: /tmp/vsockproxy.ready + entrypoint: ["/usr/local/circle/app", "--config", "/usr/local/circle/configs/app.yaml", "run-vsockproxy"] + healthcheck: + # Healthy only after the TCP listener (10316) is bound (sentinel + # written). Polls fast so it does not add ~10s to enclave startup. + test: ["CMD-SHELL", "test -e /tmp/vsockproxy.ready"] + interval: 1s + timeout: 2s + retries: 30 + depends_on: + localstack: + condition: service_healthy + enclave: image: "nitro-enclave-signer-internal:local" environment: - APP_ENV: ${APP_ENV:-dev} - AWS_PROFILE: ${AWS_PROFILE:-} - AWS_ACCESS_KEY_ID: foo - AWS_SECRET_ACCESS_KEY: bar - APP_PROVIDER_AWSKMS_LOCALSTACK_ENABLED: true - APP_PROVIDER_AWSKMS_LOCALSTACK_ENDPOINT: http://localstack:4566 - APP_PROVIDER_AWSKMS_LOCALSTACK_REGION: us-east-1 - APP_PROVIDER_AWSKMS_ARNS: arn:aws:kms:us-east-1:000000000000:alias/dev-multi-region-crypto,arn:aws:kms:us-west-2:000000000000:alias/dev-multi-region-crypto - APP_NITROENCLAVE_ENABLED: false + # Derived from the same single switch (ENABLE_ENCLAVE) as the vsockproxy + # host flag above, so host and enclave agree on Nitro mode. + APP_NITROENCLAVE_ENABLED: "${ENABLE_ENCLAVE:-false}" + # The enclave routes KMS traffic through awsproxy to the host-side + # vsockproxy. The host selects LocalStack or AWS KMS. ulimits: memlock: -1 ports: @@ -26,6 +50,8 @@ services: <<: *health-retry entrypoint: ["/usr/local/circle/run_enclave.dev.sh"] depends_on: + vsockproxy: + condition: service_healthy localstack: condition: service_healthy diff --git a/docker-bake.hcl b/docker-bake.hcl index ab421de..58b9523 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -51,6 +51,24 @@ group "default" { targets = ["signer"] } +// The signer-dev image contains the host executable and the standalone enclave executable. +target "signer-dev-meta-target" { + context = "." + dockerfile = "docker/Dockerfile" + target = "signer-dev" + + contexts = { + certs = "./docker/certs" + } + + tags = ["nitro-enclave-signer/signer-dev:latest"] + output = CI ? [] : ["type=docker"] +} + +target "signer-dev" { + inherits = ["signer-dev-meta-target", "docker-metadata-action"] +} + // Enclave uses a separate Dockerfile for reproducible builds. target "enclave-meta-target" { context = "." diff --git a/docker/Dockerfile b/docker/Dockerfile index 4a044d9..7b81109 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,13 +1,38 @@ +ARG GO_IMAGE=golang:1.25@sha256:00feed335fe561979f2cdcc30a5191231977c5631fe79c40f7d3ab63b4fa222f ARG RUST_IMAGE=rust:1.95-bookworm@sha256:adab7941580c74513aa3347f2d2a1f975498280743d29ec62978ba12e3540d3a -ARG DEBIAN_IMAGE=debian:bookworm-slim@sha256:f9c6a2fd2ddbc23e336b6257a5245e31f996953ef06cd13a59fa0a1df2d5c252 +ARG DEBIAN_IMAGE=debian:bookworm-slim@sha256:96e378d7e6531ac9a15ad505478fcc2e69f371b10f5cdf87857c4b8188404716 ARG NITRO_CLI_VERSION=v1.4.4 # SHA256 of the source tarball for ${NITRO_CLI_VERSION}. Verified at build # time so a retagged upstream cannot silently change the binary we ship. ARG NITRO_CLI_SHA256=5f8182119496a07403b755e1c7e62780acc71ba136639226cdbb508482836e01 +### Build the application binary. +FROM ${GO_IMAGE} AS app-builder + +ARG TARGETARCH + +RUN --mount=type=bind,from=certs,target=/usr/local/share/ca-certificates/ \ + update-ca-certificates && \ + go install github.com/bufbuild/buf/cmd/buf@v1.50.0 && \ + go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.36.6 && \ + go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.5.1 + +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download + +COPY . . +WORKDIR /src/proto +RUN buf generate +WORKDIR /src +RUN export CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} && \ + go build -o /out/app . && \ + go build -o /out/enclave ./cmd/enclave && \ + go build -tags=prod -o /out/app-production . + ### Build grpc-health-probe from master (404552c) to get grpc v1.79.3 (CVE-2026-33186 fix). ### TODO: Revert to `COPY --from=ghcr.io/grpc-ecosystem/grpc-health-probe:` once a tagged release includes grpc >= v1.79.3. -FROM golang:1.25 AS grpc-health-probe-builder +FROM ${GO_IMAGE} AS grpc-health-probe-builder RUN --mount=type=bind,from=certs,target=/usr/local/share/ca-certificates/ \ update-ca-certificates && \ @@ -36,40 +61,40 @@ RUN curl -sSfL -o /tmp/nitro-cli.tar.gz \ install -D -m 0755 build/nitro_cli/release/nitro-cli /out/usr/bin/nitro-cli && \ mkdir -p /out/var/log/nitro_enclaves /out/run/nitro_enclaves -### This is the base image that all other images inherit from -FROM ${DEBIAN_IMAGE} AS base - -ARG TARGETARCH +### Runtime files shared by production and development images. +FROM ${DEBIAN_IMAGE} AS runtime-base WORKDIR /usr/local/circle -COPY --chmod=0750 bin/$TARGETARCH/app ./app -COPY configs ./configs -COPY deployments ./deployments - -## Copy CA certs from the builder so apt trusts the configured CA bundle -## (e.g. when the build runs behind a proxy with a custom root CA). +# Use the builder CA bundle when the build uses a custom root CA. COPY --from=grpc-health-probe-builder /etc/ssl/certs /etc/ssl/certs +COPY --from=grpc-health-probe-builder /grpc-health-probe /usr/local/bin/grpc_health_probe -## Install common dependencies for all targets -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - socat \ - jq && \ - rm -rf /var/lib/apt/lists/* +### Runtime files shared by the production signer images. +FROM runtime-base AS production-runtime -COPY --from=grpc-health-probe-builder /grpc-health-probe /usr/local/bin/grpc_health_probe +COPY --from=app-builder --chmod=0750 /out/app-production ./app +COPY configs/app.production.yaml ./config.yaml + +### The signer image runs a production host with an external Nitro enclave. +FROM production-runtime AS signer + +COPY --chmod=0750 docker/run_proxy.production.sh ./run_proxy.sh +ENTRYPOINT ["/usr/local/circle/run_proxy.sh"] -### This is the image that runs outside the nitro enclave, but does not include the enclave binary (EIF file) -FROM base AS signer +### The signer-dev image runs the host and enclave processes without a Nitro EIF. +FROM runtime-base AS signer-dev +COPY --from=app-builder --chmod=0750 /out/app ./app +COPY --from=app-builder --chmod=0750 /out/enclave ./enclave +COPY configs ./configs +COPY deployments ./deployments COPY --chmod=0750 docker/run_proxy.sh ./run_proxy.sh -COPY --chmod=0750 docker/run_enclave.sh ./run_enclave.sh COPY --chmod=0750 docker/run_enclave.dev.sh ./run_enclave.dev.sh ENTRYPOINT ["/usr/local/circle/run_proxy.sh"] -### This is the image that runs outside the nitro enclave, including the enclave binary (EIF file) -FROM signer AS signer-with-enclave +### The signer-with-enclave image includes the production EIF. +FROM production-runtime AS signer-with-enclave ARG ENCLAVE_EIF ARG ENCLAVE_PCR0="" @@ -82,6 +107,12 @@ LABEL enclave.pcr0="${ENCLAVE_PCR0}" \ COPY --from=nitro-cli-builder /out/ / +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + jq \ + libssl3 && \ + rm -rf /var/lib/apt/lists/* + COPY $ENCLAVE_EIF ./enclave.eif COPY --chmod=0750 docker/run.sh ./run.sh diff --git a/docker/Dockerfile.enclave b/docker/Dockerfile.enclave index 75c1a22..c76caba 100644 --- a/docker/Dockerfile.enclave +++ b/docker/Dockerfile.enclave @@ -16,8 +16,8 @@ FROM ${GOLANG_IMAGE} AS build-base RUN --mount=type=bind,from=certs,target=/usr/local/share/ca-certificates/ \ update-ca-certificates -### Build the app binary inside Docker for a reproducible CGO toolchain. -FROM build-base AS app-builder +### Docker builds the enclave executable with a reproducible CGO toolchain. +FROM build-base AS enclave-builder WORKDIR /src COPY go.mod go.sum ./ @@ -30,7 +30,7 @@ RUN make -C proto proto # Build with BLS portability flag and reproducibility flags. RUN CGO_ENABLED=1 CGO_CFLAGS="-O2 -D__BLST_PORTABLE__" \ - go build -trimpath -buildvcs=false -ldflags=-buildid= -o /app . + go build -tags=prod -trimpath -buildvcs=false -ldflags=-buildid= -o /enclave ./cmd/enclave ### Enclave image - self-contained reproducible build. FROM ${DEBIAN_IMAGE} AS enclave @@ -38,32 +38,36 @@ ARG SOURCE_DATE_EPOCH ENV SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH} ARG SNAPSHOT_DATE -# Truncate debian.sources to avoid non-deterministic whiteout files. -RUN --mount=type=bind,from=build-base,source=/etc/ssl/certs,target=/etc/ssl/certs \ +# Use the builder CA bundle so that APT can use HTTPS. +# APT then installs the pinned CA bundle in the image. +RUN --mount=type=bind,from=build-base,source=/etc/ssl/certs/ca-certificates.crt,target=/tmp/ca-certificates.crt \ + mkdir -p /etc/ssl/certs && \ + cp /tmp/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt && \ : > /etc/apt/sources.list.d/debian.sources && \ echo "deb [check-valid-until=no] https://snapshot.debian.org/archive/debian/${SNAPSHOT_DATE}/ bookworm main" > /etc/apt/sources.list && \ echo "deb [check-valid-until=no] https://snapshot.debian.org/archive/debian-security/${SNAPSHOT_DATE}/ bookworm-security main" >> /etc/apt/sources.list && \ apt-get update && \ apt-get install -y --no-install-recommends \ - socat iproute2 && \ + iproute2 ca-certificates && \ + update-ca-certificates && \ rm -rf /var/lib/apt/lists/* /var/cache/ldconfig/aux-cache && \ : > /var/log/dpkg.log && \ : > /var/log/apt/eipp.log.xz && \ : > /var/log/apt/history.log && \ : > /var/log/apt/term.log && \ - find /etc /tmp /usr /var -newermt "@0" -exec touch -hcd "@0" {} + 2>/dev/null || true + (find /etc /tmp /usr /var -newermt "@0" -exec touch -hcd "@0" {} + 2>/dev/null || true) # Keep WORKDIR creation and file copy in one layer to avoid fresh timestamps. -RUN --mount=type=bind,from=app-builder,source=/app,target=/tmp/app \ - --mount=type=bind,source=configs,target=/tmp/configs \ +RUN --mount=type=bind,from=enclave-builder,source=/enclave,target=/tmp/enclave \ + --mount=type=bind,source=configs/enclave.yaml,target=/tmp/enclave.yaml \ --mount=type=bind,source=docker/run_enclave.sh,target=/tmp/run_enclave.sh \ - mkdir -p /usr/local/circle && \ - cp /tmp/app /usr/local/circle/app && \ - chmod 0750 /usr/local/circle/app && \ - cp -r /tmp/configs /usr/local/circle/configs && \ + mkdir -p /usr/local/circle/configs && \ + cp /tmp/enclave /usr/local/circle/enclave && \ + chmod 0750 /usr/local/circle/enclave && \ + cp /tmp/enclave.yaml /usr/local/circle/configs/enclave.yaml && \ cp /tmp/run_enclave.sh /usr/local/circle/run_enclave.sh && \ chmod 0750 /usr/local/circle/run_enclave.sh && \ - find /usr/local -newermt "@0" -exec touch -hcd "@0" {} + 2>/dev/null || true && \ + (find /usr/local -newermt "@0" -exec touch -hcd "@0" {} + 2>/dev/null || true) && \ touch -hcd "@0" /tmp /etc WORKDIR /usr/local/circle diff --git a/docker/run.sh b/docker/run.sh index a8c16ba..bb3c396 100755 --- a/docker/run.sh +++ b/docker/run.sh @@ -3,13 +3,14 @@ readonly EIF_PATH="/usr/local/circle/enclave.eif" readonly ENCLAVE_CPU_COUNT=${ENCLAVE_CPU_COUNT:-2} readonly ENCLAVE_MEMORY_SIZE=${ENCLAVE_MEMORY_SIZE:-4096} -readonly ENCLAVE_CID=${ENCLAVE_CID:-16} +readonly ENCLAVE_CID=16 -readonly PROXY_PORT=10340 -readonly ENCLAVE_PORT=10350 +readonly VSOCKPROXY_READY_FILE=/tmp/vsockproxy.ready +readonly VSOCKPROXY_READY_TIMEOUT_DECISECONDS=30 # 30 * 0.1s = 3s -# Global variable to store app PID +# Global variables to store process PIDs APP_PID="" +VSOCKPROXY_PID="" SHUTDOWN_IN_PROGRESS=false log() { @@ -26,12 +27,40 @@ log() { }' >&2 } +# Wait for a child to exit, then SIGKILL it if it remains alive past the +# ceiling. Polling avoids a watchdog process whose sleep child can outlive it. +wait_bounded() { + local pid="$1" + local timeout_s="${2:-35}" + local deadline=$((SECONDS + timeout_s)) + + while kill -0 "$pid" 2>/dev/null; do + # On Linux, an exited child remains visible as a zombie until wait + # reaps it; do not mistake that state for a running process. + if [ -r "/proc/$pid/stat" ] && [ "$(awk '{print $3}' "/proc/$pid/stat" 2>/dev/null)" = "Z" ]; then + break + fi + if [ "$SECONDS" -ge "$deadline" ]; then + kill -KILL "$pid" 2>/dev/null || true + break + fi + sleep 0.1 + done + + wait "$pid" 2>/dev/null || true +} + -# Shutdown the enclave and all background processes +# Shutdown the enclave and all background processes. +# Order: app first (stop emitting RPCs) β†’ vsockproxy (drain in-flight) β†’ +# enclave terminate (acceptors close last). shutdown() { + local status="${1:-0}" + + trap - SIGTERM SIGINT EXIT # Prevent multiple invocations if [ "$SHUTDOWN_IN_PROGRESS" = true ]; then - return 0 + exit "$status" fi SHUTDOWN_IN_PROGRESS=true @@ -41,7 +70,16 @@ shutdown() { if [ -n "$APP_PID" ]; then log "Stopping app (PID: $APP_PID)..." kill -TERM "$APP_PID" 2>/dev/null || true - wait "$APP_PID" 2>/dev/null || true + wait_bounded "$APP_PID" + fi + + # Stop vsockproxy after app so any in-flight KMS calls from inside + # the enclave can drain through the proxy before its listeners + # close. + if [ -n "$VSOCKPROXY_PID" ]; then + log "Stopping vsockproxy (PID: $VSOCKPROXY_PID)..." + kill -TERM "$VSOCKPROXY_PID" 2>/dev/null || true + wait_bounded "$VSOCKPROXY_PID" fi # Stop enclave if it's running @@ -50,20 +88,69 @@ shutdown() { nitro-cli terminate-enclave --enclave-id "$enclave_id" 2>/dev/null || true fi - # Kill all remaining background processes - jobs -p | xargs -r kill 2>/dev/null || true - - # Wait for cleanup to complete - wait 2>/dev/null || true - log "Stopped" - exit 0 + exit "$status" +} + +# Start the vsockproxy bridge as a separate process so its vsock listener +# is bound before nitro-cli starts the enclave (the first Initialize-time +# KMS call from inside the enclave races the listener bind otherwise). +start_vsockproxy() { + log "Starting vsockproxy..." + # Clear any stale sentinel from a previous run so the poll below cannot + # observe a false-ready before the freshly started proxy binds. + rm -f "$VSOCKPROXY_READY_FILE" + # The proxy owns its listeners, so it is the only authoritative source of + # "am I listening?": vsock listeners are not enumerable from the host via + # /proc or the tools shipped in this image. run-vsockproxy creates + # VSOCKPROXY_READY_FILE once byteproxy has bound every listener (in New, + # before serving), so we gate on that file instead of probing the kernel. + env VSOCKPROXY_READY_FILE="$VSOCKPROXY_READY_FILE" \ + /usr/local/circle/app \ + --config /usr/local/circle/config.yaml \ + run-vsockproxy & + VSOCKPROXY_PID=$! + log "Started vsockproxy with PID: $VSOCKPROXY_PID" + + # Wait until the proxy reports its listeners are bound. Track readiness + # with an explicit flag so a success on the final iteration is not + # misread as a timeout. + local ready=false + local attempts=0 + while [ "$attempts" -lt "$VSOCKPROXY_READY_TIMEOUT_DECISECONDS" ]; do + if [ -e "$VSOCKPROXY_READY_FILE" ]; then + # The sentinel only proves the proxy bound its listeners at some + # point, not that it is still alive: it could have written the + # file and then crashed (e.g. during serving). Re-check liveness + # before trusting the sentinel so we never start the enclave + # against a dead proxy. + if ! kill -0 "$VSOCKPROXY_PID" 2>/dev/null; then + log "vsockproxy exited after signaling ready (PID: $VSOCKPROXY_PID)" "ERROR" + exit 1 + fi + ready=true + break + fi + # Surface a crash (config error, port conflict, ...) immediately + # instead of waiting out the full readiness timeout. + if ! kill -0 "$VSOCKPROXY_PID" 2>/dev/null; then + log "vsockproxy exited unexpectedly (PID: $VSOCKPROXY_PID)" "ERROR" + exit 1 + fi + sleep 0.1 + attempts=$((attempts + 1)) + done + if [ "$ready" != true ]; then + log "vsockproxy did not report ready within 3s" "ERROR" + exit 1 + fi + log "vsockproxy is ready (listeners bound)" } # Start the enclave start_enclave() { log "Starting Enclave..." - nitro-cli run-enclave ${EXTRA_NITRO_CLI_FLAGS:-} \ + nitro-cli run-enclave \ --cpu-count "$ENCLAVE_CPU_COUNT" \ --memory "$ENCLAVE_MEMORY_SIZE" \ --enclave-cid "$ENCLAVE_CID" \ @@ -80,38 +167,19 @@ start_app() { touch /usr/local/circle/logs/app.log log "Starting app..." - APP_PUBLIC_SERVER_PORT=${PROXY_PORT} /usr/local/circle/app --config /usr/local/circle/configs/app.yaml run & + /usr/local/circle/app --config /usr/local/circle/config.yaml run & APP_PID=$! log "Started app with PID: $APP_PID" } -setup_vsock_bridges() { - log "Setting up VSOCK bridges..." - - # set nitro enclave environment variable - socat VSOCK-LISTEN:8000,reuseaddr,fork SYSTEM:'echo \"$APP_ENV|$DD_SERVICE|$DD_ENV|$DD_ENTITY_ID\"' & - - - # To mitigate the risk of sensitive information leakage, exporting tracing - # data to Datadog is only enabled in non-production environments. - if [ "$APP_ENV" != "prod" ]; then - socat VSOCK-LISTEN:4317,reuseaddr,fork TCP:$OTEL_HOST:4317 & # OTLP traces - socat VSOCK-LISTEN:8126,reuseaddr,fork TCP:$DD_AGENT_HOST:8126 & # Datadog APM traces - socat VSOCK-LISTEN:8125,reuseaddr,fork TCP:$APP_CONFIG_OPTION_STATSD_HOST:$APP_CONFIG_OPTION_STATSD_PORT & # DogStatsD metrics - socat -u VSOCK-LISTEN:8001,reuseaddr,fork EXEC:"cat",stderr & - fi -} - -# Setup signal handlers -trap shutdown SIGTERM SIGINT EXIT - -if [ "${ENABLE_ENCLAVE:-false}" = "true" ]; then - setup_vsock_bridges - start_enclave - sleep 3 -fi - +# Setup signal handlers. Preserve unexpected app failures through the EXIT trap. +trap 'shutdown 0' SIGTERM +trap 'shutdown 130' SIGINT +trap 'shutdown $?' EXIT +start_vsockproxy +start_enclave +sleep 3 start_app # Wait for the app process specifically diff --git a/docker/run_enclave.dev.sh b/docker/run_enclave.dev.sh index a499f00..dc891a7 100755 --- a/docker/run_enclave.dev.sh +++ b/docker/run_enclave.dev.sh @@ -2,15 +2,7 @@ set -eux -PROJECT_DIR="$(dirname "$0")/.." -BIN_DIR="${PROJECT_DIR}/bin" -CMD_DIR="${PROJECT_DIR}/cmd" - -export PROJECT_DIR -export BIN_DIR -export CMD_DIR - mkdir -p /usr/local/circle/logs touch /usr/local/circle/logs/app.log -exec env APP_PUBLIC_SERVER_PORT=10350 /usr/local/circle/app --enclave-config /usr/local/circle/configs/enclave.yaml run-enclave +exec env APP_PUBLIC_SERVER_PORT=10350 /usr/local/circle/enclave --enclave-config /usr/local/circle/configs/enclave.yaml diff --git a/docker/run_enclave.sh b/docker/run_enclave.sh index f4d924b..9b36fe8 100755 --- a/docker/run_enclave.sh +++ b/docker/run_enclave.sh @@ -1,63 +1,13 @@ #!/usr/bin/env bash -set -eux +set -eu -PROJECT_DIR="$(dirname "$0")/.." -BIN_DIR="${PROJECT_DIR}/bin" -CMD_DIR="${PROJECT_DIR}/cmd" +# Discard all output. Network setup failures are silent by design. +exec >/dev/null 2>&1 -export PROJECT_DIR -export BIN_DIR -export CMD_DIR - -# Assign IP addresses to local loopback ip addr add 127.0.0.1/32 dev lo ip link set dev lo up -ENV_DATA=$(socat VSOCK-CONNECT:3:8000 -) -IFS='|' read -r APP_ENV DD_SERVICE DD_ENV DD_ENTITY_ID _ <<< "$ENV_DATA" - -# Default APP_ENV to 'stg' if not provided by the host. This is a safe default -# as it enables non-production features like tracing. -export APP_ENV="${APP_ENV:-stg}" - -# Export Datadog env vars if provided by host (non-empty check) -[ -n "$DD_ENV" ] && export DD_ENV -[ -n "$DD_ENTITY_ID" ] && export DD_ENTITY_ID -# Add "-enclave" suffix to distinguish enclave metrics from proxy -[ -n "$DD_SERVICE" ] && export DD_SERVICE="${DD_SERVICE}-enclave" - -# Create a FIFO for logging -LOG_FIFO=/tmp/log.fifo -mkfifo "$LOG_FIFO" - -# outbound -if [ "$APP_ENV" != "prod" ]; then - export OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:4317 - socat TCP-LISTEN:4317,fork,reuseaddr VSOCK-CONNECT:3:4317 & # OTLP traces - socat TCP-LISTEN:8126,fork,reuseaddr VSOCK-CONNECT:3:8126 & # Datadog APM traces - socat TCP-LISTEN:8125,fork,reuseaddr VSOCK-CONNECT:3:8125 & # DogStatsD metrics - # Redirect FIFO to vsock and stdout - ( cat <"$LOG_FIFO" | tee >(socat - VSOCK-CONNECT:3:8001) ) & - LOG_PID=$! -else - # Make sure we drain the fifo if not sending to vsock - cat >/dev/null < "$LOG_FIFO" & - LOG_PID=$! -fi - -# redirect stdout and stderr of enclave to FIFO -APP_PUBLIC_SERVER_PORT=10350 /usr/local/circle/app \ - --enclave-config /usr/local/circle/configs/enclave.yaml run-enclave \ - 2>&1 > "$LOG_FIFO" & -APP_PID=$! - -cleanup() { - kill "$APP_PID" || true - kill "$LOG_PID" || true - rm -f "$LOG_FIFO" -} - -trap cleanup TERM INT - -wait "$APP_PID" +exec env APP_PUBLIC_SERVER_PORT=10350 \ + /usr/local/circle/enclave \ + --enclave-config /usr/local/circle/configs/enclave.yaml diff --git a/docker/run_proxy.production.sh b/docker/run_proxy.production.sh new file mode 100755 index 0000000..4373af1 --- /dev/null +++ b/docker/run_proxy.production.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +set -eu + +exec /usr/local/circle/app \ + --config /usr/local/circle/config.yaml \ + run diff --git a/docs/architecture.md b/docs/architecture.md index 333dfb5..fbbe1bc 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,127 +1,213 @@ # Architecture -This document describes the dual-process architecture, runtime flows, security model, deployment topology, and API surface of Arc Remote Signer. +This document describes the three runtime processes across two trust domains, runtime flows, security model, deployment topology, and API surface of Arc Remote Signer. For day-to-day build/run commands, see [development.md](development.md). For test conventions, see [testing.md](testing.md). ## Overview -Arc Remote Signer uses a **dual-process architecture** with a hard security boundary between the Proxy (outside the enclave) and the Enclave (inside the Nitro Enclave). This separation ensures that private key material never exists in the host process β€” all cryptographic operations happen exclusively inside the hardware-isolated enclave. +Arc Remote Signer uses **three runtime processes across two trust domains**. The host application (`app run`) and standalone KMS traffic proxy (`app run-vsockproxy`) run on the EC2 host; the enclave application runs inside the Nitro Enclave. The trust boundary ensures that private key material never exists in either host process β€” all cryptographic operations happen exclusively inside the hardware-isolated enclave. -The two processes communicate over **gRPC on vsock** (virtual socket), which is the only communication channel available in the Nitro Enclave environment. This means all inter-process contracts are typed and versioned via Protocol Buffers, and there is no shared memory or filesystem between the host and the enclave. +The host and enclave applications communicate over gRPC: TCP on localhost in dev/CI and VSOCK in Nitro production. In Nitro, VSOCK is the enclave's communication channel to the host. All inter-process contracts are typed and versioned via Protocol Buffers, and there is no shared memory or filesystem across the trust boundary. ``` Arc Chain Validator β”‚ gRPC :10340 β–Ό β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ Proxy (Host) β”‚ -β”‚ β”‚ +β”‚ EC2 Host β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β” β”‚ -β”‚ β”‚ Public β”‚ β”‚ Service β”‚ β”‚ KMS │─┼──▢ AWS KMS -β”‚ β”‚(Handlers)│─▢│ (Signer) │─▢│ Sec β”‚ β”‚ -β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ Mgr │─┼──▢ AWS Secrets Manager -β”‚ β””β”€β”€β”€β”¬β”€β”€β”€β”˜ β”‚ -β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”˜ - β”‚ gRPC over vsock :10350 - β–Ό +β”‚ β”‚ Public │─▢│ Service │─▢│ Sec │─┼──▢ AWS Secrets Manager +β”‚ β”‚(Handlers)β”‚ β”‚ (Signer) β”‚ β”‚ Mgr β”‚ β”‚ +β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ +β”‚ Host application (`app run`) β”‚ +β”‚ β”‚ +β”‚ `app run-vsockproxy` (separate process)β”œβ”€β”€β–Ά AWS KMS +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ gRPC :10350 + β”‚ TCP (dev/CI), VSOCK (Nitro) + β–Ό β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ Nitro Enclave β”‚ -β”‚ β”‚ +β”‚ Nitro Enclave β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β” β”‚ -β”‚ β”‚ Public β”‚ β”‚ Key / β”‚ β”‚ Key β”‚ β”‚ -β”‚ β”‚(Handlers)│─▢│ Attn │─▢│ Store β”‚ β”‚ -β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ NSM β”‚ β”‚ -β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ +β”‚ β”‚ Public │─▢│ Key / │─▢│ Key β”‚ β”‚ +β”‚ β”‚(Handlers)β”‚ β”‚ Attn β”‚ β”‚ Store β”‚ β”‚ +β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ +β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ +β”‚ β”‚ KMS client + NSM + awsproxy β”‚ β”‚ +β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ ``` -- **Proxy** (`internal/app/`) β€” handles incoming validator gRPC requests, orchestrates AWS integrations (KMS, Secrets Manager), caches attestation documents, and proxies signing operations to the enclave -- **Enclave** (`internal/enclave/`) β€” performs all cryptographic operations (key generation, decryption, signing), manages key material in memory, and generates attestation documents via the Nitro Security Module (NSM) +The enclave owns the KMS client. In Nitro mode, it attaches its NSM attestation +as KMS `RecipientInfo`. Requests travel through the enclave-side `awsproxy` and +the host-side `vsockproxy`. For standard AWS KMS, the host proxy dials the +regional endpoint from the route header. It limits routes to regions from the +configured ARNs. For LocalStack, the host proxy dials the fixed LocalStack +service endpoint. Nitro mode does not support LocalStack. + +- **Host application** (`app run`, `internal/app/`) β€” handles incoming validator gRPC requests, persists the wrapped key in Secrets Manager, and passes temporary AWS credentials, KMS key ARNs, and the LocalStack selector during enclave initialization. It does not construct KMS requests or handle plaintext data keys. +- **KMS traffic proxy** (`app run-vsockproxy`, `internal/vsockproxy/`) β€” runs as a separate host process. It forwards opaque KMS traffic to LocalStack or to AWS regions from the configured KMS key ARNs. +- **Enclave application** (`internal/enclave/`) β€” owns the AWS KMS client and performs all cryptographic operations (data-key mint/decrypt, key generation, decryption, signing), manages key material in memory, and requests attestation documents from the Nitro Security Module (NSM). The enclave binary is packaged as an **Enclave Image File (EIF)**, which is cryptographically measured at build time. The PCR measurements in the attestation document reflect this image, allowing KMS key policies to enforce that decryption can only happen inside a known, unmodified enclave build. ## Runtime Flows -The following sections describe how the two processes interact at runtime, from initial startup through key generation and steady-state signing. +The following sections describe how the runtime processes interact, from initial startup through key generation and steady-state signing. ### Startup Sequence -On startup, the Proxy fetches and caches the attestation document from the Enclave before handling any requests. This cached attestation is attached to all subsequent KMS API calls. The attestation document contains the enclave's public key and PCR measurements β€” KMS uses this to verify enclave identity and encrypts its response with the enclave's public key, so the plaintext data key is never exposed outside the enclave. +On startup, the host reads the configured secret from Secrets Manager. The host +asks the enclave to recover the stored key or generate a new key. The host +retrieves temporary AWS credentials for each `Initialize` attempt. The host +refreshes the credentials after an authentication failure. The host sends the +credentials, KMS key ARNs, LocalStack selector, and key source in the request. + +Each enclave attempt requests a fresh KMS Recipient attestation in Nitro mode. +The attempt creates an ephemeral KMS provider and completes the key transaction. +The transaction generates or recovers the signing key and creates a public key +attestation. The enclave publishes the Ready state only after the complete +transaction succeeds. A failed attempt does not install a key. ``` -Proxy Enclave - β”‚ β”‚ - │── GetAttestation() ───────────────────────────▢│ - │◀─ { public_key, β”‚ - β”‚ pcr_measurements, β”‚ - β”‚ nsm_signature } ───────────────────────────│ - β”‚ β”‚ - β”‚ (cache attestation_doc for KMS calls) β”‚ - β”‚ β”‚ +Host application Enclave NSM / AWS KMS + β”‚ β”‚ β”‚ + │── read stored envelope β”‚ β”‚ + │── retrieve credentials β”‚ β”‚ + │── Initialize(key source, ─────▢│ β”‚ + β”‚ credentials, KMS settings) │── fresh recipient attest ─▢│ + β”‚ │── generate or recover key ─▢│ + β”‚ │── attest public key ───────▢│ + β”‚ │── publish Ready β”‚ + │◀─ public key, attestation, ─────│ β”‚ + β”‚ optional generated envelope β”‚ β”‚ + │── persist generated envelope β”‚ β”‚ [ready to serve requests] ``` ### Key Generation Flow -Key generation is triggered on first startup when no key is found in Secrets Manager for the configured key ID: +Key generation is triggered when the configured, pre-created Secrets Manager secret has no stored key value: -1. Proxy calls KMS `GenerateDataKey` with the attestation cached during startup β€” KMS verifies the PCR measurements and encrypts the response with the enclave's public key -2. Proxy forwards the enclave-encrypted data key to the Enclave via `GenerateKey` -3. Enclave decrypts the data key internally, generates an ed25519 keypair, and encrypts the private key with the data key (AES-GCM) -4. Proxy stores the encrypted private key and KMS-encrypted data key in Secrets Manager +1. The host calls Enclave `Initialize` with `generate_new` set to the configured algorithm. +2. The enclave creates an ephemeral KMS provider for the attempt. In Nitro mode, the provider uses a fresh KMS Recipient attestation. +3. The enclave calls KMS `GenerateDataKey`. KMS returns `CiphertextForRecipient` in Nitro mode, and the enclave decrypts it with its in-memory RSA private key. KMS returns the plaintext data key in dev/CI. +4. The enclave generates the signing keypair and encrypts the private key with AES-GCM. It creates `SecretEnvelope { algorithm, kms_encrypted_data_key, encrypted_private_key, nonce }`. +5. In Nitro mode, the enclave creates an attestation document for the public key. +6. The enclave installs the signing key and publishes the Ready state. +7. The enclave returns the public key, attestation document, and generated envelope. The host maps the envelope to a stored `Header` and writes it to Secrets Manager. ``` -Proxy AWS KMS Enclave Secrets Manager - β”‚ β”‚ β”‚ β”‚ - │── GetAttestation() ──────────────────────────────────▢│ β”‚ - │◀─ attestation_doc ────────────────────────────────────│ β”‚ - β”‚ β”‚ β”‚ β”‚ - │── GenerateDataKey(attestation)──▢│ β”‚ β”‚ - │◀─ { kms_encrypted_data_key, β”‚ β”‚ β”‚ - β”‚ enclave_encrypted_data_key } β”‚ β”‚ β”‚ - β”‚ β”‚ β”‚ β”‚ - │── GenerateKey(enclave_encrypted_data_key) ───────────▢│ β”‚ - β”‚ (Enclave decrypts data key, generates ed25519, β”‚ β”‚ - β”‚ encrypts private key with data key via AES-GCM, β”‚ β”‚ - β”‚ caches plaintext key in enclave memory) β”‚ β”‚ - │◀─ { public_key, encrypted_private_key, nonce } ───────│ β”‚ - β”‚ β”‚ β”‚ β”‚ - │── PutSecret(encrypted_private_key, β”‚ β”‚ - β”‚ kms_encrypted_data_key, nonce) ──────────────────────────────────▢│ - β”‚ β”‚ β”‚ β”‚ +Host application Enclave NSM / AWS KMS Secrets Manager + β”‚ β”‚ β”‚ β”‚ + │── Initialize ───▢│ β”‚ β”‚ + β”‚ (generate_new) │── GenerateDataKey ───▢│ β”‚ + β”‚ │◀─ ciphertext blobs ───│ β”‚ + β”‚ β”‚ enclave decrypts data key and wraps signing keyβ”‚ + β”‚ │── enclave requests public-key attestation ─▢│ β”‚ + β”‚ │◀─ NSM returns attestation document ─│ β”‚ + β”‚ β”‚ enclave publishes Ready β”‚ + │◀─ key + document β”‚ β”‚ β”‚ + β”‚ + envelope β”‚ β”‚ β”‚ + │── PutSecret(Header from envelope) ────────────────────────────────▢│ + β”‚ β”‚ β”‚ β”‚ ``` -The Proxy caches `encrypted_private_key`, `kms_encrypted_data_key`, and `nonce` in memory. The plaintext private key exists only inside the enclave's memory-isolated environment. +The host persists a generated `SecretEnvelope`. It keeps the public key and its +attestation document in memory. Only enclave memory contains the plaintext +private key and plaintext data key. ### Signing Flow -Because the Proxy caches the encrypted key material after generation (or after loading from Secrets Manager), signing does not require a round-trip to Secrets Manager or KMS on the hot path: +After successful initialization, signing uses only the installed enclave key: + +1. The host sends only the message to Enclave `SignMessage`. +2. The enclave signs the message with the installed key. +3. The enclave returns the signature. + +``` +Validator Host application Enclave + β”‚ β”‚ β”‚ + │── Sign(msg) ────────▢│ β”‚ + β”‚ │── SignMessage ──▢│ + β”‚ β”‚ (message only) │── sign with installed key + β”‚ │◀─ signature ─────│ + │◀─ signature ─────────│ β”‚ +``` -1. Proxy retrieves cached `encrypted_private_key`, `enclave_encrypted_data_key`, and `nonce` -2. Proxy calls Enclave `SignMessage` with the message and encrypted key material -3. Enclave looks up the plaintext key from its in-memory cache (keyed by `hash(encrypted_private_key, dataKey, nonce)`); on cache miss, decrypts via the enclave-encrypted data key -4. Enclave performs the ed25519 signature and returns the result +Steady-state signing does not send the envelope and does not call KMS or +Secrets Manager. + +### Public Key Attestation Flow + +During signer initialization, Enclave `Initialize` returns the installed public +key and its attestation document. In Nitro mode, the enclave asks NSM to create +the document before it publishes the Ready state. The signed `user_data` field +contains the exact public key bytes. The host keeps the public key and document +in memory. It also reads the earliest certificate expiry from the document. +The background refresh process uses Enclave `GetPublicKey` to create a later +attestation document for the installed key. + +Each public `SignerService.PublicKey` request returns the pair from host memory. +The request does not call the enclave or NSM. The request does not contain a +nonce. The cache does not use a fixed TTL. A background task gets a new document +before the first certificate expires. It only accepts the same public key and a +later certificate expiry. A refresh error keeps the last valid document and +starts a bounded retry. `PublicKey` returns `Unavailable` after the document +expires. + +The signed timestamp records when NSM created the document. It does not prove +that the enclave is currently live. + +An external verifier runs outside the host and proxy trust boundary. It rejects +a missing or empty document. It validates the NSM signature and certificate +chain against a pinned AWS Nitro root CA. It checks each certificate validity +period at verification time. The verifier also validates the PCR values. The +verifier checks that `user_data` matches `public_key`. In dev/CI, the enclave +returns an empty attestation document. ``` -Validator Proxy Enclave - β”‚ β”‚ β”‚ - │── Sign(msg) ──▢│ β”‚ - β”‚ │── SignMessage(msg, β”‚ - β”‚ β”‚ encrypted_private_key, β”‚ - β”‚ β”‚ enclave_encrypted_data_key, β”‚ - β”‚ β”‚ nonce) ───────────────────────▢│ - β”‚ β”‚ (Enclave looks up plaintext key β”‚ - β”‚ β”‚ from in-memory cache by hash; β”‚ - β”‚ β”‚ on cache miss, decrypts via β”‚ - β”‚ β”‚ enclave_encrypted_data_key) β”‚ - β”‚ β”‚ ◀─ signature ──────────────────────│ - │◀─ signature ───│ β”‚ - β”‚ β”‚ β”‚ +Signer startup Host application Enclave NSM + β”‚ β”‚ β”‚ β”‚ + β”‚ │── Initialize ───▢│ β”‚ + β”‚ β”‚ │── Attest(public key) β–Άβ”‚ + β”‚ β”‚ │◀─ signed document ────│ + β”‚ │◀─ key + document β”‚ β”‚ + β”‚ β”‚ cache pair β”‚ β”‚ + β”‚ β”‚ β”‚ β”‚ +Validator β”‚ β”‚ β”‚ + │── PublicKey() ──────▢│ β”‚ β”‚ + │◀─ cached pair ───────│ β”‚ β”‚ + β”‚ β”‚ β”‚ β”‚ + β”‚ │── GetPublicKey ─▢│ β”‚ + β”‚ β”‚ │── Attest(public key) β–Άβ”‚ + β”‚ β”‚ │◀─ signed document ────│ + β”‚ │◀─ key + document β”‚ β”‚ + β”‚ β”‚ refresh cache β”‚ β”‚ ``` -The enclave's in-memory key cache is keyed by `hash(encrypted_private_key, dataKey, nonce)`. A cache miss triggers full decryption using the enclave-encrypted data key β€” the KMS-encrypted data key stored in Secrets Manager is never used on the signing path. +### Multi-Region KMS Routing and Failover + +`APP_PROVIDER_AWSKMS_ARNS` accepts one or more KMS key ARNs (typically the regional replicas of a multi-region key). The enclave builds one KMS client per ARN, each pinned to that ARN's region, and fails over across them in order: if a KMS call fails, the failing client is moved to the back and the next ARN's region is tried, so a single-region KMS disruption degrades to the next configured region rather than failing the request. + +Because the enclave has no direct network access, each KMS call uses the proxy chain: + +1. The enclave's KMS client prepends a small route header (AWS service + region) when it connects to the enclave-side `awsproxy`, which relays it alongside the request bytes over TCP in dev/CI or VSOCK in Nitro production +2. The host-side `vsockproxy` reads that header. It dials LocalStack when `APP_PROVIDER_AWSKMS_LOCALSTACK_ENABLED=true`. +3. Otherwise, `vsockproxy` dials `kms..amazonaws.com:443`. The configured ARN regions form an **allowlist**. The proxy rejects a route for another region. + +The allowlist is a security control, not just routing: the host relays bytes it cannot decrypt on behalf of the enclave, and bounding the dial target to the operator-configured ARN regions prevents the relayed stream from being used to reach an arbitrary host (SSRF). The host `vsockproxy` therefore consumes the same `APP_PROVIDER_AWSKMS_ARNS` value as the enclave to build this allowlist. + +The enclave independently requires the credentials region and every KMS ARN +region to appear in its compiled AWS commercial KMS region allowlist. Unknown +values and non-commercial partitions are rejected before any endpoint, TLS +server name, proxy route, or SDK client is constructed. When AWS adds a region +to the [KMS endpoints reference](https://docs.aws.amazon.com/general/latest/gr/kms.html), +the enclave validator and its positive region test must be updated together. +The resulting EIF has new PCR measurements, so deploying it also requires the +normal image release and KMS key policy rollout. ## Security Model @@ -134,58 +220,49 @@ Arc Remote Signer protects validator private keys through hardware isolation, en | Privileged access | Keys isolated in hardware-backed enclave; root/hypervisor cannot access | | Memory inspection | CPU-enforced memory isolation; no shared memory with host | | Malware and code tampering | Validated code identity verified via attestation | -| Key exfiltration | Keys never leave enclave in plaintext; only signatures returned | +| Key exfiltration | Private keys never leave the enclave in plaintext; the validator-facing API returns only public keys and signatures | | Key at rest | Private key stored in Secrets Manager encrypted with AES-GCM data key; data key itself encrypted by AWS KMS β€” neither is usable without enclave decryption | ### Envelope Encryption -Arc Remote Signer uses envelope encryption to protect validator private keys at rest. Rather than encrypting the key directly with KMS (which has throughput limits), a short-lived data key performs the actual encryption while KMS protects only the data key. The enclave's public key adds a final layer ensuring only the enclave itself can unwrap the key material: +Arc Remote Signer uses envelope encryption to protect validator private keys at rest. A per-validator data key encrypts the private key with AES-GCM, and AWS KMS protects that data key: 1. Validator private key encrypted with a data key -2. Data key protected by AWS KMS key material -3. KMS responses encrypted with enclave public key from attestation -4. Only enclave private key can decrypt returned key material +2. KMS-encrypted data key stored as a `CiphertextBlob` ``` Validator Private Key └── encrypted by Data Key (AES-GCM) - └── Data Key encrypted by AWS KMS Key - └── KMS response encrypted by Enclave Public Key (from attestation) - └── only Enclave Private Key can unwrap ← lives exclusively in enclave memory + └── Data Key encrypted by AWS KMS Key (`CiphertextBlob`) ``` +Attestation protects the separate KMS response path. The enclave process generates an in-memory RSA key pair, and NSM signs an attestation document containing the RSA public key. With that attestation in `RecipientInfo`, KMS returns the data key as `CiphertextForRecipient`, encrypted to the attested public key. The proxy chain can relay that response but cannot decrypt it; the enclave process decrypts it with the corresponding in-memory RSA private key. + ### Attestation and KMS Integration -Attestation is the mechanism that binds KMS access to a specific, verified enclave build. KMS key policies enforce PCR conditions, so decryption only succeeds if the request originates from an unmodified enclave with matching measurements β€” even a privileged host process cannot satisfy these conditions. +Attestation binds recovery of KMS-protected data keys to an approved enclave release on an authorized parent EC2 instance. KMS key policies enforce PCR0, PCR1, PCR2, and PCR4 conditions. KMS encrypts successful responses to the attested enclave key. Host credentials alone cannot recover plaintext data keys on another host. -1. Enclave generates attestation document (public key + PCR measurements + signed metadata) -2. Proxy sends attestation with KMS requests -3. KMS validates attestation and encrypts response to enclave key -4. Enclave decrypts and uses key material internally +1. The enclave process generates an in-memory RSA key pair and asks NSM to sign an attestation document containing the RSA public key, PCR measurements, and metadata +2. The **enclave** owns the KMS client and attaches its attestation as `RecipientInfo` on KMS `GenerateDataKey` and `Decrypt` calls; `awsproxy` inside the enclave and `vsockproxy` on the host relay the network traffic without constructing the requests +3. KMS validates the attestation against the key policy and encrypts the response to the enclave's key (`CiphertextForRecipient`) +4. The enclave process decrypts the response with the corresponding in-memory RSA private key β€” the plaintext never leaves the enclave, even though the proxy chain carries the bytes See the AWS documentation for further reference: - [Using cryptographic attestation with AWS KMS](https://docs.aws.amazon.com/enclaves/latest/user/kms.html) β€” attestation flow and KMS key policy configuration - [Cryptographic attestation support in AWS KMS](https://docs.aws.amazon.com/kms/latest/developerguide/cryptographic-attestation.html) β€” KMS-side attestation mechanism details ``` -Enclave Proxy AWS KMS - β”‚ β”‚ β”‚ - β”‚ (on startup) β”‚ β”‚ - │◀─ GetAttestation() ────│ β”‚ - │── attestation_doc ────▢│ β”‚ - β”‚ (public_key + β”‚ β”‚ - β”‚ PCR measurements) β”‚ β”‚ - β”‚ β”‚ β”‚ - β”‚ │── KMS request + β”‚ - β”‚ β”‚ attestation_doc ──────────▢│ - β”‚ β”‚ (KMS checks PCR β”‚ - β”‚ β”‚ against policy)β”‚ - β”‚ │◀─ response encrypted β”‚ - β”‚ β”‚ with enclave public_key ───│ - β”‚ β”‚ β”‚ - │◀─ encrypted response ──│ β”‚ - β”‚ (only enclave private β”‚ β”‚ - β”‚ key can decrypt) β”‚ β”‚ +Enclave awsproxy / host vsockproxy AWS KMS + β”‚ β”‚ β”‚ + β”‚ (NSM attestation held β”‚ β”‚ + β”‚ in-enclave) β”‚ β”‚ + │── KMS request + attestation ────▢│── relay ────────────────────▢│ + β”‚ β”‚ (KMS checks PCR β”‚ + β”‚ β”‚ against policy)β”‚ + │◀─ response encrypted ────────────│◀─ CiphertextForRecipient ────│ + β”‚ to enclave key (relayed; β”‚ β”‚ + β”‚ host cannot read it) β”‚ β”‚ + β”‚ (RSA-decrypted in-enclave) β”‚ β”‚ ``` ## Deployment @@ -194,46 +271,108 @@ Arc Remote Signer is deployed as a 1-to-1 sidecar alongside each Arc Chain valid ### Docker Images -The project uses `docker buildx bake` (configured in `docker-bake.hcl`) to build three Docker image targets across two Dockerfiles. The `enclave` target lives in `docker/Dockerfile.enclave` (built reproducibly), and `signer` / `signer-with-enclave` live in `docker/Dockerfile`: +The `docker-bake.hcl` file specifies the Docker image targets. The project uses `docker buildx bake` to build these targets. The project has three production targets and one development target. The `enclave` target uses `docker/Dockerfile.enclave`. The build from this Dockerfile is reproducible. The signer targets use `docker/Dockerfile`: + +1. **enclave** (`docker/Dockerfile.enclave`) runs inside an AWS Nitro Enclave. + - The build uses the `prod` tag. + - The enclave listens on port `10350` over VSOCK. + - The entry point is `run_enclave.sh`. +2. **signer** (`docker/Dockerfile`) runs the production host application outside the enclave. + - The build uses the `prod` tag. + - The host handles external gRPC requests. + - The host communicates with the enclave over VSOCK. + - The image does not contain development configuration or executables. + - The host listens on port `10340` for the gRPC API. + - The entry point is `run_proxy.sh`. +3. **signer-with-enclave** (`docker/Dockerfile`) is the complete production image. + - The image includes the proxy and enclave applications. + - The image contains the enclave image file (EIF). + - The launcher always starts the VSOCK proxy, the EIF, and the host application. + - The entry point is `run.sh`. +4. **signer-dev** (`docker/Dockerfile`) is the local and CI development image. + - The image uses the common runtime base image. + - The image does not inherit from `signer`. + - The image uses the development host and enclave builds. + - The image contains the standalone enclave executable and the development startup script. + - The launcher starts the enclave process directly over TCP. + - The image does not use a Nitro EIF. + - The image is not part of the production image matrix. + +#### Secure EIF boundary + +Production uses one `enclave` target and one EIF. The build configuration has no +debug EIF target. Staging and production use the same `signer-with-enclave` +image digest. + +The host does not send `APP_ENV`, Datadog variables, logs, traces, or metrics to +the EIF. The enclave launcher sends standard output and standard error to +`/dev/null`. Host configuration and host observability continue to operate +outside the enclave. + +The `signer-dev` target runs the enclave executable as a host process outside a +Nitro enclave. It can show local process logs. It does not have the security +boundary of the production EIF. + +CI uses a test-only shell wrapper. The wrapper adds the Nitro CLI `--debug-mode` +option when it starts the production EIF. The production launcher does not +accept this option. Debug mode does not change the EIF bytes. This test does not +validate a production KMS policy that uses PCR values. + +### Using and Building Images + +#### Circle-published production image + +Circle publishes the tested production image to Cloudsmith. See +[Reproducible Enclave Builds](../README.md#reproducible-enclave-builds) for the +registry location. The linked section explains how Circle delivers the image to +partners. + +Use the supplied token to log in to Cloudsmith. Use the supplied image reference +to pull the image. Record the image digest after the pull: -1. **enclave** (`docker/Dockerfile.enclave`) β€” Runs inside AWS Nitro Enclave - - Contains the enclave application binary - - Listens on port `10350` (internal vsock communication only) - - Entry point: `run_enclave.sh` -2. **signer** (`docker/Dockerfile`) β€” Runs as proxy outside the enclave - - Handles external gRPC requests - - Communicates with enclave via vsock - - Listens on port `10340` for gRPC API - - Entry point: `run_proxy.sh` -3. **signer-with-enclave** (`docker/Dockerfile`) β€” Complete production setup - - Includes both proxy and enclave applications - - Contains the enclave image file (EIF) - - Manages enclave lifecycle automatically - - Entry point: `run.sh` +```bash +docker pull docker.cloudsmith.io/circle/arc/arc-remote-signer: +docker image inspect \ + --format '{{ index .RepoDigests 0 }}' \ + docker.cloudsmith.io/circle/arc/arc-remote-signer: +``` -### Building Images +Use the recorded digest in the deployment configuration. Read the PCR0, PCR1, +and PCR2 values from the labels on the pulled image. Configure the KMS key +policy with the PCR0, PCR1, and PCR2 values and the calculated parent instance +PCR4 value. You do not have to build an EIF when you use the Circle-published +image. -#### Local `signer` image (macOS or Linux) +#### Local `signer-dev` image (macOS or Linux) -A single `make` target builds the Go binary and the host-arch Docker image for the `signer` target β€” recommended for local development against a mock enclave (no `nitro-cli` or EIF needed): +Use this command to build the `signer-dev` target. Use this image to run the host and enclave processes in a local environment. This build does not require `nitro-cli` or an EIF: ```bash -# macOS (Apple Silicon β†’ arm64 image) make local-enclave-docker - -# Linux or when you need a linux/amd64 image -APP_ENV=qa make local-enclave-docker ``` -`APP_ENV=qa` is the toggle that switches `scripts/build.sh` into cross-compiling `linux/amd64` so Docker's `COPY bin/$TARGETARCH/app` resolves correctly. +The `app-builder` stage builds the development host, the production host, and +the development enclave executable. The production host uses the +`prod` tag. You do not need a precompiled binary. The +`docker buildx bake` command does all build steps. The production `signer` and +`signer-with-enclave` targets do not contain development assets. -#### Production image with bundled enclave (`signer-with-enclave`) +#### Custom production image with bundled enclave (`signer-with-enclave`) The `signer-with-enclave` target bundles a real Enclave Image File (EIF). Building the EIF only requires the `nitro-cli` binary β€” **a Nitro-Enclaves-enabled EC2 host is not needed to build an EIF**; that is only required to run an enclave. -1. Install `nitro-cli` by following the official AWS guide: https://docs.aws.amazon.com/enclaves/latest/user/nitro-enclave-cli-install.html +Use the following commands only if you build a custom image from the public +source code. Record the measured PCR values (PCR0, PCR1, and PCR2). If you +publish the custom image, record the registry digest after publication. +Configure the KMS key policy with the measured PCR0, PCR1, and PCR2 values and +the calculated parent instance PCR4 value. + +1. Use the official AWS guide to install `nitro-cli`: https://docs.aws.amazon.com/enclaves/latest/user/nitro-enclave-cli-install.html -2. Create a `docker-container` builder. The enclave bake target uses `rewrite-timestamp=true` for reproducible builds, which requires the `docker-container` driver (BuildKit >= 0.13.0): +2. Create a `docker-container` builder. The `enclave` target for + `docker buildx bake` uses `rewrite-timestamp=true` to create a reproducible + build. This option requires the `docker-container` driver and BuildKit 0.13.0 + or later: ```bash docker buildx create --name enclave-builder --driver docker-container --use @@ -242,25 +381,37 @@ The `signer-with-enclave` target bundles a real Enclave Image File (EIF). Buildi 3. Run the end-to-end build: ```bash - # 1. Build the linux/amd64 host binary - APP_ENV=qa make build - - # 2. Build the enclave Docker image + # 1. Build the enclave Docker image docker buildx bake --provenance=false --allow fs.read=./docker/certs enclave - # 3. Package the Docker image into an EIF + # 2. Package the Docker image into an EIF nitro-cli build-enclave \ --docker-uri nitro-enclave-signer/enclave:latest \ --output-file enclave.eif - # 4. Bundle the EIF into the final signer-with-enclave image + # 3. Record the PCR values + nitro-cli describe-eif \ + --eif-path enclave.eif \ + > describe_eif.json + + PCR0="$(jq -er '.Measurements.PCR0' describe_eif.json)" + PCR1="$(jq -er '.Measurements.PCR1' describe_eif.json)" + PCR2="$(jq -er '.Measurements.PCR2' describe_eif.json)" + + # 4. Bundle the EIF and PCR labels into the final custom image docker buildx bake \ --set "signer-with-enclave.args.ENCLAVE_EIF=./enclave.eif" \ + --set "signer-with-enclave.args.ENCLAVE_PCR0=${PCR0}" \ + --set "signer-with-enclave.args.ENCLAVE_PCR1=${PCR1}" \ + --set "signer-with-enclave.args.ENCLAVE_PCR2=${PCR2}" \ signer-with-enclave ``` > [!NOTE] -> Different versions of `nitro-cli` bundle different kernel and init ramdisk images into the EIF, which changes the PCR0 and PCR2 measurements even when the application code is identical. Pin `nitro-cli` to a specific version and coordinate upgrades with downstream KMS key policy updates. +> Different versions of `nitro-cli` can change the EIF build measurements even +> when the application code is identical. Pin `nitro-cli` to a specific version. +> For every new EIF, record PCR0, PCR1, and PCR2 again and coordinate any KMS key +> policy updates before deployment. ### AWS Prerequisites @@ -292,30 +443,265 @@ Before deploying, provision the following AWS resources in the account that will #### AWS KMS key -The signer uses **envelope encryption**: a symmetric KMS key wraps a per-validator data key, which in turn encrypts the validator's private key stored in Secrets Manager. +The signer uses **envelope encryption**: a symmetric AWS KMS key wraps a per-validator data key, which encrypts the validator private key stored in Secrets Manager. -- Create a **symmetric** key with key spec `SYMMETRIC_DEFAULT`. **Asymmetric / RSA keys are not supported.** +- Create a **symmetric** key with key spec `SYMMETRIC_DEFAULT`. **Asymmetric and RSA keys are not supported.** - A **multi-region** key is recommended β€” `APP_PROVIDER_AWSKMS_ARNS` accepts multiple ARNs and the signer fails over across them. -- The **key policy** must include an attestation-based condition so that only your enclave can call `Decrypt` / `GenerateDataKey` with the `Recipient` parameter. Use the PCR0 hash printed by `nitro-cli build-enclave` (the enclave image is built reproducibly β€” see [Reproducible Enclave Builds](../README.md#reproducible-enclave-builds) β€” so this hash is stable across rebuilds of the same source). - - Enforce `kms:RecipientAttestation:ImageSha384` (PCR0 β€” the enclave image hash). The enclave image is built reproducibly, so this hash is stable across rebuilds of the same source with the same `nitro-cli` version. +- The policy must allow `kms:Decrypt` and `kms:GenerateDataKey` only when `kms:RecipientAttestation:PCR0`, `PCR1`, `PCR2`, and `PCR4` match the approved enclave release and parent EC2 instance. + +##### Enclave measurements and identity + +AWS Nitro Enclaves attestation provides cryptographic measurements in Platform Configuration Registers (PCRs): + +- **PCR0**: SHA-384 hash of the Enclave Image File (EIF). +- **PCR1**: SHA-384 hash of the Linux kernel and bootstrap ramfs. +- **PCR2**: SHA-384 hash of the user applications. +- **PCR4**: SHA-384 hash of the parent EC2 instance ID. -> **Example only β€” replace `000000000000` with your account ID, `arn:aws:iam::000000000000:role/arc-signer-host` with the IAM role attached to the signer's EC2 instance, and the placeholder PCR values with the actual values from your enclave build.** +PCR0, PCR1, and PCR2 establish the approved enclave release identity. PCR4 establishes the deployment identity of the parent EC2 instance. Requiring all four measurements prevents an attacker who compromises the host from copying credentials and ciphertexts to decrypt validator keys on another EC2 instance. + +##### Obtaining and calculating PCR values + +For the Circle-published image, use the digest that you recorded in +[Circle-published production image](#circle-published-production-image). Read +the PCR labels from that exact digest: + +```bash +docker image inspect \ + --format $'PCR0: {{ index .Config.Labels "enclave.pcr0" }}\nPCR1: {{ index .Config.Labels "enclave.pcr1" }}\nPCR2: {{ index .Config.Labels "enclave.pcr2" }}' \ + docker.cloudsmith.io/circle/arc/arc-remote-signer@sha256: +``` + +For a custom image, record the PCR values from the build output or run `describe-eif`: + +```bash +nitro-cli describe-eif \ + --eif-path enclave.eif \ + > describe_eif.json + +PCR0="$(jq -er '.Measurements.PCR0' describe_eif.json)" +PCR1="$(jq -er '.Measurements.PCR1' describe_eif.json)" +PCR2="$(jq -er '.Measurements.PCR2' describe_eif.json)" +``` + +Calculate the `PCR4` measurement for the parent EC2 instance. `PCR4` is the SHA-384 hash of 48 null bytes (`\x00` * 48) followed by the UTF-8 instance ID string (for example, `i-0123456789abcdef0`): + +```bash +# Retrieve instance ID via IMDSv2 +TOKEN="$(curl -s -S -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600")" +INSTANCE_ID="$(curl -s -S -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/instance-id)" + +# Calculate the SHA-384 PCR4 measurement +python3 -c 'import hashlib, sys; h=hashlib.sha384(b"\x00"*48 + sys.argv[1].encode("utf-8")); print(h.hexdigest())' "$INSTANCE_ID" +``` + +Use the measured PCR0, PCR1, and PCR2 values and the calculated PCR4 value in the KMS key policy. Do not use measurements from an unverified build or a different instance. + +##### Canonical key policy + +The following key policy uses six controls: + +1. A dedicated administration role can manage the key. This policy does not + authorize the role to call `Decrypt`, `GenerateDataKey`, or `CreateGrant`. +2. Explicit `Deny` statements reject data-key requests that omit any required + measurement (`PCR0`, `PCR1`, `PCR2`, or `PCR4`). +3. Explicit `Deny` statements reject data-key requests containing unexpected + measurements. Each measurement has a separate `Deny` statement with a + `StringNotEqualsIgnoreCase` condition. Do not combine negative conditions + into a single statement. A combined negative condition can permit unexpected + access if condition semantics evaluate partially. +4. The policy denies `CreateGrant`. The signer does not need grants. Without + this denial, an existing KMS grant can authorize new access. +5. The policy denies `ReEncryptFrom`. Without this denial, a caller can rewrap a + stored encrypted data key under a key that the host controls. The `ReEncrypt` + API has no `Recipient` parameter. +6. A separate `Allow` statement authorizes the signer host role only when the + request contains matching `PCR0`, `PCR1`, `PCR2`, and `PCR4` measurements. + +Use the complete policy when you create the primary key. Do not use the AWS +default policy. While the default policy is active, an IAM policy can authorize +a data-key request that has no recipient attestation. + +The principal that creates a key must have `kms:CreateKey` in its IAM policy. +This requirement applies in the primary region and each replica region. A key +policy cannot authorize `kms:CreateKey`. + +> **Example only. Replace both role ARNs. Replace the account ID. Replace the +> PCR placeholders with your recorded measurements.** +> +> Do not authorize the AWS account principal to call `kms:*`. If the key policy +> authorizes the AWS account principal to call `kms:*`, IAM policies can +> authorize requests without attestation. ```json { "Version": "2012-10-17", "Statement": [ { - "Sid": "EnableRootAccountAdministration", + "Sid": "AllowKeyAdministration", "Effect": "Allow", - "Principal": { "AWS": "arn:aws:iam::000000000000:root" }, - "Action": "kms:*", + "Principal": { + "AWS": "arn:aws:iam::000000000000:role/arc-signer-key-admin" + }, + "Action": [ + "kms:CancelKeyDeletion", + "kms:DescribeKey", + "kms:DisableKey", + "kms:DisableKeyRotation", + "kms:EnableKey", + "kms:EnableKeyRotation", + "kms:GetKeyPolicy", + "kms:GetKeyRotationStatus", + "kms:ListGrants", + "kms:PutKeyPolicy", + "kms:ReplicateKey", + "kms:RevokeGrant", + "kms:ScheduleKeyDeletion", + "kms:UpdateKeyDescription", + "kms:UpdatePrimaryRegion" + ], + "Resource": "*" + }, + { + "Sid": "DenyDataKeyOperationsWithoutPCR0", + "Effect": "Deny", + "Principal": "*", + "Action": [ + "kms:Decrypt", + "kms:GenerateDataKey" + ], + "Resource": "*", + "Condition": { + "Null": { + "kms:RecipientAttestation:PCR0": "true" + } + } + }, + { + "Sid": "DenyDataKeyOperationsWithoutPCR1", + "Effect": "Deny", + "Principal": "*", + "Action": [ + "kms:Decrypt", + "kms:GenerateDataKey" + ], + "Resource": "*", + "Condition": { + "Null": { + "kms:RecipientAttestation:PCR1": "true" + } + } + }, + { + "Sid": "DenyDataKeyOperationsWithoutPCR2", + "Effect": "Deny", + "Principal": "*", + "Action": [ + "kms:Decrypt", + "kms:GenerateDataKey" + ], + "Resource": "*", + "Condition": { + "Null": { + "kms:RecipientAttestation:PCR2": "true" + } + } + }, + { + "Sid": "DenyDataKeyOperationsWithoutPCR4", + "Effect": "Deny", + "Principal": "*", + "Action": [ + "kms:Decrypt", + "kms:GenerateDataKey" + ], + "Resource": "*", + "Condition": { + "Null": { + "kms:RecipientAttestation:PCR4": "true" + } + } + }, + { + "Sid": "DenyDataKeyOperationsForUnexpectedPCR0", + "Effect": "Deny", + "Principal": "*", + "Action": [ + "kms:Decrypt", + "kms:GenerateDataKey" + ], + "Resource": "*", + "Condition": { + "StringNotEqualsIgnoreCase": { + "kms:RecipientAttestation:PCR0": "TRUSTED_PCR0_FOR_SELECTED_IMAGE_DIGEST" + } + } + }, + { + "Sid": "DenyDataKeyOperationsForUnexpectedPCR1", + "Effect": "Deny", + "Principal": "*", + "Action": [ + "kms:Decrypt", + "kms:GenerateDataKey" + ], + "Resource": "*", + "Condition": { + "StringNotEqualsIgnoreCase": { + "kms:RecipientAttestation:PCR1": "TRUSTED_PCR1_FOR_SELECTED_IMAGE_DIGEST" + } + } + }, + { + "Sid": "DenyDataKeyOperationsForUnexpectedPCR2", + "Effect": "Deny", + "Principal": "*", + "Action": [ + "kms:Decrypt", + "kms:GenerateDataKey" + ], + "Resource": "*", + "Condition": { + "StringNotEqualsIgnoreCase": { + "kms:RecipientAttestation:PCR2": "TRUSTED_PCR2_FOR_SELECTED_IMAGE_DIGEST" + } + } + }, + { + "Sid": "DenyDataKeyOperationsForUnexpectedPCR4", + "Effect": "Deny", + "Principal": "*", + "Action": [ + "kms:Decrypt", + "kms:GenerateDataKey" + ], + "Resource": "*", + "Condition": { + "StringNotEqualsIgnoreCase": { + "kms:RecipientAttestation:PCR4": "TRUSTED_PCR4_FOR_PARENT_INSTANCE_ID" + } + } + }, + { + "Sid": "DenyGrantCreation", + "Effect": "Deny", + "Principal": "*", + "Action": "kms:CreateGrant", "Resource": "*" }, { - "Sid": "AllowEnclaveDecryptAndGenerateDataKey", + "Sid": "DenyReEncryptFrom", + "Effect": "Deny", + "Principal": "*", + "Action": "kms:ReEncryptFrom", + "Resource": "*" + }, + { + "Sid": "AllowAttestedDataKeyOperations", "Effect": "Allow", - "Principal": { "AWS": "arn:aws:iam::000000000000:role/arc-signer-host" }, + "Principal": { + "AWS": "arn:aws:iam::000000000000:role/arc-signer-host" + }, "Action": [ "kms:Decrypt", "kms:GenerateDataKey" @@ -323,7 +709,10 @@ The signer uses **envelope encryption**: a symmetric KMS key wraps a per-validat "Resource": "*", "Condition": { "StringEqualsIgnoreCase": { - "kms:RecipientAttestation:ImageSha384": "EXAMPLE_PCR0_REPLACE_WITH_VALUE_FROM_NITRO_CLI_BUILD_OUTPUT" + "kms:RecipientAttestation:PCR0": "TRUSTED_PCR0_FOR_SELECTED_IMAGE_DIGEST", + "kms:RecipientAttestation:PCR1": "TRUSTED_PCR1_FOR_SELECTED_IMAGE_DIGEST", + "kms:RecipientAttestation:PCR2": "TRUSTED_PCR2_FOR_SELECTED_IMAGE_DIGEST", + "kms:RecipientAttestation:PCR4": "TRUSTED_PCR4_FOR_PARENT_INSTANCE_ID" } } } @@ -331,9 +720,204 @@ The signer uses **envelope encryption**: a symmetric KMS key wraps a per-validat } ``` -Reference: [Using cryptographic attestation with AWS KMS](https://docs.aws.amazon.com/enclaves/latest/user/kms.html). +References: + +- [Using cryptographic attestation with AWS KMS](https://docs.aws.amazon.com/enclaves/latest/user/kms.html) +- [AWS KMS Nitro Enclaves condition keys](https://docs.aws.amazon.com/kms/latest/developerguide/conditions-nitro-enclave.html) +- [Grants in AWS KMS](https://docs.aws.amazon.com/kms/latest/developerguide/grants.html) +- [ReEncrypt](https://docs.aws.amazon.com/kms/latest/APIReference/API_ReEncrypt.html) + +The IAM policy for the EC2 host does not need permissions for KMS data-key +operations. Do not add a key policy statement that authorizes `Decrypt` or +`GenerateDataKey` without recipient attestation. Do not add a KMS grant that +authorizes these operations. Do not authorize `ReEncryptFrom` on the signer key. +The policy denies `CreateGrant` for every principal. The signer key does not need +grants. Do not authorize the signer host role to call `kms:PutKeyPolicy`. Do not +authorize the signer host role to assume the key administration role. The key +administration role can change the key policy. Protect the key administration +role. Preserve an approved method to recover administrative access to the key. + +When you update a key policy, AWS KMS does not delete existing grants. Before +production use, assume the key administration role. List the grants for the +primary key and every replica: + +```bash +aws kms list-grants \ + --key-id "$KEY_ARN" \ + --region "$REGION" +``` + +Revoke each grant that includes `Decrypt`, `GenerateDataKey`, `ReEncryptFrom`, +or `CreateGrant`. The signer does not need grants for these operations. Run the +following command for each grant that you must revoke: + +```bash +aws kms revoke-grant \ + --key-id "$KEY_ARN" \ + --grant-id "$GRANT_ID" \ + --region "$REGION" +``` + +Run `list-grants` again. Record the result. The explicit denies prevent an +existing grant from authorizing the denied operations. You must still remove +each grant that permits a denied operation. + +##### Multi-region replication + +Apply the complete policy to the primary key and each multi-region replica key. +The release measurements (PCR0, PCR1, and PCR2) are identical across all regions +for the same EIF. The deployment measurement (PCR4) is identical across regions +because it binds to the parent EC2 instance. Add the ARN for a replica to +`APP_PROVIDER_AWSKMS_ARNS` only after all tests for that replica pass. + +##### Instance replacement procedure + +Follow these steps to replace the parent EC2 instance without service interruption: + +1. **Launch the new instance**: + Launch a new EC2 instance with Nitro Enclaves enabled. + Record the instance ID of the new instance. + +2. **Calculate the new PCR4 measurement**: + Calculate the PCR4 measurement for the new instance ID. + +3. **Update key policies across all regions**: + Update the KMS key policy on the primary key and all replica keys. + Add the new PCR4 measurement to `AllowAttestedDataKeyOperations`: + ```json + "kms:RecipientAttestation:PCR4": [ + "OLD_PCR4_VALUE", + "NEW_PCR4_VALUE" + ] + ``` + Update `DenyDataKeyOperationsForUnexpectedPCR4` to allow both values during cutover: + ```json + "StringNotEqualsIgnoreCase": { + "kms:RecipientAttestation:PCR4": [ + "OLD_PCR4_VALUE", + "NEW_PCR4_VALUE" + ] + } + ``` -> **Note:** The attestation condition lives in the **KMS key policy** above β€” it is not required in the EC2 IAM role policy. The IAM role grants the host permission to call KMS; the key policy then enforces that only requests carrying a valid enclave attestation are honoured. +4. **Deploy and validate the signer**: + Deploy the approved signer release on the new EC2 instance. + Start the signer container. + Verify that the signer reads the validator key from Secrets Manager and KMS. + Verify that health probes pass. + +5. **Cut over and stop the old signer**: + Route validator traffic to the new signer. + Use the trusted EC2 control plane to stop the old instance, and wait until + the instance reaches the `stopped` state. Verify that the old signer no + longer accepts requests. Do not rely on an in-guest shutdown when the old + host might be compromised. + + Removing the old PCR4 measurement does not stop a running enclave. The old + enclave can continue signing with the validator key already cached in its + memory without calling KMS again. + +6. **Remove the old PCR4 measurement**: + Update the KMS key policy across all regions. + Remove the old PCR4 measurement from `AllowAttestedDataKeyOperations` and `DenyDataKeyOperationsForUnexpectedPCR4`. + Verify that only the new PCR4 measurement remains authorized. + + This is the final rollback window for the old instance. To roll back, use + the trusted EC2 control plane to stop the new instance and wait until it + reaches the `stopped` state. Verify that the new signer no longer accepts + requests. Restore the old PCR4 measurement to the key policies across all + regions, restart and validate the old signer, and then route validator + traffic back to it. Never run both signers at the same time with the same + validator key. + +7. **Terminate the old instance**: + Terminate the stopped EC2 instance. Termination is irreversible. Any later + rollback requires launching another instance and authorizing its new PCR4 + measurement. + +##### Incident guidance for suspected host compromise + +If you suspect an attacker compromised the parent host before you enforced PCR4: + +1. **Begin validator key rotation immediately**: + Adding PCR4 to the key policy does not protect a key that an attacker already + recovered. If an attacker copied temporary credentials and decrypted the + data key, the attacker can sign messages outside your infrastructure. Treat + the old validator key as compromised. + +2. **Create an isolated replacement environment**: + Deploy a fresh EC2 instance with a new signer host role. Provision a new KMS + key, including any required multi-region replicas, and apply the complete + PCR0, PCR1, PCR2, and new-instance PCR4 policy to every key. Do not reuse the + old secret, KMS key, or host role for the replacement validator. + +3. **Generate and verify the new validator key**: + Generate a new validator signing key in the new enclave and store its + encrypted envelope in a new Secrets Manager secret protected by the new KMS + key. Restart the new signer to clear its enclave memory. Verify that it can + recover the new validator key and sign after restart using only the new + secret, KMS key, and host role. + +4. **Replace the compromised public key on-chain**: + Submit the protocol-specific transaction that replaces or revokes the old + validator public key; registering an additional key is not sufficient if + the old key remains authorized. Wait for the transaction to reach the + required finality. Verify that signatures from the new key are accepted and + signatures from the old key are rejected. + +5. **Stop and terminate the compromised host**: + Use the trusted EC2 control plane to stop the compromised instance. Verify + that the old signer no longer accepts requests, and then terminate the + instance. Do not rely on an in-guest shutdown. + +6. **Retire the old AWS recovery path**: + Confirm that the replacement signer and all other workloads do not depend on + the old secret, KMS key, or host role. Delete the old secret, revoke the old + role sessions and permissions, and remove the old PCR4 measurement from all + old KMS key policies. Disable an old KMS key only after confirming that no + remaining workload uses it. These actions prevent future recovery of the old + validator key, but the on-chain replacement in the previous step is what + invalidates a copy that an attacker already recovered. + +##### AWS key policy validation tests + +Before production use, run the following positive and negative tests against the key policy: + +1. **Positive test (Approved EIF on authorized instance)**: + Start the approved EIF on the authorized parent EC2 instance. + Verify that `kms:GenerateDataKey` and `kms:Decrypt` succeed through the attested enclave. + +2. **Negative test (Unauthorized parent instance / stolen credentials)**: + Copy the temporary IAM role credentials and encrypted secret envelope to a different EC2 instance. + Start the same approved EIF on that different EC2 instance. + Verify that KMS rejects the request with `AccessDeniedException` because PCR4 does not match. + +3. **Negative test (Modified enclave image)**: + Start an unapproved or modified EIF on the authorized EC2 instance. + Verify that KMS rejects the request with `AccessDeniedException` because PCR0, PCR1, or PCR2 does not match. + +4. **Negative test (Missing recipient attestation)**: + Make a `kms:Decrypt` or `kms:GenerateDataKey` request directly from the host without recipient attestation. + Verify that KMS rejects the request with `AccessDeniedException`. + +5. **Negative test (Unauthorized grant or re-encryption)**: + Call `CreateGrant` for the signer key. Verify that KMS rejects the request + with `AccessDeniedException` because the key policy denies + `kms:CreateGrant`. + + Call the `ReEncrypt` API with a ciphertext encrypted under the signer key as + the source and a test-controlled KMS key as the destination. Verify that KMS + rejects the request with `AccessDeniedException` because the signer key + policy denies the required `kms:ReEncryptFrom` permission. + +6. **Multi-Region validation**: + Repeat these tests for every multi-region replica key. + +7. **Auditor evidence**: + Collect and record the following evidence for security auditors: + - The active KMS key policy document for the primary key and all replicas (`aws kms get-key-policy`). + - AWS CloudTrail event logs showing `AccessDeniedException` for all negative test attempts and success for authorized operations. + - The immutable Git commit SHA of the approved release. #### AWS Secrets Manager secret @@ -358,29 +942,58 @@ You will pass this name (or its full ARN) to the signer via `APP_SERVICE_SIGNER_ #### IAM role on the EC2 host -Attach an instance profile / role to the signer's EC2 host with the four permissions below, scoped to the specific KMS key ARN and secret ARN you just created β€” not `*`. +The signer writes to Secrets Manager only during **first-run key generation**. +During this run, the configured secret has no stored value. The host calls +`Initialize` with `generate_new` and persists the wrapped key with +`secretsmanager:PutSecretValue` (see [Key Generation Flow](#key-generation-flow)). +Once a non-empty secret exists, every subsequent startup calls `Initialize` +with `existing_key` and does not call `PutSecretValue`. Steady-state operation +needs read access only. -> **Example only (single-region minimum) β€” replace `000000000000`, the KMS key ID, and the secret name with your own values.** +Remove `secretsmanager:PutSecretValue` permission after bootstrap because the +long-running host proxy is untrusted. Authorize write access for the bootstrap run +only. Confirm that the host stored the key. Then use a runtime role with +read-only access. Use two least-privilege IAM policies. Scope both policies to +the validator secret ARN. The KMS key policy allows `Decrypt` and +`GenerateDataKey` only for requests with valid recipient attestation. + +**Bootstrap policy (first run only).** Attach the bootstrap policy to a dedicated +bootstrap role. Alternatively, attach the bootstrap policy to the runtime role +only for the first key-generation run. The bootstrap policy permits +`GetSecretValue` and `PutSecretValue` on the specified validator secret. + +> **Example only. Replace the account ID and secret name with your own values.** ```json { "Version": "2012-10-17", "Statement": [ { - "Sid": "KMSEnvelopeOps", + "Sid": "SecretsManagerValidatorKeyBootstrap", "Effect": "Allow", "Action": [ - "kms:GenerateDataKey", - "kms:Decrypt" + "secretsmanager:GetSecretValue", + "secretsmanager:PutSecretValue" ], - "Resource": "arn:aws:kms:us-east-1:000000000000:key/EXAMPLE-KEY-ID-1234-5678" - }, + "Resource": "arn:aws:secretsmanager:us-east-1:000000000000:secret:arc-chain/dev/validator-1/key-*" + } + ] +} +``` + +**Runtime policy (steady-state).** After bootstrap completes, the host runs under this policy. It is identical to the bootstrap policy except that the Secrets Manager statement drops `PutSecretValue`, leaving read-only access. + +> **Example only β€” replace the placeholders as above.** + +```json +{ + "Version": "2012-10-17", + "Statement": [ { - "Sid": "SecretsManagerValidatorKey", + "Sid": "SecretsManagerValidatorKeyRuntime", "Effect": "Allow", "Action": [ - "secretsmanager:GetSecretValue", - "secretsmanager:PutSecretValue" + "secretsmanager:GetSecretValue" ], "Resource": "arn:aws:secretsmanager:us-east-1:000000000000:secret:arc-chain/dev/validator-1/key-*" } @@ -390,16 +1003,16 @@ Attach an instance profile / role to the signer's EC2 host with the four permiss The trailing `-*` on the secret ARN is required because Secrets Manager appends a random 6-character suffix to each secret's full ARN. -**Multi-region keys:** if you created a multi-region KMS key and replicated it to additional regions, add one `Resource` ARN per replica to the `KMSEnvelopeOps` statement. For example: +#### Bootstrap-to-runtime permission lifecycle -```json -"Resource": [ - "arn:aws:kms:us-east-1:000000000000:key/mrk-EXAMPLE-KEY-ID-1234-5678", - "arn:aws:kms:us-west-2:000000000000:key/mrk-EXAMPLE-KEY-ID-1234-5678" -] -``` +Perform these steps once per validator secret, in order. The goal is to leave the long-running host with read-only Secrets Manager access. + +1. **Bootstrap run.** Deploy the signer under the bootstrap policy. On first run, the empty secret makes the host call `Initialize` with `generate_new`. The host writes the returned wrapped key with `PutSecretValue`. +2. **Confirm the wrapped key was stored.** Before removing write access, verify the secret now holds a non-empty value β€” for example, `aws secretsmanager get-secret-value --secret-id arc-chain/dev/validator-1/key --query SecretString --output text` returns the base64 envelope. Do not proceed until this succeeds. +3. **Switch to the runtime role.** Move the host to the runtime policy (drop `PutSecretValue`), and **invalidate the bootstrap access**: remove, expire, or explicitly revoke the bootstrap role/session so its write permission can no longer be assumed. +4. **Verify write access is gone.** Under the runtime role, confirm `secretsmanager:PutSecretValue` on the validator secret returns `AccessDenied` (for example, a `put-secret-value` call is rejected). A successful denial confirms a compromised host proxy can no longer overwrite or evict the recoverable validator key. -Each replica key also requires its own key policy with the attestation condition (the PCR0 value is the same across regions for the same EIF). +An existing non-empty secret makes the host call `Initialize` with `existing_key`. Restarts and redeployments do not call `PutSecretValue`, so the runtime role needs no write access. #### Wiring it up @@ -408,9 +1021,9 @@ Once the resources above exist, point the signer at them by setting: - `APP_PROVIDER_AWSKMS_ARNS` β€” the KMS key ARN (or comma-separated list for multi-region) - `APP_SERVICE_SIGNER_KEYID` β€” the Secrets Manager secret name or ARN -See [Configuration](#configuration) below for the full environment variable reference. +See [Common Production Settings](#common-production-settings) below for the settings most commonly needed in production. -### Configuration +### Common Production Settings The service uses [Viper](https://github.com/spf13/viper) for configuration management with the following precedence (highest to lowest): @@ -418,30 +1031,54 @@ The service uses [Viper](https://github.com/spf13/viper) for configuration manag 2. Configuration file 3. Default values -All environment variables use the `APP_` prefix. Nested keys use underscore style in env vars (for example, `provider.awskms.arns` -> `APP_PROVIDER_AWSKMS_ARNS`). +Application settings loaded through Viper use the `APP_` prefix. Nested keys use +underscores in environment variable names. For example, +`provider.awskms.arns` maps to `APP_PROVIDER_AWSKMS_ARNS`. The production build +applies its security policy after Viper loads the settings. This policy enables +Nitro, sets the VSOCK CID and ports, and disables LocalStack. In the production +build, `APP_ENV=stg` is only a deployment label. The environment label does not +select runtime behavior. The production AWS client ignores configured endpoint +overrides. Standard observability variables configure the host only. The +launcher does not send these variables to the EIF. + +The production images do not set a default AWS Region. Set `AWS_REGION` before +you start the signer. You can also set the region in a standard AWS shared +configuration file. The host reads the region from the AWS configuration. +During initialization, the host sends the region to the enclave. The production +endpoint policy ignores endpoint overrides. The AWS client continues to use the +region. + +The production launcher does not set `APP_PUBLIC_SERVER_PORT`. The packaged +production configuration file sets the default port to `10340`. Set +`APP_PUBLIC_SERVER_PORT` to use a different port. ```bash # Required - AWS configuration -# APP_PROVIDER_SECRETS_LOCALSTACK_REGION overrides the AWS region for Secrets Manager. -# On EC2, the SDK resolves region from instance metadata automatically β€” only set this if -# the SDK cannot detect the region (e.g. running outside EC2 or with --network bridge). -# In production, prefer the standard AWS_REGION env var instead. -export APP_PROVIDER_SECRETS_LOCALSTACK_REGION=us-east-1 +export AWS_REGION=us-east-1 export APP_PROVIDER_AWSKMS_ARNS=arn:aws:kms:us-east-1:000000000000:key/EXAMPLE-KEY-ID-1234-5678 # APP_SERVICE_SIGNER_KEYID accepts either the secret name or its full ARN. export APP_SERVICE_SIGNER_KEYID=arc-chain/dev/validator-1/key # Enclave configuration -export APP_PROVIDER_ENCLAVE_NITROENCLAVE_ENABLED=true -export APP_PROVIDER_ENCLAVE_NITROENCLAVE_CID=16 -export APP_PROVIDER_ENCLAVE_NITROENCLAVE_PORT=10350 +# Only signer-dev uses ENABLE_ENCLAVE. The production launcher always starts Nitro. +# A host binary with the prod build tag uses VSOCK CID 16 and port 10350. +# APP_PROVIDER_ENCLAVE_NITROENCLAVE_CID and +# APP_PROVIDER_ENCLAVE_NITROENCLAVE_PORT cannot change this target. # Service configuration export APP_PUBLIC_SERVER_PORT=10340 +export APP_PUBLIC_SERVER_TLS_ENABLED=true +export APP_PUBLIC_SERVER_TLS_CERT=/etc/tls/server.crt +export APP_PUBLIC_SERVER_TLS_KEY=/etc/tls/server.key +# Optional mTLS client authentication. Enable only after Malachite and every +# health probe using port 10340 have trusted client certificates. +# export APP_PUBLIC_SERVER_TLS_CLIENTAUTH_ENABLED=true +# export APP_PUBLIC_SERVER_TLS_CLIENTAUTH_CA=/etc/tls/client-ca.crt export APP_PROFILER_ENABLED=false -# Observability (optional) -export APP_OTLP_ENDPOINT=localhost:4317 +# Host observability (optional) +# Traces use the standard OpenTelemetry endpoint variable. +export OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:4317 export DD_AGENT_HOST=localhost export DD_SERVICE=arc-signer export DD_ENV=prod @@ -453,15 +1090,16 @@ export APP_METRICS_PROMETHEUS_PORT=9090 export APP_METRICS_PROMETHEUS_PATH=/metrics ``` +`APP_PROVIDER_SECRETS_LOCALSTACK_REGION` is a development/LocalStack setting and is not the production AWS region selector. The public gRPC listener is plaintext by default. Enabling the server certificate settings provides server-authenticated TLS; enabling client authentication additionally requires certificates issued by the configured client CA. Client authentication fails startup when TLS is disabled or the CA path is missing, unreadable, empty, or not composed entirely of valid PEM-encoded X.509 certificates. The application verifies CA membership rather than a specific SAN or SPIFFE identity, so the client CA must be dedicated to authorized Malachite workloads. + +Client authentication covers every service on the public listener, including the gRPC health service. Before enabling it, configure Malachite and every health probe that uses port 10340 to present a trusted client certificate. Roll out the client identity and probe support before requiring client certificates on the server; otherwise the fail-closed handshake will make signing and health checks unavailable. This setting does not authenticate callers of the enclave's port 10350 TCP/VSOCK API. + Example config file (`configs/app.yaml`): ```yaml provider: awskms: arns: arn:aws:kms:us-east-1:000000000000:key/EXAMPLE-KEY-ID-1234-5678 - secrets: - localstack: - region: us-east-1 service: signer: @@ -515,6 +1153,19 @@ scrape_configs: - External API port: `10340` - Enclave operations remain inside Nitro Enclave boundary +When running the image directly with Docker, the host must have Nitro Enclaves +enabled and enough CPU and memory configured in the Nitro Enclaves allocator. +Pass `/dev/nitro_enclaves` into the container and use a seccomp profile that +allows `AF_VSOCK` (the smoke test uses `--security-opt seccomp=unconfined`). +Expose port `10340` with normal Docker port publishing or host networking; host +networking is not itself a Nitro Enclaves requirement. + +On Kubernetes, request the Nitro Enclaves device resource and sufficient +hugepage memory for the configured enclave size. The production docker-arc +deployment requests one `aws.ec2.nitro/nitro_enclaves` device and 4 GiB of +1 GiB hugepages. Its seccomp policy must also allow `AF_VSOCK`; regular pod +networking can expose port `10340`, so `hostNetwork` is not required. + Deployment scripts in `docker/`: - `run.sh` @@ -539,19 +1190,19 @@ Deployment scripts in `docker/`: | Method | Request | Response | |--------|---------|----------| -| `PublicKey` | _(empty)_ | `public_key bytes` | +| `PublicKey` | _(empty)_ | `public_key`, `attestation_document` | | `Sign` | `message bytes` | `signature bytes` | -**EnclaveService** (internal, port 10350 vsock): +**EnclaveService** (internal, port 10350 over TCP in dev/CI or VSOCK in Nitro production): | Method | Request | Response | |--------|---------|----------| -| `GenerateKey` | `algorithm`, `enclave_encrypted_data_key` | `public_key`, `encrypted_private_key`, `nonce` | -| `GetPublicKey` | `algorithm`, `encrypted_key_material` | `public_key` | -| `SignMessage` | `algorithm`, `encrypted_key_material`, `message` | `signature` | -| `GetAttestation` | _(empty)_ | `attestation_document` | +| `Initialize` | AWS credentials, KMS key ARNs, LocalStack selector, and one key source: `existing_key` or `generate_new` | `public_key`, `attestation_document`, and `secret_envelope` only for a generated key | +| `GenerateKey` | `algorithm` | Returns `FAILED_PRECONDITION`. Use `Initialize` to generate a key. | +| `GetPublicKey` | _(empty)_ | Installed `public_key`, `attestation_document` | +| `SignMessage` | `message` | `signature` | -`encrypted_key_material` is a composite field containing `encrypted_private_key`, `enclave_encrypted_data_key`, and `nonce`. For full field definitions see the proto files listed above. +`secret_envelope` is the wrapped key: `{ algorithm, kms_encrypted_data_key, encrypted_private_key, nonce }`. `Initialize` returns it when the enclave generates a key. The host persists it in Secrets Manager. On a later startup, the host sends it as `existing_key` in `Initialize`, and the enclave recovers the key. `GenerateKey` and `GetPublicKey` remain in the internal API for compatibility, but they are not the initialization path. For full field definitions, see the proto files listed above. ### Error Handling @@ -559,9 +1210,12 @@ Deployment scripts in `docker/`: |------|---------|---------------| | `OK (0)` | Success | β€” | | `INVALID_ARGUMENT (3)` | Bad request | Missing or malformed message payload, unknown algorithm | -| `NOT_FOUND (5)` | Resource missing | Key not found in Secrets Manager for the configured key ID | -| `INTERNAL (13)` | Server error | KMS/Secrets Manager call failed, enclave decryption error, signing failure | -| `UNAVAILABLE (14)` | Service unreachable | Enclave not running, vsock connection failure | +| `PERMISSION_DENIED (7)` | KMS rejected the request | Invalid IAM/KMS permissions or attestation policy mismatch | +| `FAILED_PRECONDITION (9)` | Operation requires the Ready state | `Initialize` has not completed, `GenerateKey` was called directly, or the enclave already has a different key source | +| `INTERNAL (13)` | Server error | Uninitialized service state, enclave key unwrap/decryption error, signing failure | +| `UNAVAILABLE (14)` | Dependency unavailable | Enclave transport failure, retryable KMS failure, or expired public key attestation | + +Secrets Manager is accessed during service initialization. Its failures prevent startup rather than returning a public signer RPC status. ## Troubleshooting @@ -570,18 +1224,15 @@ Deployment scripts in `docker/`: Service fails with enclave connection error: - Verify enclave is running: `nitro-cli describe-enclaves` -- Verify configured CID (default `16`) +- Verify that the enclave CID is `16` - Inspect enclave logs: `nitro-cli console --enclave-id ` -Failed to retrieve attestation document: - -- Verify `/dev/nsm` exists in enclave runtime -- Verify attestation path in enclave startup logs +Unexpected signing latency: -Persistent signing latency: - -- First operation warm-up is expected -- For sustained latency, inspect profiling output (`pprof` path) +- Verify that startup `Initialize` completed. It generates or recovers the key and installs it in the enclave. +- Steady-state signing uses the installed key. It does not call KMS or Secrets Manager. +- In Nitro mode, `Initialize` calls NSM for KMS Recipient attestation and public key attestation. A background task calls NSM again before the certificate for the public-key attestation expires. +- For sustained latency, inspect traces and enclave logs. When `APP_PROFILER_ENABLED=true`, inspect the Datadog continuous profiler output AWS permission errors: @@ -595,12 +1246,6 @@ Proto mismatch issues: ### Debugging -Enable debug logging: - -```bash -APP_LOG_LEVEL=debug ./bin/app -``` - For enclave-focused debugging: - `nitro-cli console --enclave-id ` diff --git a/docs/development.md b/docs/development.md index 0e1916e..34923b6 100644 --- a/docs/development.md +++ b/docs/development.md @@ -1,6 +1,6 @@ # Development -Local development uses `deployments/docker-compose.yaml` with localstack to simulate AWS services. All workflows are driven through `make` targets, which handle dependency ordering automatically. +Local development uses `deployments/docker-compose.yaml`. The default configuration uses LocalStack to simulate AWS services. All workflows use `make` targets, which handle dependency ordering. For architecture, deployment topology, and the security model, see [architecture.md](architecture.md). For test conventions and coverage expectations, see [testing.md](testing.md). @@ -18,10 +18,21 @@ For test conventions and coverage expectations, see [testing.md](testing.md). | Command | Description | |---------|-------------| -| `make up` | Build local enclave Docker image and start localstack | +| `make up` | Build the local `signer-dev` image. Start LocalStack. | | `make down` | Tear down the local environment: stop the app (via `app.pid`) and run `docker-compose down --remove-orphans` | | `make dev` | `make up` + launch the app (typical dev entry point) | +The development settings use separate provider switches: + +- `APP_PROVIDER_SECRETS_LOCALSTACK_ENABLED` selects LocalStack for Secrets Manager. +- `APP_PROVIDER_AWSKMS_LOCALSTACK_ENABLED` selects LocalStack for KMS. + +The signer-dev image can use standard AWS KMS while `APP_ENV=dev`. Set +`APP_PROVIDER_AWSKMS_LOCALSTACK_ENABLED=false` and provide +`APP_PROVIDER_AWSKMS_ARNS` and AWS credentials. The development compose stack +still starts LocalStack, but the KMS proxy does not use it. Standard AWS KMS in +non-Nitro mode does not send enclave attestation. + ### Build | Command | Description | @@ -43,10 +54,10 @@ See [testing.md](testing.md) for the full test matrix and conventions. Top-level ## Conventions - **Go version**: 1.25 -- **Copyright header**: All `.go` files must carry the Circle Internet Group Apache 2.0 license header. Enforced by the `check-copyright-golang` pre-commit hook. +- **Copyright header**: All `.go` files except generated `*_mock.go` files must carry the Circle Internet Group Apache 2.0 license header. The exception matches the `check-copyright-golang` pre-commit hook. - **Commit messages**: `type(ticket|NOSTORY): description`, following [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/) for the `type` semantics. Valid tickets follow the project's Jira prefix. - **Mocking**: `github.com/golang/mock` (v1.6.0). Mocks are `*_mock.go` files co-located with the interfaces they mock; regenerate with `mockgen`. -- **Config**: Viper-based, env var prefix `APP_`. See [architecture.md#configuration](architecture.md#configuration). +- **Config**: Viper-based, env var prefix `APP_`. See [architecture.md#common-production-settings](architecture.md#common-production-settings). - **Logging**: Use the repo-local logger in `internal/common/logging/`. - **Protocol Buffers**: Managed by `buf` (config version v2), definitions in `proto/`. Run `make proto` after editing. - **gRPC**: Shared foundation in `internal/common/grpc/` with standardized client/server lifecycle and error normalization. @@ -56,8 +67,8 @@ See [testing.md](testing.md) for the full test matrix and conventions. Top-level Enforced via `.pre-commit-config.yaml`: - `go-fmt`, `golangci-lint`, `go-mod-tidy`, `go-unit-tests` -- `no-go-testing` β€” use `testify` assertions, not raw `testing` -- `check-copyright-golang` β€” Circle copyright header on every `.go` file +- `no-go-testing` β€” scans files named `test_*.go` for the literal `testing.T`; it does not parse imports, enforce Testify assertion style, or match the repository's standard `*_test.go` filenames +- `check-copyright-golang` β€” Circle copyright header on `.go` files except generated `*_mock.go` - `terraform_fmt` β€” for `deploy/` configs Run `pre-commit install` once after cloning to enable the hooks locally. @@ -69,23 +80,33 @@ Run `pre-commit install` once after cloning to enable the hooks locally. 3. `make test-it` β€” when changes touch provider integrations, gRPC behavior, enclave communication, or config 4. `make test-all` β€” for high-risk or release-critical changes +Signing-critical and signer-image changes also rely on the CI-only `smoke_eif` +job, which has no equivalent local `make` target. See +[CI-only EIF Image Validation](testing.md#ci-only-eif-image-validation). + ## Project Structure ``` . -β”œβ”€β”€ cmd/ # CLI entry points +β”œβ”€β”€ cmd/ # CLI entry points (app, run-vsockproxy) β”œβ”€β”€ internal/ -β”‚ β”œβ”€β”€ app/ # Proxy (host-side) +β”‚ β”œβ”€β”€ app/ # Host application (app run) β”‚ β”‚ β”œβ”€β”€ public/ # gRPC handlers -β”‚ β”‚ β”œβ”€β”€ service/ # Business logic -β”‚ β”‚ └── provider/ # AWS integrations (KMS, Secrets Manager) +β”‚ β”‚ β”œβ”€β”€ service/ # Business logic (signer) +β”‚ β”‚ β”œβ”€β”€ provider/ # Secrets Manager + enclave client (no KMS on the host) +β”‚ β”‚ └── metrics/ # API metrics β”‚ β”œβ”€β”€ enclave/ # Enclave-side (inside Nitro Enclave) β”‚ β”‚ β”œβ”€β”€ public/ # Enclave gRPC handlers β”‚ β”‚ β”œβ”€β”€ service/ # Enclave business logic -β”‚ β”‚ └── provider/ # Key storage, attestation +β”‚ β”‚ β”œβ”€β”€ provider/ # awskms (KMS client), awsproxy, enclave (NSM) +β”‚ β”‚ └── common/crypto/ # In-enclave AES, BLS, Ed25519 primitives +β”‚ β”œβ”€β”€ vsockproxy/ # Host-side vsock↔AWS KMS bridge (app run-vsockproxy) β”‚ β”œβ”€β”€ common/ # Shared infrastructure +β”‚ β”‚ β”œβ”€β”€ byteproxy/ # vsock byte proxy + AWS route framing +β”‚ β”‚ β”œβ”€β”€ config/ # Viper config loader β”‚ β”‚ β”œβ”€β”€ crypto/ # AES, random generation β”‚ β”‚ β”œβ”€β”€ grpc/ # gRPC client/server/interceptors +β”‚ β”‚ β”œβ”€β”€ lifecycle/ # Runnable lifecycle management β”‚ β”‚ β”œβ”€β”€ logging/ # Structured logging β”‚ β”‚ β”œβ”€β”€ metric/ # Datadog metrics β”‚ β”‚ └── telemetry/ # OpenTelemetry @@ -94,6 +115,7 @@ Run `pre-commit install` once after cloning to enable the hooks locally. β”‚ β”œβ”€β”€ arc/signer/v1/ # SignerService (external API) β”‚ └── arc/enclave/v1/ # EnclaveService (internal API) β”œβ”€β”€ docker/ # Docker build configuration +β”œβ”€β”€ deploy/ # Terraform deployment configs β”œβ”€β”€ deployments/ # Docker Compose (localstack) └── scripts/ # Build and utility scripts ``` diff --git a/docs/testing.md b/docs/testing.md index 3f8d201..c374a91 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -8,16 +8,48 @@ For the contribution workflow that wraps these commands, see [development.md](de | Command | Scope | Notes | |---------|-------|-------| -| `make test` | Unit tests + lint | β€” | +| `make test` | Unit tests + lint | Also checks development and production image contracts | | `make test-it` | Unit + integration | Starts localstack via `make up` | | `make smoke` | Smoke (end-to-end) | Starts localstack and launches `bin/app` automatically (via `scripts/smoke.sh`) | -| `make test-all` | All of the above | Combines the above | +| `make test-all` | All local test targets | Combines the above; does not include CI Nitro EIF image validation | Test files are co-located with source and use build tags to control scope: -- Integration tests: `//go:build integration` (files named `integration_test.go`) +- Integration tests: `//go:build integration` (files named `*_integration_test.go`, e.g. `awskms_integration_test.go`) - Smoke tests: `//go:build smoke`, located in `internal/smoke/` +## CI-only EIF Image Validation + +The PR pipeline conditionally runs `smoke_eif` when signer image or runtime +inputs change, or when the `build-docker` label is present. The job downloads +the built `signer-with-enclave` Docker archive. It runs `scripts/smoke-eif.sh` on +a Nitro runner. The script checks the configured image entry point and the +production files in the image. It also checks that development files and +settings are absent. The CI-only wrapper then starts the same `docker/run.sh` +launcher with conflicting development settings. The smoke test includes image +startup, VSOCK communication, KMS recipient encryption, and signing. + +The test uses a CI-only shell wrapper to add the Nitro CLI `--debug-mode` option. +The production launcher does not accept this option. Debug mode does not change +the EIF bytes or create a second EIF. The test does not validate a production +KMS policy that uses PCR values. A production-mode test is necessary for each +PCR policy change. + +For an internal release, the pipeline runs the same test on the +`signer-with-enclave` Docker archive. The pipeline publishes the Docker archive +only after this test passes. It pushes the same digest to Cloudsmith and ECR. +Staging and production must use this digest. + +The public mirror does not publish the Docker archive. Its release workflow +runs the public CI tests before it syncs the source to +`circlefin/arc-remote-signer`. + +No local `make` target includes this validation because it requires a Nitro +device, the CI AWS role, and the built image artifact. `make smoke` and +`make test-all` use the local LocalStack-based flow instead. When `smoke_eif` is +selected for a PR, its result is included in the merge-blocking required-checks +rollup. + ## Conventions ### Framework @@ -25,7 +57,7 @@ Test files are co-located with source and use build tags to control scope: - `testify/require` by default; use `require` for preconditions and dependency checks (`NoError`, `NotNil`, etc.). - `testify/assert` only when intentionally collecting multiple independent failures in one run. - `testify/suite` only when shared setup/teardown across many tests is genuinely needed. -- The `no-go-testing` pre-commit hook forbids raw `testing` assertions β€” use `testify`. +- Testify is repository policy. The pinned `no-go-testing` pre-commit hook scans files named `test_*.go` for the literal `testing.T`; it does not parse imports, enforce assertion style, or match standard `*_test.go` filenames. ### Naming @@ -67,7 +99,7 @@ Test files are co-located with source and use build tags to control scope: - Tests must be deterministic and reproducible. - Avoid real wall clock, real network, and external services in unit tests. -- For async coordination, use channels, context deadlines, or bounded polling β€” never fixed `time.Sleep`. +- New or modified tests should use channels, context deadlines, or bounded polling for async coordination rather than fixed `time.Sleep`. Some legacy tests still contain bounded sleeps; replace them when touched where practical. ### State Safety diff --git a/go.mod b/go.mod index c5dcc67..c2ff8e8 100644 --- a/go.mod +++ b/go.mod @@ -9,11 +9,12 @@ require ( github.com/aws/aws-sdk-go-v2/config v1.29.16 github.com/aws/aws-sdk-go-v2/service/kms v1.41.0 github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.35.6 - github.com/edgebitio/nitro-enclaves-sdk-go v1.0.0 github.com/golang/mock v1.6.0 github.com/google/uuid v1.6.0 github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0 github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.3 + github.com/hdevalence/ed25519consensus v0.2.0 + github.com/hf/nsm v0.0.0-20220930140112-cd181bd646b9 github.com/mdlayher/vsock v1.2.1 github.com/prometheus/client_golang v1.23.2 github.com/spf13/cobra v1.9.1 @@ -22,17 +23,18 @@ require ( github.com/supranational/blst v0.3.14 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 - go.opentelemetry.io/otel v1.43.0 + go.opentelemetry.io/otel v1.44.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0 - go.opentelemetry.io/otel/sdk v1.43.0 - go.opentelemetry.io/otel/trace v1.43.0 - google.golang.org/grpc v1.79.3 + go.opentelemetry.io/otel/sdk v1.44.0 + go.opentelemetry.io/otel/trace v1.44.0 + google.golang.org/grpc v1.83.1 google.golang.org/protobuf v1.36.11 gopkg.in/DataDog/dd-trace-go.v1 v1.73.1 sigs.k8s.io/yaml v1.6.0 ) require ( + filippo.io/edwards25519 v1.0.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/prometheus/client_model v0.6.2 // indirect @@ -41,7 +43,7 @@ require ( ) require ( - cel.dev/expr v0.25.1 // indirect + cel.dev/expr v0.25.2 // indirect github.com/DataDog/gostackparse v0.7.0 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/antlr4-go/antlr/v4 v4.13.1 // indirect @@ -56,21 +58,20 @@ require ( github.com/aws/aws-sdk-go-v2/service/sso v1.25.4 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.2 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.33.21 // indirect - github.com/aws/smithy-go v1.23.2 // indirect + github.com/aws/smithy-go v1.23.2 github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect - github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-viper/mapstructure/v2 v2.4.0 // indirect + github.com/go-viper/mapstructure/v2 v2.4.0 github.com/gogo/protobuf v1.3.2 // indirect github.com/google/cel-go v0.27.0 // indirect github.com/google/pprof v0.0.0-20250501235452-c0086092b71a // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect - github.com/hf/nsm v0.0.0-20220930140112-cd181bd646b9 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mdlayher/socket v0.4.1 // indirect @@ -92,19 +93,19 @@ require ( go.opentelemetry.io/collector/pdata v1.33.0 // indirect go.opentelemetry.io/collector/pdata/pprofile v0.127.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect - go.opentelemetry.io/otel/metric v1.43.0 // indirect + go.opentelemetry.io/otel/metric v1.44.0 // indirect go.opentelemetry.io/proto/otlp v1.7.1 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.1 // indirect go.yaml.in/yaml/v2 v2.4.2 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/exp v0.0.0-20250813145105-42675adae3e6 // indirect - golang.org/x/net v0.48.0 // indirect - golang.org/x/sync v0.19.0 // indirect - golang.org/x/sys v0.42.0 // indirect - golang.org/x/text v0.32.0 // indirect + golang.org/x/net v0.55.0 // indirect + golang.org/x/sync v0.20.0 + golang.org/x/sys v0.45.0 // indirect + golang.org/x/text v0.37.0 // indirect golang.org/x/time v0.11.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index d8166ed..56bed60 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,10 @@ buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.11-202602092021 buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.11-20260209202127-80ab13bee0bf.1/go.mod h1:tvtbpgaVXZX4g6Pn+AnzFycuRK3MOz5HJfEGeEllXYM= buf.build/go/protovalidate v1.1.3 h1:m2GVEgQWd7rk+vIoAZ+f0ygGjvQTuqPQapBBdcpWVPE= buf.build/go/protovalidate v1.1.3/go.mod h1:9XIuohWz+kj+9JVn3WQneHA5LZP50mjvneZMnbLkiIE= -cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= -cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +cel.dev/expr v0.25.2 h1:K6j46C81hXtZQfuX60cVWQFBJahKSE2gfRbNuvr5bFs= +cel.dev/expr v0.25.2/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +filippo.io/edwards25519 v1.0.0 h1:0wAIcmJUqRdI8IJ/3eGi5/HwXZWPujYXXlkrQogz0Ek= +filippo.io/edwards25519 v1.0.0/go.mod h1:N1IkdkCkiLB6tki+MYJoSx2JTY9NUlxZE7eHn5EwJns= github.com/DataDog/appsec-internal-go v1.9.0 h1:cGOneFsg0JTRzWl5U2+og5dbtyW3N8XaYwc5nXe39Vw= github.com/DataDog/appsec-internal-go v1.9.0/go.mod h1:wW0cRfWBo4C044jHGwYiyh5moQV2x0AhnwqMuiX7O/g= github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.64.0-rc.1 h1:XHITEDEb6NVc9n+myS8KJhdK0vKOvY0BTWSFrFynm4s= @@ -95,8 +97,6 @@ github.com/eapache/queue/v2 v2.0.0-20230407133247-75960ed334e4 h1:8EXxF+tCLqaVk8 github.com/eapache/queue/v2 v2.0.0-20230407133247-75960ed334e4/go.mod h1:I5sHm0Y0T1u5YjlyqC5GVArM7aNZRUYtTjmJ8mPJFds= github.com/ebitengine/purego v0.8.2 h1:jPPGWs2sZ1UgOSgD2bClL0MJIqu58nOmIcBuXr62z1I= github.com/ebitengine/purego v0.8.2/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= -github.com/edgebitio/nitro-enclaves-sdk-go v1.0.0 h1:PkHLQAsU3gMDb5Q+3KJeSPOKXFgNNcn0y+/jBYg1cQg= -github.com/edgebitio/nitro-enclaves-sdk-go v1.0.0/go.mod h1:hDAX5hYfgVR/TzewAODjCifHo3urkCACUnsMeFDkt2Y= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= @@ -145,6 +145,8 @@ github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9 github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4= github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc= github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A= +github.com/hdevalence/ed25519consensus v0.2.0 h1:37ICyZqdyj0lAZ8P4D1d1id3HqbbG1N3iBb1Tb4rdcU= +github.com/hdevalence/ed25519consensus v0.2.0/go.mod h1:w3BHWjwJbFU29IRHL1Iqkw3sus+7FctEyM4RqDxYNzo= github.com/hf/nsm v0.0.0-20220930140112-cd181bd646b9 h1:pU32bJGmZwF4WXb9Yaz0T8vHDtIPVxqDOdmYdwTQPqw= github.com/hf/nsm v0.0.0-20220930140112-cd181bd646b9/go.mod h1:MJsac5D0fKcNWfriUERtln6segcGfD6Nu0V5uGBbPf8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -277,20 +279,20 @@ go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.4 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0/go.mod h1:Mjt1i1INqiaoZOMGR1RIUJN+i3ChKoFRqzrRQhlkbs0= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= -go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= -go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 h1:GqRJVj7UmLjCVyVJ3ZFLdPRmhDUp2zFmQe3RHIOsw24= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0/go.mod h1:ri3aaHSmCTVYu2AWv44YMauwAQc0aqI9gHKIcSbI1pU= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0 h1:lwI4Dc5leUqENgGuQImwLo4WnuXFPetmPpkLi2IrX54= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0/go.mod h1:Kz/oCE7z5wuyhPxsXDuaPteSWqjSBD5YaSdbxZYGbGk= -go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= -go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= -go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= -go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= -go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= -go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= -go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= -go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= go.opentelemetry.io/proto/otlp v1.7.1 h1:gTOMpGDb0WTBOP8JaO72iL3auEZhVmAQg4ipjOVAtj4= go.opentelemetry.io/proto/otlp v1.7.1/go.mod h1:b2rVh6rfI/s2pHWNlB7ILJcRALpcNDzKhACevjI+ZnE= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= @@ -315,21 +317,21 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk= -golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc= +golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= +golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= -golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= +golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= +golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= -golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -338,13 +340,13 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= -golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= +golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= -golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= +golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= +golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -360,14 +362,14 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= -gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= -gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= -google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 h1:fCvbg86sFXwdrl5LgVcTEvNC+2txB5mgROGmRL5mrls= -google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:+rXWjjaukWZun3mLfjmVnQi18E1AsFbDN9QdJ5YXLto= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 h1:gRkg/vSppuSQoDjxyiGfN4Upv/h/DQmIR10ZU8dh4Ww= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= -google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE= -google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:Kjn0N0tCrDgiAFW+lGO4JZ3ck44CehvJQMAwj9QF0G8= +google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.83.1 h1:HIO0+BEtBP6soyqvqC8sNUjZ7bTs+0hFQuFF+RAy++Y= +google.golang.org/grpc v1.83.1/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= diff --git a/internal/app/app.go b/internal/app/app.go index 106e967..aa0e435 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -16,14 +16,13 @@ package app import ( "context" - "fmt" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/circlefin/arc-remote-signer/internal/app/metrics" - "github.com/circlefin/arc-remote-signer/internal/app/provider/awskms" enclaveProvider "github.com/circlefin/arc-remote-signer/internal/app/provider/enclave" "github.com/circlefin/arc-remote-signer/internal/app/provider/secrets" "github.com/circlefin/arc-remote-signer/internal/app/public" - "github.com/circlefin/arc-remote-signer/internal/app/service/signer" + signerv1 "github.com/circlefin/arc-remote-signer/internal/app/service/signer/v1" "github.com/circlefin/arc-remote-signer/internal/common/lifecycle" "github.com/circlefin/arc-remote-signer/internal/common/logging" "github.com/circlefin/arc-remote-signer/internal/common/metric" @@ -50,6 +49,7 @@ func getLogger() *logging.Logger { // Run the application. func Run(cfg *Config) { + applyBuildPolicy(cfg) ctx := context.Background() logger := getLogger() @@ -81,7 +81,7 @@ func Run(cfg *Config) { } logger.Info(ctx, "initializing the providers...", nil) - enclavePvd, conn, err := enclaveProvider.New(cfg.Provider.Enclave) + enclavePvd, conn, err := newRuntimeEnclave(cfg.Provider.Enclave) if err != nil { logger.ErrorErr(ctx, "failed to initialize the enclave provider", err, nil) panic(err) @@ -92,28 +92,25 @@ func Run(cfg *Config) { } }() - logger.Info(ctx, "getting the attestation document from the enclave...", nil) - attestationDocument, err := getAttestationDocument(ctx, enclavePvd, cfg) - if err != nil { - logger.ErrorErr(ctx, "failed to get attestation document", err, nil) - panic(err) - } - logger.Info(ctx, "initializing the AWS providers...", nil) - awsConfig, err := retrieveAWSConfig(ctx, cfg, getLogger()) + awsConfig, err := loadRuntimeAWSConfig(ctx, cfg, getLogger()) if err != nil { logger.ErrorErr(ctx, "failed to retrieve the AWS config", err, nil) panic(err) } + secretPvd := secrets.New(awsConfig) - awskmsPvd, err := awskms.New(ctx, cfg.Provider.AWSKMS, awsConfig, attestationDocument) - if err != nil { - logger.ErrorErr(ctx, "failed to initialize the aws kms provider", err, nil) - panic(err) - } + initializeKey := newInitializeKeyFunc( + cfg.Provider.Enclave, + enclavePvd, + awsConfig, + cfg.Provider.AWSKMS.Arns, + cfg.Provider.AWSKMS.Localstack.Enabled, + logger, + ) logger.Info(ctx, "initializing the services...", nil) - signerSvc, err := signer.New(ctx, cfg.Provider.Enclave.NitroEnclave.Enabled, cfg.Service.Signer, secretPvd, enclavePvd, awskmsPvd) + signerSvc, err := signerv1.New(ctx, cfg.Service.Signer, secretPvd, enclavePvd, initializeKey) if err != nil { logger.ErrorErr(ctx, "failed to initialize the signer service", err, nil) panic(err) @@ -153,6 +150,33 @@ func Run(cfg *Config) { lc.Run() } +func newInitializeKeyFunc( + cfg *enclaveProvider.ProviderConfig, + enclavePvd pb.EnclaveServiceClient, + awsConfig aws.Config, + kmsKeyArns []string, + kmsLocalstackEnabled bool, + logger *logging.Logger, +) func(context.Context, *pb.SecretEnvelope, pb.Algorithm) (*pb.InitializeResponse, error) { + return func( + ctx context.Context, + existingKey *pb.SecretEnvelope, + generateNew pb.Algorithm, + ) (*pb.InitializeResponse, error) { + startupCtx, cancel := context.WithTimeout(ctx, cfg.StartupTimeout()) + defer cancel() + return initializeEnclave( + startupCtx, + enclavePvd, + awsConfig, + kmsKeyArns, + kmsLocalstackEnabled, + enclaveKeySource{existingKey: existingKey, generateNew: generateNew}, + logger, + ) + } +} + func initializeMetricsProviders(ctx context.Context, metricsCfg *metric.Config) metricProviders { if metricsCfg == nil { panic("metrics config unavailable") @@ -175,14 +199,3 @@ func initializeMetricsProviders(ctx context.Context, metricsCfg *metric.Config) prometheus: prometheusProvider, } } - -func getAttestationDocument(ctx context.Context, enclavePvd pb.EnclaveServiceClient, cfg *Config) ([]byte, error) { - if cfg.Provider.Enclave.NitroEnclave.Enabled { - resp, err := enclavePvd.GetAttestation(ctx, &pb.GetAttestationRequest{}) - if err != nil { - return nil, fmt.Errorf("failed to get attestation document: %w", err) - } - return resp.AttestationDocument, nil - } - return nil, nil -} diff --git a/internal/app/app_test.go b/internal/app/app_test.go new file mode 100644 index 0000000..8ed9333 --- /dev/null +++ b/internal/app/app_test.go @@ -0,0 +1,52 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package app + +import ( + "context" + "testing" + "time" + + enclaveProvider "github.com/circlefin/arc-remote-signer/internal/app/provider/enclave" + "github.com/circlefin/arc-remote-signer/internal/common/logging" + "github.com/circlefin/arc-remote-signer/proto/pb" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" +) + +func TestNewInitializeKeyFunc_AppliesTotalTimeout(t *testing.T) { + ctrl := gomock.NewController(t) + enclavePvd := enclaveProvider.NewMockEnclaveServiceClient(ctrl) + enclavePvd.EXPECT(). + Initialize(gomock.Any(), gomock.Any()). + DoAndReturn(func(ctx context.Context, _ *pb.InitializeRequest, _ ...any) (*pb.InitializeResponse, error) { + deadline, ok := ctx.Deadline() + require.True(t, ok) + require.WithinDuration(t, time.Now().Add(time.Hour), deadline, time.Second) + return &pb.InitializeResponse{}, nil + }) + + initializeKey := newInitializeKeyFunc( + &enclaveProvider.ProviderConfig{StartupTimeoutMS: int(time.Hour.Milliseconds())}, + enclavePvd, + newTestAwsCfg(t), + testArns, + false, + logging.Get("test"), + ) + resp, err := initializeKey(context.Background(), nil, pb.Algorithm_ALGORITHM_ED25519) + + require.NoError(t, err) + require.NotNil(t, resp) +} diff --git a/internal/app/config.go b/internal/app/config.go index 9b25847..fe82e4d 100644 --- a/internal/app/config.go +++ b/internal/app/config.go @@ -14,19 +14,11 @@ package app import ( - "context" - "errors" - - "github.com/aws/aws-sdk-go-v2/aws" - awsSdkConfig "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/credentials" - "github.com/circlefin/arc-remote-signer/internal/app/provider/awskms" enclave "github.com/circlefin/arc-remote-signer/internal/app/provider/enclave" "github.com/circlefin/arc-remote-signer/internal/app/provider/secrets" - "github.com/circlefin/arc-remote-signer/internal/app/service/signer" + signerv1 "github.com/circlefin/arc-remote-signer/internal/app/service/signer/v1" "github.com/circlefin/arc-remote-signer/internal/common/config" grpcServer "github.com/circlefin/arc-remote-signer/internal/common/grpc/server" - "github.com/circlefin/arc-remote-signer/internal/common/logging" "github.com/circlefin/arc-remote-signer/internal/common/metric" "github.com/circlefin/arc-remote-signer/internal/common/telemetry" ) @@ -68,20 +60,34 @@ func (c *Config) GetName() string { // ServiceConfig provides configuration for the service. type ServiceConfig struct { - Signer *signer.Config + Signer *signerv1.Config } // ProviderConfig lists the providers' config. type ProviderConfig struct { Secrets *secrets.Config Enclave *enclave.ProviderConfig - AWSKMS *awskms.Config + AWSKMS *AWSKMSConfig +} + +// AWSKMSConfig contains the KMS settings that the host sends to the enclave. +type AWSKMSConfig struct { + Localstack *AWSKMSLocalstackConfig `json:"localstack" mapstructure:"localstack"` + Arns []string `json:"arns" mapstructure:"arns"` +} + +// AWSKMSLocalstackConfig selects LocalStack as the KMS backend. +type AWSKMSLocalstackConfig struct { + Enabled bool `json:"enabled" mapstructure:"enabled"` } -// NewConfig returns a new Config with default dev (non-prod) environment. -func NewConfig() *Config { +func newConfig( + baseConfig *config.BaseConfig, + secretsConfig *secrets.Config, + awsKMSConfig *AWSKMSConfig, +) *Config { return &Config{ - BaseConfig: config.NewBaseConfig(), + BaseConfig: baseConfig, Public: &PublicConfig{ Server: &grpcServer.Config{ Host: "0.0.0.0", @@ -97,46 +103,12 @@ func NewConfig() *Config { Metrics: metric.NewConfig(), Telemetry: telemetry.NewConfig(), Service: &ServiceConfig{ - Signer: signer.NewConfig(), + Signer: signerv1.NewConfig(), }, Provider: &ProviderConfig{ - Secrets: secrets.NewConfig(), + Secrets: secretsConfig, Enclave: enclave.NewProviderConfig(), - AWSKMS: awskms.NewProviderConfig(), + AWSKMS: awsKMSConfig, }, } } - -// MergeAwsConfigWithLocalstack will generate final awsConfig according endpoint and region in secrets config. -// If LocalstackEndpoint is empty in config, it will use default config of aws. -func MergeAwsConfigWithLocalstack(cfg *Config) (aws.Config, error) { - awsEndpoint := cfg.Provider.Secrets.Localstack.Endpoint - awsRegion := cfg.Provider.Secrets.Localstack.Region - - configOptions := []func(*awsSdkConfig.LoadOptions) error{} - if awsRegion != "" { - configOptions = append(configOptions, awsSdkConfig.WithRegion(awsRegion)) - } - if awsEndpoint != "" { - configOptions = append(configOptions, awsSdkConfig.WithBaseEndpoint(awsEndpoint)) - // Localstack does not validate credentials; inject static credentials to prevent - // the SDK from attempting EC2 IMDS discovery in environments without AWS access. - configOptions = append(configOptions, awsSdkConfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider("test", "test", ""))) - } - - return awsSdkConfig.LoadDefaultConfig(context.Background(), configOptions...) -} - -func retrieveAWSConfig(ctx context.Context, cfg *Config, logger *logging.Logger) (aws.Config, error) { - if cfg.BaseConfig == nil || cfg.Provider.Secrets == nil || cfg.Provider.Secrets.Localstack == nil { - return aws.Config{}, errors.New("secrets config unavailable") - } - - if (cfg.Env == config.Dev || cfg.Env == config.QA) && - (cfg.Provider.Secrets.Localstack.Enabled && cfg.Provider.Secrets.Localstack.Endpoint != "") { - logger.Info(ctx, "Using localstack aws config...", nil) - return MergeAwsConfigWithLocalstack(cfg) - } - logger.Info(ctx, "Using standard aws config...", nil) - return awsSdkConfig.LoadDefaultConfig(ctx) -} diff --git a/internal/app/config_test.go b/internal/app/config_test.go index 7dc0c6f..5e6cbe8 100644 --- a/internal/app/config_test.go +++ b/internal/app/config_test.go @@ -11,6 +11,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !prod + package app import ( @@ -185,6 +187,7 @@ func TestNewConfig(t *testing.T) { // Check public server defaults require.Equal(t, "0.0.0.0", cfg.Public.Server.Host) require.Equal(t, 8080, cfg.Public.Server.Port) + require.False(t, cfg.Public.Server.Reflection.Enabled) // Check profiler defaults require.False(t, cfg.Profiler.Enabled) @@ -197,12 +200,16 @@ func TestNewConfig_TLSDefaultEnablesEnvBinding(t *testing.T) { // public.server.tls.* keys and APP_PUBLIC_SERVER_TLS_* env vars can bind. require.NotNil(t, cfg.Public.Server.TLS) require.False(t, cfg.Public.Server.TLS.Enabled) + require.False(t, cfg.Public.Server.TLS.ClientAuth.Enabled) + require.Empty(t, cfg.Public.Server.TLS.ClientAuth.CA) } func TestLoadConfig_TLSEnvOverride(t *testing.T) { t.Setenv("APP_PUBLIC_SERVER_TLS_ENABLED", "true") t.Setenv("APP_PUBLIC_SERVER_TLS_CERT", "/etc/tls/server.crt") t.Setenv("APP_PUBLIC_SERVER_TLS_KEY", "/etc/tls/server.key") + t.Setenv("APP_PUBLIC_SERVER_TLS_CLIENTAUTH_ENABLED", "true") + t.Setenv("APP_PUBLIC_SERVER_TLS_CLIENTAUTH_CA", "/etc/tls/client-ca.crt") cfg := NewConfig() config.LoadConfig(cfg, "") @@ -211,6 +218,26 @@ func TestLoadConfig_TLSEnvOverride(t *testing.T) { require.True(t, cfg.Public.Server.TLS.Enabled) require.Equal(t, "/etc/tls/server.crt", cfg.Public.Server.TLS.Cert) require.Equal(t, "/etc/tls/server.key", cfg.Public.Server.TLS.Key) + require.True(t, cfg.Public.Server.TLS.ClientAuth.Enabled) + require.Equal(t, "/etc/tls/client-ca.crt", cfg.Public.Server.TLS.ClientAuth.CA) +} + +func TestLoadConfig_ReflectionEnvOverride(t *testing.T) { + t.Setenv("APP_PUBLIC_SERVER_REFLECTION_ENABLED", "true") + + cfg := NewConfig() + config.LoadConfig(cfg, "") + + require.True(t, cfg.Public.Server.Reflection.Enabled) +} + +func TestLoadConfig_AWSKMSLocalstackEnvOverride(t *testing.T) { + t.Setenv("APP_PROVIDER_AWSKMS_LOCALSTACK_ENABLED", "false") + + cfg := NewConfig() + config.LoadConfig(cfg, "") + + require.False(t, cfg.Provider.AWSKMS.Localstack.Enabled) } func TestConfig_GetName(t *testing.T) { @@ -326,7 +353,7 @@ func TestRetrieveAWSConfig_ErrorCases(t *testing.T) { cfg.BaseConfig = nil return cfg }, - wantError: "secrets config unavailable", + wantError: "secrets configuration is unavailable", }, { name: "nil secrets config", @@ -335,7 +362,7 @@ func TestRetrieveAWSConfig_ErrorCases(t *testing.T) { cfg.Provider.Secrets = nil return cfg }, - wantError: "secrets config unavailable", + wantError: "secrets configuration is unavailable", }, { name: "nil localstack config", @@ -344,7 +371,7 @@ func TestRetrieveAWSConfig_ErrorCases(t *testing.T) { cfg.Provider.Secrets.Localstack = nil return cfg }, - wantError: "secrets config unavailable", + wantError: "secrets configuration is unavailable", }, } diff --git a/internal/app/enclaveinit.go b/internal/app/enclaveinit.go new file mode 100644 index 0000000..a38b24f --- /dev/null +++ b/internal/app/enclaveinit.go @@ -0,0 +1,137 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package app + +import ( + "context" + "fmt" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/circlefin/arc-remote-signer/internal/common/logging" + "github.com/circlefin/arc-remote-signer/proto/pb" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// initRetryAttempts is the total number of Initialize attempts. +const initRetryAttempts = 3 + +// initRetryBackoff is the fixed delay between Initialize retry attempts. +// Overridable via package-private var for tests so the retry table does +// not block test runs for several seconds. +var initRetryBackoff = 1 * time.Second + +type enclaveKeySource struct { + existingKey *pb.SecretEnvelope + generateNew pb.Algorithm +} + +// initializeEnclave runs the host-side enclave initialization flow. +// Normal production and development use the AWS default credential chain. +// When LocalStack is enabled with an endpoint, the host uses static placeholder +// credentials. +// The function sends the credentials to the enclave in the Initialize RPC. +// It retries transient failures for up to initRetryAttempts total attempts. +// It returns all errors to app.Run. The caller supplies a cancellable startup +// context with the total budget. +// +// Credentials and the request struct live only inside this function's +// stack frame and are not retained by the Service. Strings are +// proto-generated and cannot be explicitly zeroed; we rely on the +// frame going out of scope plus the runtime eventually reusing the +// memory. +func initializeEnclave( + ctx context.Context, + enclavePvd pb.EnclaveServiceClient, + awsCfg aws.Config, + kmsKeyArns []string, + kmsLocalstackEnabled bool, + keySource enclaveKeySource, + logger *logging.Logger, +) (*pb.InitializeResponse, error) { + if len(kmsKeyArns) == 0 { + return nil, fmt.Errorf("kmsKeyArns must not be empty") + } + if (keySource.existingKey == nil) == (keySource.generateNew == pb.Algorithm_ALGORITHM_UNSPECIFIED) { + return nil, fmt.Errorf("exactly one enclave key source is required") + } + + var lastErr error + invalidateCredentials := false + for attempt := 1; attempt <= initRetryAttempts; attempt++ { + if invalidateCredentials { + if cache, ok := awsCfg.Credentials.(interface{ Invalidate() }); ok { + cache.Invalidate() + } + } + creds, err := awsCfg.Credentials.Retrieve(ctx) + if err != nil { + return nil, fmt.Errorf("retrieve aws credentials: %w", err) + } + + req := &pb.InitializeRequest{ + Credentials: &pb.AwsCredentials{ + AccessKeyId: creds.AccessKeyID, + SecretAccessKey: creds.SecretAccessKey, + SessionToken: creds.SessionToken, + Region: awsCfg.Region, + }, + KmsKeyArns: kmsKeyArns, + KmsLocalstackEnabled: kmsLocalstackEnabled, + } + if keySource.existingKey != nil { + req.KeySource = &pb.InitializeRequest_ExistingKey{ExistingKey: keySource.existingKey} + } else { + req.KeySource = &pb.InitializeRequest_GenerateNew{GenerateNew: keySource.generateNew} + } + + resp, err := enclavePvd.Initialize(ctx, req) + if err == nil { + logger.Info(ctx, "enclave Initialize succeeded", logging.Entries{"attempt": attempt}) + return resp, nil + } + + code := status.Code(err) + switch code { + case codes.Unauthenticated, codes.Unavailable, codes.DeadlineExceeded: + // Retry DeadlineExceeded only while the caller's total startup + // budget permits another attempt. + lastErr = err + invalidateCredentials = code == codes.Unauthenticated + if attempt < initRetryAttempts { + logger.Info(ctx, "enclave Initialize returned retryable error, will retry", logging.Entries{ + "attempt": attempt, + "code": code.String(), + "error": err.Error(), + }) + } + case codes.PermissionDenied, codes.InvalidArgument, codes.FailedPrecondition: + return nil, fmt.Errorf("enclave Initialize rejected request (code=%s, non-retryable): %w", code, err) + default: + return nil, fmt.Errorf("enclave Initialize failed (code=%s): %w", code, err) + } + + if attempt < initRetryAttempts { + // Keep the backoff inside the cancellable startup budget. + select { + case <-ctx.Done(): + return nil, fmt.Errorf("enclave Initialize aborted during retry backoff: %w", ctx.Err()) + case <-time.After(initRetryBackoff): + } + } + } + + return nil, fmt.Errorf("enclave Initialize exhausted %d retry attempts: %w", initRetryAttempts, lastErr) +} diff --git a/internal/app/enclaveinit_test.go b/internal/app/enclaveinit_test.go new file mode 100644 index 0000000..1c427a4 --- /dev/null +++ b/internal/app/enclaveinit_test.go @@ -0,0 +1,434 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package app + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/credentials" + enclaveProvider "github.com/circlefin/arc-remote-signer/internal/app/provider/enclave" + "github.com/circlefin/arc-remote-signer/internal/common/logging" + "github.com/circlefin/arc-remote-signer/proto/pb" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// withFastBackoff temporarily zeroes the Initialize retry backoff so the +// retry table can exercise three attempts in milliseconds rather than +// seconds. Mutating package-level state is acceptable here because the +// app-level tests do not run in parallel. +func withFastBackoff(t *testing.T) { + t.Helper() + saved := initRetryBackoff + initRetryBackoff = 0 + t.Cleanup(func() { initRetryBackoff = saved }) +} + +func newTestAwsCfg(t *testing.T) aws.Config { + t.Helper() + return aws.Config{ + Region: "us-east-1", + Credentials: credentials.NewStaticCredentialsProvider( + "AKIA-test", + "secret", + "session", + ), + } +} + +var testArns = []string{"arn:aws:kms:us-east-1:000000000000:key/abc"} +var testKeySource = enclaveKeySource{generateNew: pb.Algorithm_ALGORITHM_ED25519} + +func TestInitializeEnclave_Success(t *testing.T) { + withFastBackoff(t) + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockClient := enclaveProvider.NewMockEnclaveServiceClient(ctrl) + mockClient.EXPECT(). + Initialize(gomock.Any(), gomock.Any()). + Return(&pb.InitializeResponse{}, nil). + Times(1) + + _, err := initializeEnclave(context.Background(), mockClient, newTestAwsCfg(t), testArns, false, testKeySource, logging.Get("test")) + require.NoError(t, err) +} + +func TestInitializeEnclave_RequestShape(t *testing.T) { + withFastBackoff(t) + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + var captured *pb.InitializeRequest + mockClient := enclaveProvider.NewMockEnclaveServiceClient(ctrl) + mockClient.EXPECT(). + Initialize(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, req *pb.InitializeRequest, _ ...any) (*pb.InitializeResponse, error) { + captured = req + return &pb.InitializeResponse{}, nil + }). + Times(1) + + _, err := initializeEnclave(context.Background(), mockClient, newTestAwsCfg(t), testArns, true, testKeySource, logging.Get("test")) + require.NoError(t, err) + require.NotNil(t, captured) + require.NotNil(t, captured.Credentials) + require.Equal(t, "AKIA-test", captured.Credentials.AccessKeyId) + require.Equal(t, "secret", captured.Credentials.SecretAccessKey) + require.Equal(t, "session", captured.Credentials.SessionToken) + require.Equal(t, "us-east-1", captured.Credentials.Region) + require.Equal(t, testArns, captured.KmsKeyArns) + require.True(t, captured.KmsLocalstackEnabled) + require.Equal(t, pb.Algorithm_ALGORITHM_ED25519, captured.GetGenerateNew()) +} + +func TestInitializeEnclave_ExistingKeyRequestShape(t *testing.T) { + withFastBackoff(t) + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + existingKey := &pb.SecretEnvelope{ + Algorithm: pb.Algorithm_ALGORITHM_ED25519, + KmsEncryptedDataKey: []byte("kms-key"), + EncryptedPrivateKey: []byte("private-key"), + Nonce: []byte("nonce"), + } + mockClient := enclaveProvider.NewMockEnclaveServiceClient(ctrl) + mockClient.EXPECT(). + Initialize(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, req *pb.InitializeRequest, _ ...any) (*pb.InitializeResponse, error) { + require.Equal(t, existingKey, req.GetExistingKey()) + require.Equal(t, pb.Algorithm_ALGORITHM_UNSPECIFIED, req.GetGenerateNew()) + return &pb.InitializeResponse{PublicKey: []byte("public-key")}, nil + }) + + resp, err := initializeEnclave( + context.Background(), + mockClient, + newTestAwsCfg(t), + testArns, + false, + enclaveKeySource{existingKey: existingKey}, + logging.Get("test"), + ) + + require.NoError(t, err) + require.Equal(t, []byte("public-key"), resp.PublicKey) +} + +func TestInitializeEnclave_RetriesUnavailableThenSucceeds(t *testing.T) { + withFastBackoff(t) + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + unavailErr := status.Error(codes.Unavailable, "enclave not ready") + mockClient := enclaveProvider.NewMockEnclaveServiceClient(ctrl) + gomock.InOrder( + mockClient.EXPECT().Initialize(gomock.Any(), gomock.Any()).Return(nil, unavailErr), + mockClient.EXPECT().Initialize(gomock.Any(), gomock.Any()).Return(nil, unavailErr), + mockClient.EXPECT().Initialize(gomock.Any(), gomock.Any()).Return(&pb.InitializeResponse{}, nil), + ) + + _, err := initializeEnclave(context.Background(), mockClient, newTestAwsCfg(t), testArns, false, testKeySource, logging.Get("test")) + require.NoError(t, err) +} + +func TestInitializeEnclave_RetrievesCredentialsForEachAttempt(t *testing.T) { + withFastBackoff(t) + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + credentialsProvider := &rotatingCredentialProvider{ + values: []aws.Credentials{ + {AccessKeyID: "AKIA-first", SecretAccessKey: "secret-1", SessionToken: "session-1"}, + {AccessKeyID: "AKIA-second", SecretAccessKey: "secret-2", SessionToken: "session-2"}, + }, + } + var accessKeyIDs []string + mockClient := enclaveProvider.NewMockEnclaveServiceClient(ctrl) + gomock.InOrder( + mockClient.EXPECT(). + Initialize(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, req *pb.InitializeRequest, _ ...any) (*pb.InitializeResponse, error) { + accessKeyIDs = append(accessKeyIDs, req.GetCredentials().GetAccessKeyId()) + return nil, status.Error(codes.Unavailable, "try again") + }), + mockClient.EXPECT(). + Initialize(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, req *pb.InitializeRequest, _ ...any) (*pb.InitializeResponse, error) { + accessKeyIDs = append(accessKeyIDs, req.GetCredentials().GetAccessKeyId()) + return &pb.InitializeResponse{}, nil + }), + ) + + _, err := initializeEnclave( + context.Background(), + mockClient, + aws.Config{Region: "us-east-1", Credentials: credentialsProvider}, + testArns, + false, + testKeySource, + logging.Get("test"), + ) + + require.NoError(t, err) + require.Equal(t, []string{"AKIA-first", "AKIA-second"}, accessKeyIDs) + require.Equal(t, 2, credentialsProvider.retrieveCalls) +} + +func TestInitializeEnclave_InvalidatesCachedCredentialsAfterUnauthenticated(t *testing.T) { + withFastBackoff(t) + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + credentialsProvider := &rotatingCredentialProvider{ + values: []aws.Credentials{ + {AccessKeyID: "AKIA-expired", SecretAccessKey: "secret-1", SessionToken: "session-1"}, + {AccessKeyID: "AKIA-refreshed", SecretAccessKey: "secret-2", SessionToken: "session-2"}, + }, + } + cachedCredentials := aws.NewCredentialsCache(credentialsProvider) + mockClient := enclaveProvider.NewMockEnclaveServiceClient(ctrl) + gomock.InOrder( + mockClient.EXPECT(). + Initialize(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, req *pb.InitializeRequest, _ ...any) (*pb.InitializeResponse, error) { + require.Equal(t, "AKIA-expired", req.GetCredentials().GetAccessKeyId()) + return nil, status.Error(codes.Unauthenticated, "expired credentials") + }), + mockClient.EXPECT(). + Initialize(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, req *pb.InitializeRequest, _ ...any) (*pb.InitializeResponse, error) { + require.Equal(t, "AKIA-refreshed", req.GetCredentials().GetAccessKeyId()) + return &pb.InitializeResponse{}, nil + }), + ) + + _, err := initializeEnclave( + context.Background(), + mockClient, + aws.Config{Region: "us-east-1", Credentials: cachedCredentials}, + testArns, + false, + testKeySource, + logging.Get("test"), + ) + + require.NoError(t, err) + require.Equal(t, 2, credentialsProvider.retrieveCalls) +} + +func TestInitializeEnclave_ExhaustsRetriesOnUnavailable(t *testing.T) { + withFastBackoff(t) + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + unavailErr := status.Error(codes.Unavailable, "enclave still not ready") + mockClient := enclaveProvider.NewMockEnclaveServiceClient(ctrl) + mockClient.EXPECT(). + Initialize(gomock.Any(), gomock.Any()). + Return(nil, unavailErr). + Times(initRetryAttempts) + + _, err := initializeEnclave(context.Background(), mockClient, newTestAwsCfg(t), testArns, false, testKeySource, logging.Get("test")) + require.Error(t, err) + require.ErrorIs(t, err, unavailErr) + require.Contains(t, err.Error(), "exhausted") +} + +func TestInitializeEnclave_RetriesDeadlineExceededThenSucceeds(t *testing.T) { + withFastBackoff(t) + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + deadlineErr := status.Error(codes.DeadlineExceeded, "deadline exceeded during enclave cold start") + mockClient := enclaveProvider.NewMockEnclaveServiceClient(ctrl) + gomock.InOrder( + mockClient.EXPECT().Initialize(gomock.Any(), gomock.Any()).Return(nil, deadlineErr), + mockClient.EXPECT().Initialize(gomock.Any(), gomock.Any()).Return(&pb.InitializeResponse{}, nil), + ) + + _, err := initializeEnclave(context.Background(), mockClient, newTestAwsCfg(t), testArns, false, testKeySource, logging.Get("test")) + require.NoError(t, err) +} + +func TestInitializeEnclave_EmptyKmsKeyArnsRejected(t *testing.T) { + withFastBackoff(t) + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + // Mock not expected to be called β€” guard fires before any RPC. + mockClient := enclaveProvider.NewMockEnclaveServiceClient(ctrl) + + _, err := initializeEnclave(context.Background(), mockClient, newTestAwsCfg(t), nil, false, testKeySource, logging.Get("test")) + require.Error(t, err) + require.Contains(t, err.Error(), "kmsKeyArns must not be empty") +} + +func TestInitializeEnclave_NonRetryablePermissionDenied(t *testing.T) { + withFastBackoff(t) + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + deniedErr := status.Error(codes.PermissionDenied, "kms:GenerateDataKey not allowed") + mockClient := enclaveProvider.NewMockEnclaveServiceClient(ctrl) + mockClient.EXPECT(). + Initialize(gomock.Any(), gomock.Any()). + Return(nil, deniedErr). + Times(1) + + _, err := initializeEnclave(context.Background(), mockClient, newTestAwsCfg(t), testArns, false, testKeySource, logging.Get("test")) + require.Error(t, err) + require.ErrorIs(t, err, deniedErr) + require.Contains(t, err.Error(), "non-retryable") +} + +func TestInitializeEnclave_NonRetryableInvalidArgument(t *testing.T) { + withFastBackoff(t) + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + invalidErr := status.Error(codes.InvalidArgument, "session_token: value length must be at least 1 characters") + mockClient := enclaveProvider.NewMockEnclaveServiceClient(ctrl) + mockClient.EXPECT(). + Initialize(gomock.Any(), gomock.Any()). + Return(nil, invalidErr). + Times(1) + + _, err := initializeEnclave(context.Background(), mockClient, newTestAwsCfg(t), testArns, false, testKeySource, logging.Get("test")) + require.Error(t, err) + require.ErrorIs(t, err, invalidErr) + require.Contains(t, err.Error(), "non-retryable") +} + +func TestInitializeEnclave_NonRetryableInternal(t *testing.T) { + withFastBackoff(t) + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + internalErr := status.Error(codes.Internal, "enclave Initialize crashed") + mockClient := enclaveProvider.NewMockEnclaveServiceClient(ctrl) + mockClient.EXPECT(). + Initialize(gomock.Any(), gomock.Any()). + Return(nil, internalErr). + Times(1) + + // codes.Internal is not in the retryable list and not in the + // explicit non-retryable-with-distinct-message list, so it falls + // through to the default branch β€” single attempt, surface error. + _, err := initializeEnclave(context.Background(), mockClient, newTestAwsCfg(t), testArns, false, testKeySource, logging.Get("test")) + require.Error(t, err) + require.ErrorIs(t, err, internalErr) + require.Contains(t, err.Error(), "code=Internal") +} + +// TestInitializeEnclave_CtxCancelledDuringBackoff exercises the +// ctx-aware sleep in the retry loop: with a non-zero backoff and a +// pre-cancelled ctx, the function must abort during the select instead +// of busy-looping or sleeping the full backoff. Uses a real +// initRetryBackoff so cancellation must beat the timer. +func TestInitializeEnclave_CtxCancelledDuringBackoff(t *testing.T) { + saved := initRetryBackoff + initRetryBackoff = 5 * time.Second // would dominate test duration if sleep was not ctx-aware + t.Cleanup(func() { initRetryBackoff = saved }) + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + unavailErr := status.Error(codes.Unavailable, "enclave not ready") + mockClient := enclaveProvider.NewMockEnclaveServiceClient(ctrl) + // First Initialize attempt returns Unavailable, triggering the backoff. + // Second attempt is never reached because ctx cancels during backoff. + mockClient.EXPECT().Initialize(gomock.Any(), gomock.Any()).Return(nil, unavailErr).Times(1) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() // pre-cancelled so the select fires Done immediately on the first sleep + + start := time.Now() + _, err := initializeEnclave(ctx, mockClient, newTestAwsCfg(t), testArns, false, testKeySource, logging.Get("test")) + elapsed := time.Since(start) + + require.Error(t, err) + require.ErrorIs(t, err, context.Canceled) + require.Less(t, elapsed, initRetryBackoff, + "ctx-aware sleep must abort before the timer fires") +} + +func TestInitializeEnclave_TotalTimeoutEnforcedAcrossRetries(t *testing.T) { + saved := initRetryBackoff + initRetryBackoff = 5 * time.Second + t.Cleanup(func() { initRetryBackoff = saved }) + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + unavailErr := status.Error(codes.Unavailable, "enclave not ready") + mockClient := enclaveProvider.NewMockEnclaveServiceClient(ctrl) + mockClient.EXPECT().Initialize(gomock.Any(), gomock.Any()).Return(nil, unavailErr).Times(1) + + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + start := time.Now() + _, err := initializeEnclave(ctx, mockClient, newTestAwsCfg(t), testArns, false, testKeySource, logging.Get("test")) + + require.Error(t, err) + require.ErrorIs(t, err, context.DeadlineExceeded) + require.Less(t, time.Since(start), initRetryBackoff) +} + +func TestInitializeEnclave_CredentialsRetrieveError(t *testing.T) { + withFastBackoff(t) + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + // Mock not expected to be called β€” credential retrieval fails first. + mockClient := enclaveProvider.NewMockEnclaveServiceClient(ctrl) + + awsCfg := aws.Config{ + Region: "us-east-1", + Credentials: failingCredentialProvider{}, + } + + _, err := initializeEnclave(context.Background(), mockClient, awsCfg, testArns, false, testKeySource, logging.Get("test")) + require.Error(t, err) + require.Contains(t, err.Error(), "retrieve aws credentials") +} + +// failingCredentialProvider is used by TestInitializeEnclave_CredentialsRetrieveError +// to force the early-exit path when the AWS SDK cannot mint credentials. +type failingCredentialProvider struct{} + +func (failingCredentialProvider) Retrieve(_ context.Context) (aws.Credentials, error) { + return aws.Credentials{}, errors.New("imds unreachable") +} + +type rotatingCredentialProvider struct { + values []aws.Credentials + retrieveCalls int +} + +func (p *rotatingCredentialProvider) Retrieve(_ context.Context) (aws.Credentials, error) { + index := p.retrieveCalls + p.retrieveCalls++ + if index >= len(p.values) { + index = len(p.values) - 1 + } + return p.values[index], nil +} diff --git a/internal/app/provider/awskms/awskms_test.go b/internal/app/provider/awskms/awskms_test.go deleted file mode 100644 index a008f7e..0000000 --- a/internal/app/provider/awskms/awskms_test.go +++ /dev/null @@ -1,160 +0,0 @@ -// Copyright (c) 2026, Circle Internet Group, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package awskms - -import ( - "context" - "errors" - "testing" - - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/stretchr/testify/require" -) - -func newTestClient(t *testing.T, arn string) *client { - t.Helper() - return &client{arn: arn} -} - -func TestCall_FirstClientSuccess(t *testing.T) { - first := newTestClient(t, "arn:aws:kms:us-east-1:123456789012:key/first") - p := &provider{ - clients: []*client{first}, - } - - err := p.call(func(gotClient *client) error { - require.Same(t, first, gotClient) - return nil - }) - - require.NoError(t, err) - require.Len(t, p.clients, 1) - require.Same(t, first, p.clients[0]) -} - -func TestCall_PartialFailureFallback(t *testing.T) { - first := newTestClient(t, "arn:aws:kms:us-east-1:123456789012:key/first") - second := newTestClient(t, "arn:aws:kms:us-west-2:123456789012:key/second") - p := &provider{ - clients: []*client{first, second}, - } - - firstErr := errors.New("first client failed") - var callOrder []*client - err := p.call(func(gotClient *client) error { - callOrder = append(callOrder, gotClient) - if gotClient == first { - return firstErr - } - require.Same(t, second, gotClient) - return nil - }) - - require.NoError(t, err) - require.Equal(t, []*client{first, second}, callOrder) - require.Len(t, p.clients, 2) - require.Same(t, second, p.clients[0]) - require.Same(t, first, p.clients[1]) -} - -func TestCall_AllClientsFail(t *testing.T) { - first := newTestClient(t, "arn:aws:kms:us-east-1:123456789012:key/first") - second := newTestClient(t, "arn:aws:kms:us-west-2:123456789012:key/second") - p := &provider{ - clients: []*client{first, second}, - } - - firstErr := errors.New("first client failed") - lastErr := errors.New("last client failed") - err := p.call(func(gotClient *client) error { - if gotClient == first { - return firstErr - } - require.Same(t, second, gotClient) - return lastErr - }) - - require.Error(t, err) - require.ErrorContains(t, err, "all multi-region keys are invalid") - require.ErrorIs(t, err, lastErr) -} - -func TestDecrypt_EmptyCiphertext(t *testing.T) { - tests := []struct { - name string - ciphertext []byte - }{ - { - name: "nil ciphertext", - ciphertext: nil, - }, - { - name: "empty ciphertext", - ciphertext: []byte{}, - }, - } - - for _, tt := range tests { - tt := tt - t.Run(tt.name, func(t *testing.T) { - p := &provider{ - clients: []*client{ - newTestClient(t, "arn:aws:kms:us-east-1:123456789012:key/test"), - }, - } - - var ( - plaintext []byte - ciphertextForRecipient []byte - err error - ) - require.NotPanics(t, func() { - plaintext, ciphertextForRecipient, err = p.Decrypt(context.Background(), tt.ciphertext) - }) - - require.EqualError(t, err, "invalid ciphertext") - require.Nil(t, plaintext) - require.Nil(t, ciphertextForRecipient) - }) - } -} - -func TestExtractRegionFromKmsKeyArn_ValidArn(t *testing.T) { - region, err := extractRegionFromKmsKeyArn("arn:aws:kms:us-east-1:123456789012:key/1234abcd") - - require.NoError(t, err) - require.Equal(t, "us-east-1", region) -} - -func TestExtractRegionFromKmsKeyArn_MalformedArn(t *testing.T) { - region, err := extractRegionFromKmsKeyArn("not-an-arn") - - require.Error(t, err) - require.Empty(t, region) -} - -func TestInitClients_EmptyArns(t *testing.T) { - clients, err := initClients(aws.Config{}, nil, 0) - - require.Error(t, err) - require.Nil(t, clients) -} - -func TestInitClients_InvalidArn(t *testing.T) { - clients, err := initClients(aws.Config{}, []string{"bad-arn"}, 0) - - require.Error(t, err) - require.ErrorContains(t, err, "invalid arn") - require.Nil(t, clients) -} diff --git a/internal/app/provider/awskms/config.go b/internal/app/provider/awskms/config.go deleted file mode 100644 index 2465945..0000000 --- a/internal/app/provider/awskms/config.go +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright (c) 2026, Circle Internet Group, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package awskms - -// Config ... -type Config struct { - Localstack *LocalstackConfig - Arns []string `json:"arns"` // example: arn:aws:kms:us-east-2:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab - ConnectTimeout int `json:"connectTimeout"` -} - -// LocalstackConfig represents the configuration for the localstack secrets provider. -type LocalstackConfig struct { - Enabled bool - Endpoint string - Region string -} - -// NewProviderConfig returns a new Config with defaults. -func NewProviderConfig() *Config { - return &Config{ - Localstack: &LocalstackConfig{ - Enabled: true, - Endpoint: "http://localhost:4566", - Region: "us-east-1", - }, - Arns: []string{"arn:aws:kms:us-east-1:000000000000:alias/dev-multi-region-crypto", "arn:aws:kms:us-west-2:000000000000:alias/dev-multi-region-crypto"}, - ConnectTimeout: 1500, - } -} diff --git a/internal/app/provider/enclave/config.go b/internal/app/provider/enclave/config.go index 759be26..e53708c 100644 --- a/internal/app/provider/enclave/config.go +++ b/internal/app/provider/enclave/config.go @@ -14,6 +14,8 @@ package enclave import ( + "time" + "github.com/circlefin/arc-remote-signer/internal/common/grpc/client" ) @@ -22,34 +24,71 @@ const providerName = "enclave" // enclaveDefaultURL is the default dev URL for the enclave. const enclaveDefaultURL = "localhost:10350" -// enclaveDefaultCID is the default CID for the enclave. -// https://github.com/circlefin/arc-remote-signer/blob/master/docker/run.sh#L6 +// enclaveDefaultCID is the development default. The production policy pins it. const enclaveDefaultCID = 16 +const ( + defaultSignMessageTimeoutMS = 500 + defaultStartupTimeoutMS = 30000 +) + // ProviderConfig contains configuration for the enclave gRPC provider. type ProviderConfig struct { - NitroEnclave *NitroEnclave - Client *client.Config + NitroEnclave *NitroEnclave `json:"nitroEnclave" mapstructure:"nitroEnclave"` + Client *client.Config `json:"client" mapstructure:"client"` + StartupTimeoutMS int `json:"startupTimeoutMS" mapstructure:"startupTimeoutMS"` } // NitroEnclave contains config for the Nitro Enclave. type NitroEnclave struct { Enabled bool - // CID is the context ID of the enclave (typically 16 for first enclave). + // CID is the context ID of the enclave. CID uint32 // Port is the VSOCK port the enclave listens on. Port uint32 } +// DefaultMethodTimeouts returns default method timeouts for enclave client RPCs. +func DefaultMethodTimeouts() map[string]client.MethodConfig { + return map[string]client.MethodConfig{ + "initialize": { + TimeoutMS: defaultStartupTimeoutMS, + MaxAttempts: 1, + }, + "generatekey": { + TimeoutMS: defaultStartupTimeoutMS, + MaxAttempts: 1, + }, + "getpublickey": { + TimeoutMS: defaultStartupTimeoutMS, + }, + "signmessage": { + TimeoutMS: defaultSignMessageTimeoutMS, + MaxAttempts: 1, + }, + } +} + +// StartupTimeout returns the total budget for host-side enclave initialization. +func (c *ProviderConfig) StartupTimeout() time.Duration { + if c != nil && c.StartupTimeoutMS > 0 { + return time.Duration(c.StartupTimeoutMS) * time.Millisecond + } + return time.Duration(defaultStartupTimeoutMS) * time.Millisecond +} + // NewProviderConfig provides a new provider config with defaults. func NewProviderConfig() *ProviderConfig { cfg := client.NewClientConfig(providerName, enclaveDefaultURL) + cfg.RequestTimeoutMS = defaultSignMessageTimeoutMS + cfg.Methods = DefaultMethodTimeouts() return &ProviderConfig{ NitroEnclave: &NitroEnclave{ Enabled: false, CID: enclaveDefaultCID, Port: 10350, }, - Client: cfg, + Client: cfg, + StartupTimeoutMS: defaultStartupTimeoutMS, } } diff --git a/internal/app/provider/enclave/config_test.go b/internal/app/provider/enclave/config_test.go index 30568be..d29d1da 100644 --- a/internal/app/provider/enclave/config_test.go +++ b/internal/app/provider/enclave/config_test.go @@ -15,6 +15,7 @@ package enclave import ( "testing" + "time" grpcClient "github.com/circlefin/arc-remote-signer/internal/common/grpc/client" "github.com/stretchr/testify/require" @@ -31,12 +32,20 @@ func TestNewProviderConfig_Defaults(t *testing.T) { require.Equal(t, uint32(enclaveDefaultCID), cfg.NitroEnclave.CID) require.Equal(t, uint32(10350), cfg.NitroEnclave.Port) - expectedClientCfg := grpcClient.NewClientConfig(providerName, enclaveDefaultURL) - require.Equal(t, expectedClientCfg.Name, cfg.Client.Name) - require.Equal(t, expectedClientCfg.BaseURL, cfg.Client.BaseURL) - require.Equal(t, expectedClientCfg.RequestTimeoutMS, cfg.Client.RequestTimeoutMS) + require.Equal(t, providerName, cfg.Client.Name) + require.Equal(t, enclaveDefaultURL, cfg.Client.BaseURL) + require.Equal(t, defaultSignMessageTimeoutMS, cfg.Client.RequestTimeoutMS) require.NotNil(t, cfg.Client.Retry) - require.NotNil(t, expectedClientCfg.Retry) - require.Equal(t, expectedClientCfg.Retry.MaxAttempts, cfg.Client.Retry.MaxAttempts) - require.Equal(t, expectedClientCfg.Retry.RetryCodes, cfg.Client.Retry.RetryCodes) + require.Equal(t, uint(1), cfg.Client.Retry.MaxAttempts) + require.Equal(t, DefaultMethodTimeouts(), cfg.Client.Methods) + require.Equal(t, uint(1), cfg.Client.Methods["initialize"].MaxAttempts) + require.Equal(t, 30*time.Second, cfg.StartupTimeout()) +} + +func TestProviderConfig_StartupTimeoutUsesIndependentTotalBudget(t *testing.T) { + cfg := NewProviderConfig() + cfg.StartupTimeoutMS = 1250 + cfg.Client.Methods["initialize"] = grpcClient.MethodConfig{TimeoutMS: 60000} + + require.Equal(t, 1250*time.Millisecond, cfg.StartupTimeout()) } diff --git a/internal/app/provider/enclave/enclave.go b/internal/app/provider/enclave/enclave.go index 189bfd2..b3d9496 100644 --- a/internal/app/provider/enclave/enclave.go +++ b/internal/app/provider/enclave/enclave.go @@ -18,6 +18,7 @@ package enclave import ( "fmt" + "strings" "github.com/circlefin/arc-remote-signer/internal/common/grpc/client" "github.com/circlefin/arc-remote-signer/proto/pb" @@ -26,6 +27,28 @@ import ( // New creates a new enclave provider. func New(pc *ProviderConfig) (pb.EnclaveServiceClient, *grpc.ClientConn, error) { + var extraDialOptions []grpc.DialOption + if pc != nil && pc.NitroEnclave != nil && pc.NitroEnclave.Enabled { + if pc.NitroEnclave.CID == 0 || pc.NitroEnclave.Port == 0 { + return nil, nil, fmt.Errorf("nitro enclave mode requires a valid CID and port") + } + extraDialOptions = append(extraDialOptions, grpc.WithContextDialer(NewVsockDialer(pc.NitroEnclave.CID, pc.NitroEnclave.Port))) + } + return newClient(pc, extraDialOptions...) +} + +// NewNitro creates an enclave client that always uses VSOCK. +func NewNitro(pc *ProviderConfig) (pb.EnclaveServiceClient, *grpc.ClientConn, error) { + if pc == nil || pc.NitroEnclave == nil || pc.NitroEnclave.CID == 0 || pc.NitroEnclave.Port == 0 { + return nil, nil, fmt.Errorf("nitro enclave mode requires a valid CID and port") + } + return newClient( + pc, + grpc.WithContextDialer(NewVsockDialer(pc.NitroEnclave.CID, pc.NitroEnclave.Port)), + ) +} + +func newClient(pc *ProviderConfig, extraDialOptions ...grpc.DialOption) (pb.EnclaveServiceClient, *grpc.ClientConn, error) { if pc == nil { return nil, nil, fmt.Errorf("provider config is nil") } @@ -33,20 +56,33 @@ func New(pc *ProviderConfig) (pb.EnclaveServiceClient, *grpc.ClientConn, error) return nil, nil, fmt.Errorf("provider client config is nil") } - nitroEnabled := pc.NitroEnclave != nil && pc.NitroEnclave.Enabled - if nitroEnabled && (pc.NitroEnclave.CID <= 0 || pc.NitroEnclave.Port == 0) { - return nil, nil, fmt.Errorf("nitro enclave is enabled but cid or port is invalid") - } - - var extraDialOptions []grpc.DialOption - if nitroEnabled { - extraDialOptions = append(extraDialOptions, grpc.WithContextDialer(NewVsockDialer(pc.NitroEnclave.CID, pc.NitroEnclave.Port))) - } - - conn, err := client.NewInsecureClientConn(pc.Client.BaseURL, *pc.Client, extraDialOptions...) + clientConfig := pinInitializeTransportAttempts(*pc.Client) + conn, err := client.NewInsecureClientConn(pc.Client.BaseURL, clientConfig, extraDialOptions...) if err != nil { return nil, nil, fmt.Errorf("failed to create enclave client connection: %w", err) } return pb.NewEnclaveServiceClient(conn), conn, nil } + +func pinInitializeTransportAttempts(cfg client.Config) client.Config { + methods := make(map[string]client.MethodConfig, len(cfg.Methods)+1) + for method, methodConfig := range cfg.Methods { + if slash := strings.LastIndex(method, "/"); slash >= 0 { + if strings.EqualFold(method[slash+1:], "Initialize") { + methodConfig.MaxAttempts = 1 + } + } else if strings.EqualFold(method, "Initialize") { + methodConfig.MaxAttempts = 1 + } + methods[method] = methodConfig + } + initializeConfig, ok := methods["initialize"] + if !ok { + initializeConfig.TimeoutMS = defaultStartupTimeoutMS + } + initializeConfig.MaxAttempts = 1 + methods["initialize"] = initializeConfig + cfg.Methods = methods + return cfg +} diff --git a/internal/app/provider/enclave/enclave_mock.go b/internal/app/provider/enclave/enclave_mock.go index 0573ca9..6d89b9f 100644 --- a/internal/app/provider/enclave/enclave_mock.go +++ b/internal/app/provider/enclave/enclave_mock.go @@ -56,44 +56,44 @@ func (mr *MockEnclaveServiceClientMockRecorder) GenerateKey(arg0, arg1 interface return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GenerateKey", reflect.TypeOf((*MockEnclaveServiceClient)(nil).GenerateKey), varargs...) } -// GetAttestation mocks base method. -func (m *MockEnclaveServiceClient) GetAttestation(arg0 context.Context, arg1 *pb.GetAttestationRequest, arg2 ...grpc.CallOption) (*pb.GetAttestationResponse, error) { +// GetPublicKey mocks base method. +func (m *MockEnclaveServiceClient) GetPublicKey(arg0 context.Context, arg1 *pb.GetPublicKeyRequest, arg2 ...grpc.CallOption) (*pb.GetPublicKeyResponse, error) { m.ctrl.T.Helper() varargs := []interface{}{arg0, arg1} for _, a := range arg2 { varargs = append(varargs, a) } - ret := m.ctrl.Call(m, "GetAttestation", varargs...) - ret0, _ := ret[0].(*pb.GetAttestationResponse) + ret := m.ctrl.Call(m, "GetPublicKey", varargs...) + ret0, _ := ret[0].(*pb.GetPublicKeyResponse) ret1, _ := ret[1].(error) return ret0, ret1 } -// GetAttestation indicates an expected call of GetAttestation. -func (mr *MockEnclaveServiceClientMockRecorder) GetAttestation(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { +// GetPublicKey indicates an expected call of GetPublicKey. +func (mr *MockEnclaveServiceClientMockRecorder) GetPublicKey(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() varargs := append([]interface{}{arg0, arg1}, arg2...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAttestation", reflect.TypeOf((*MockEnclaveServiceClient)(nil).GetAttestation), varargs...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPublicKey", reflect.TypeOf((*MockEnclaveServiceClient)(nil).GetPublicKey), varargs...) } -// GetPublicKey mocks base method. -func (m *MockEnclaveServiceClient) GetPublicKey(arg0 context.Context, arg1 *pb.GetPublicKeyRequest, arg2 ...grpc.CallOption) (*pb.GetPublicKeyResponse, error) { +// Initialize mocks base method. +func (m *MockEnclaveServiceClient) Initialize(arg0 context.Context, arg1 *pb.InitializeRequest, arg2 ...grpc.CallOption) (*pb.InitializeResponse, error) { m.ctrl.T.Helper() varargs := []interface{}{arg0, arg1} for _, a := range arg2 { varargs = append(varargs, a) } - ret := m.ctrl.Call(m, "GetPublicKey", varargs...) - ret0, _ := ret[0].(*pb.GetPublicKeyResponse) + ret := m.ctrl.Call(m, "Initialize", varargs...) + ret0, _ := ret[0].(*pb.InitializeResponse) ret1, _ := ret[1].(error) return ret0, ret1 } -// GetPublicKey indicates an expected call of GetPublicKey. -func (mr *MockEnclaveServiceClientMockRecorder) GetPublicKey(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { +// Initialize indicates an expected call of Initialize. +func (mr *MockEnclaveServiceClientMockRecorder) Initialize(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() varargs := append([]interface{}{arg0, arg1}, arg2...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPublicKey", reflect.TypeOf((*MockEnclaveServiceClient)(nil).GetPublicKey), varargs...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Initialize", reflect.TypeOf((*MockEnclaveServiceClient)(nil).Initialize), varargs...) } // SignMessage mocks base method. diff --git a/internal/app/provider/enclave/enclave_test.go b/internal/app/provider/enclave/enclave_test.go index ddc92ef..ed5c010 100644 --- a/internal/app/provider/enclave/enclave_test.go +++ b/internal/app/provider/enclave/enclave_test.go @@ -14,12 +14,32 @@ package enclave import ( + "context" + "net" + "sync/atomic" "testing" grpcClient "github.com/circlefin/arc-remote-signer/internal/common/grpc/client" + "github.com/circlefin/arc-remote-signer/proto/pb" "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) +type internalInitializeServer struct { + pb.UnimplementedEnclaveServiceServer + calls atomic.Int32 +} + +func (s *internalInitializeServer) Initialize( + context.Context, + *pb.InitializeRequest, +) (*pb.InitializeResponse, error) { + s.calls.Add(1) + return nil, status.Error(codes.Internal, "cryptographic failure") +} + func TestNew_Success(t *testing.T) { cfg := NewProviderConfig() cfg.Client.BaseURL = ":10350" @@ -59,7 +79,7 @@ func TestNew_NitroEnabledWithInvalidCID(t *testing.T) { require.Nil(t, client) require.Nil(t, conn) require.Error(t, err) - require.EqualError(t, err, "nitro enclave is enabled but cid or port is invalid") + require.EqualError(t, err, "nitro enclave mode requires a valid CID and port") } func TestNew_NitroEnabledWithInvalidPort(t *testing.T) { @@ -73,7 +93,7 @@ func TestNew_NitroEnabledWithInvalidPort(t *testing.T) { require.Nil(t, client) require.Nil(t, conn) require.Error(t, err) - require.EqualError(t, err, "nitro enclave is enabled but cid or port is invalid") + require.EqualError(t, err, "nitro enclave mode requires a valid CID and port") } func TestNew_InvalidTarget(t *testing.T) { @@ -90,3 +110,115 @@ func TestNew_InvalidTarget(t *testing.T) { require.Error(t, err) require.ErrorContains(t, err, "failed to create enclave client connection") } + +func TestNew_InitializeInternalResponseHasOneTransportAttempt(t *testing.T) { + listener, err := net.Listen("tcp", "localhost:0") + require.NoError(t, err) + server := grpc.NewServer() + enclaveServer := &internalInitializeServer{} + pb.RegisterEnclaveServiceServer(server, enclaveServer) + go func() { _ = server.Serve(listener) }() + t.Cleanup(func() { + server.Stop() + _ = listener.Close() + }) + + cfg := NewProviderConfig() + cfg.Client.BaseURL = listener.Addr().String() + cfg.Client.Retry.MaxAttempts = 3 + cfg.Client.Methods["initialize"] = grpcClient.MethodConfig{ + TimeoutMS: 1000, + MaxAttempts: 3, + } + cfg.Client.Methods["Initialize"] = grpcClient.MethodConfig{ + TimeoutMS: 1000, + MaxAttempts: 3, + } + client, conn, err := New(cfg) + require.NoError(t, err) + t.Cleanup(func() { _ = conn.Close() }) + + resp, err := client.Initialize(context.Background(), &pb.InitializeRequest{}) + + require.Nil(t, resp) + require.Equal(t, codes.Internal, status.Code(err)) + require.Equal(t, int32(1), enclaveServer.calls.Load()) +} + +func TestPinInitializeTransportAttempts(t *testing.T) { + t.Run("missing Initialize entry uses startup timeout", func(t *testing.T) { + cfg := grpcClient.Config{ + RequestTimeoutMS: 500, + Methods: map[string]grpcClient.MethodConfig{ + "signmessage": {TimeoutMS: 500, MaxAttempts: 3}, + }, + } + + got := pinInitializeTransportAttempts(cfg) + + require.Equal(t, grpcClient.MethodConfig{ + TimeoutMS: defaultStartupTimeoutMS, + MaxAttempts: 1, + }, got.Methods["initialize"]) + }) + + t.Run("custom Initialize aliases retain timeouts", func(t *testing.T) { + cfg := grpcClient.Config{ + Methods: map[string]grpcClient.MethodConfig{ + "initialize": {TimeoutMS: 41000, MaxAttempts: 4}, + "Initialize": {TimeoutMS: 42000, MaxAttempts: 5}, + "/arc.enclave.v1.EnclaveService/Initialize": {TimeoutMS: 43000, MaxAttempts: 6}, + }, + } + + got := pinInitializeTransportAttempts(cfg) + + require.Equal(t, grpcClient.MethodConfig{TimeoutMS: 41000, MaxAttempts: 1}, got.Methods["initialize"]) + require.Equal(t, grpcClient.MethodConfig{TimeoutMS: 42000, MaxAttempts: 1}, got.Methods["Initialize"]) + require.Equal(t, grpcClient.MethodConfig{ + TimeoutMS: 43000, + MaxAttempts: 1, + }, got.Methods["/arc.enclave.v1.EnclaveService/Initialize"]) + }) +} + +func TestNewNitro_IgnoresRuntimeToggle(t *testing.T) { + cfg := NewProviderConfig() + cfg.NitroEnclave.Enabled = false + + client, conn, err := NewNitro(cfg) + + require.NoError(t, err) + require.NotNil(t, client) + require.NotNil(t, conn) + require.NoError(t, conn.Close()) +} + +func TestNewNitro_RequiresCIDAndPort(t *testing.T) { + nilNitroConfig := NewProviderConfig() + nilNitroConfig.NitroEnclave = nil + zeroCIDConfig := NewProviderConfig() + zeroCIDConfig.NitroEnclave.CID = 0 + zeroPortConfig := NewProviderConfig() + zeroPortConfig.NitroEnclave.Port = 0 + + tests := []struct { + name string + cfg *ProviderConfig + }{ + {name: "provider configuration is nil", cfg: nil}, + {name: "Nitro configuration is nil", cfg: nilNitroConfig}, + {name: "CID is zero", cfg: zeroCIDConfig}, + {name: "port is zero", cfg: zeroPortConfig}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + client, conn, err := NewNitro(tt.cfg) + + require.Nil(t, client) + require.Nil(t, conn) + require.ErrorContains(t, err, "requires a valid CID and port") + }) + } +} diff --git a/internal/app/provider/enclave/integration_test.go b/internal/app/provider/enclave/integration_test.go index 6624ca7..53f9775 100644 --- a/internal/app/provider/enclave/integration_test.go +++ b/internal/app/provider/enclave/integration_test.go @@ -19,7 +19,6 @@ import ( "context" "testing" - "github.com/circlefin/arc-remote-signer/internal/common/crypto/rand" "github.com/circlefin/arc-remote-signer/proto/pb" "github.com/stretchr/testify/suite" "google.golang.org/grpc" @@ -27,15 +26,16 @@ import ( "google.golang.org/grpc/status" ) +// testKMSKeyARN is the LocalStack multi-region key alias created by +// deployments/localstack_scripts/create-kms-keys.sh. +const testKMSKeyARN = "arn:aws:kms:us-east-1:000000000000:alias/dev-multi-region-crypto" + type integrationTestSuite struct { suite.Suite - client pb.EnclaveServiceClient - conn *grpc.ClientConn - testingEncryptedDataKey []byte - testingEncryptedPrivateKey []byte - testingNonce []byte - testingPubKey []byte + client pb.EnclaveServiceClient + conn *grpc.ClientConn + testingPubKey []byte } func (suite *integrationTestSuite) SetupSuite() { @@ -44,46 +44,48 @@ func (suite *integrationTestSuite) SetupSuite() { suite.Require().NoError(err) suite.client = client suite.conn = conn -} -func (suite *integrationTestSuite) TearDownSuite() { - err := suite.conn.Close() + // The enclave now mints its own data key via KMS, so it must be + // initialized with credentials and the LocalStack KMS ARN before any + // GenerateKey/read RPC. Initialize is idempotent, so running it once per + // suite is sufficient. Credentials match the LocalStack values in + // docker-compose; session_token is a placeholder to satisfy the proto's + // min_len validation (LocalStack ignores it). + resp, err := suite.client.Initialize(context.Background(), &pb.InitializeRequest{ + Credentials: &pb.AwsCredentials{ + AccessKeyId: "foo", + SecretAccessKey: "bar", + SessionToken: "session", + Region: "us-east-1", + }, + KmsKeyArns: []string{testKMSKeyARN}, + KmsLocalstackEnabled: true, + KeySource: &pb.InitializeRequest_GenerateNew{ + GenerateNew: pb.Algorithm_ALGORITHM_ED25519, + }, + }) suite.Require().NoError(err) + suite.Require().NotNil(resp) + suite.testingPubKey = resp.PublicKey } -func (suite *integrationTestSuite) SetupTest() { - suite.testingEncryptedDataKey = rand.MustGenerateRandomBytes(32) - result, err := suite.client.GenerateKey(context.Background(), &pb.GenerateKeyRequest{ - Algorithm: pb.Algorithm_ALGORITHM_ED25519, - EnclaveEncryptedDataKey: suite.testingEncryptedDataKey, - }) +func (suite *integrationTestSuite) TearDownSuite() { + err := suite.conn.Close() suite.Require().NoError(err) - suite.Require().NotNil(result) - suite.Require().NotEmpty(result.PublicKey) - suite.Require().NotEmpty(result.EncryptedPrivateKey) - suite.Require().NotEmpty(result.Nonce) - - suite.testingEncryptedPrivateKey = result.EncryptedPrivateKey - suite.testingNonce = result.Nonce - suite.testingPubKey = result.PublicKey } func (suite *integrationTestSuite) TestGenerateKey() { - suite.Run("success", func() { + suite.Run("disabled after initialization", func() { result, err := suite.client.GenerateKey(context.Background(), &pb.GenerateKeyRequest{ - Algorithm: pb.Algorithm_ALGORITHM_ED25519, - EnclaveEncryptedDataKey: suite.testingEncryptedDataKey, + Algorithm: pb.Algorithm_ALGORITHM_ED25519, }) - suite.Require().NoError(err) - suite.Require().NotNil(result) - suite.Require().NotEmpty(result.PublicKey) - suite.Require().NotEmpty(result.EncryptedPrivateKey) - suite.Require().NotEmpty(result.Nonce) + suite.Require().Error(err) + suite.Require().Nil(result) + suite.Require().Equal(codes.FailedPrecondition, status.Code(err)) }) suite.Run("invalid algorithm", func() { result, err := suite.client.GenerateKey(context.Background(), &pb.GenerateKeyRequest{ - Algorithm: pb.Algorithm_ALGORITHM_UNSPECIFIED, - EnclaveEncryptedDataKey: suite.testingEncryptedDataKey, + Algorithm: pb.Algorithm_ALGORITHM_UNSPECIFIED, }) suite.Require().Error(err) suite.Require().Nil(result) @@ -93,14 +95,7 @@ func (suite *integrationTestSuite) TestGenerateKey() { func (suite *integrationTestSuite) TestGetPublicKey() { suite.Run("success", func() { - result, err := suite.client.GetPublicKey(context.Background(), &pb.GetPublicKeyRequest{ - Algorithm: pb.Algorithm_ALGORITHM_ED25519, - EncryptedKeyMaterial: &pb.EncryptedKeyMaterial{ - EncryptedPrivateKey: suite.testingEncryptedPrivateKey, - EnclaveEncryptedDataKey: suite.testingEncryptedDataKey, - Nonce: suite.testingNonce, - }, - }) + result, err := suite.client.GetPublicKey(context.Background(), &pb.GetPublicKeyRequest{}) suite.Require().NoError(err) suite.Require().NotNil(result) suite.Require().NotEmpty(result.PublicKey) @@ -111,38 +106,20 @@ func (suite *integrationTestSuite) TestGetPublicKey() { func (suite *integrationTestSuite) TestSignMessage() { suite.Run("success", func() { result, err := suite.client.SignMessage(context.Background(), &pb.SignMessageRequest{ - Algorithm: pb.Algorithm_ALGORITHM_ED25519, - EncryptedKeyMaterial: &pb.EncryptedKeyMaterial{ - EncryptedPrivateKey: suite.testingEncryptedPrivateKey, - EnclaveEncryptedDataKey: suite.testingEncryptedDataKey, - Nonce: suite.testingNonce, - }, Message: []byte("test message"), }) suite.Require().NoError(err) suite.Require().NotNil(result) suite.Require().NotEmpty(result.Signature) }) - suite.Run("invalid algorithm", func() { - result, err := suite.client.SignMessage(context.Background(), &pb.SignMessageRequest{ - Algorithm: pb.Algorithm_ALGORITHM_UNSPECIFIED, - }) + suite.Run("empty message", func() { + result, err := suite.client.SignMessage(context.Background(), &pb.SignMessageRequest{}) suite.Require().Error(err) suite.Require().Nil(result) suite.Require().Equal(codes.InvalidArgument, status.Code(err)) }) } -func (suite *integrationTestSuite) TestGetAttestation() { - // Local/CI environments do not run enclave service inside Nitro Enclave. - // Therefore this integration test can only validate the "enclave not enabled" - // behavior, which should return FailedPrecondition. - resp, err := suite.client.GetAttestation(context.Background(), &pb.GetAttestationRequest{}) - suite.Require().Error(err) - suite.Require().Equal(codes.FailedPrecondition, status.Code(err)) - suite.Require().Nil(resp) -} - func TestEnclaveProviderIntegration(t *testing.T) { suite.Run(t, new(integrationTestSuite)) } diff --git a/internal/app/public/public.go b/internal/app/public/public.go index 2e9d7fc..c64e70f 100644 --- a/internal/app/public/public.go +++ b/internal/app/public/public.go @@ -53,7 +53,9 @@ func New(cfg *grpcServer.Config, params CreateServerParams) (lifecycle.Runnable, } grpcSrv := grpcServer.NewServer(engineParams, opts...) - reflection.Register(grpcSrv) + if cfg.Reflection.Enabled { + reflection.Register(grpcSrv) + } pb.RegisterSignerServiceServer(grpcSrv, params.SignerSvc) runnable, err := grpcServer.NewRunnable( diff --git a/internal/app/public/public_test.go b/internal/app/public/public_test.go index 07ef7f3..485932c 100644 --- a/internal/app/public/public_test.go +++ b/internal/app/public/public_test.go @@ -14,15 +14,103 @@ package public import ( + "context" + "io" "testing" + "time" "github.com/circlefin/arc-remote-signer/internal/common/config" grpcServer "github.com/circlefin/arc-remote-signer/internal/common/grpc/server" "github.com/circlefin/arc-remote-signer/internal/common/metric" "github.com/circlefin/arc-remote-signer/proto/pb" "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + reflectionpb "google.golang.org/grpc/reflection/grpc_reflection_v1" + "google.golang.org/grpc/status" ) +func reflectionListServices(t *testing.T, cfg *grpcServer.Config) (*reflectionpb.ListServiceResponse, error) { + t.Helper() + + runnable, err := New(cfg, CreateServerParams{ + ServiceName: "app.public", + SignerSvc: &pb.UnimplementedSignerServiceServer{}, + Env: config.Dev, + }) + require.NoError(t, err) + + server, ok := runnable.(*grpcServer.RunnableImpl) + require.True(t, ok) + require.NoError(t, server.Run()) + t.Cleanup(func() { + if err := server.Shutdown(context.Background()); err != nil { + t.Errorf("shutdown gRPC server: %v", err) + } + }) + + conn, err := grpc.NewClient( + server.Addr().String(), + grpc.WithTransportCredentials(insecure.NewCredentials()), + ) + require.NoError(t, err) + t.Cleanup(func() { + if err := conn.Close(); err != nil { + t.Errorf("close reflection client connection: %v", err) + } + }) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + stream, err := reflectionpb.NewServerReflectionClient(conn).ServerReflectionInfo(ctx) + require.NoError(t, err) + err = stream.Send(&reflectionpb.ServerReflectionRequest{ + MessageRequest: &reflectionpb.ServerReflectionRequest_ListServices{ + ListServices: "", + }, + }) + if err != nil && err != io.EOF { + return nil, err + } + + resp, err := stream.Recv() + if err != nil { + return nil, err + } + return resp.GetListServicesResponse(), nil +} + +func TestNew_DisablesReflectionByDefault(t *testing.T) { + cfg := &grpcServer.Config{ + Host: "127.0.0.1", + Port: 0, + } + + services, err := reflectionListServices(t, cfg) + require.Error(t, err) + require.Equal(t, codes.Unimplemented, status.Code(err)) + require.Nil(t, services) +} + +func TestNew_EnablesReflectionWhenConfigured(t *testing.T) { + cfg := &grpcServer.Config{ + Host: "127.0.0.1", + Port: 0, + Reflection: grpcServer.ReflectionConfig{ + Enabled: true, + }, + } + + services, err := reflectionListServices(t, cfg) + require.NoError(t, err) + require.NotNil(t, services) + require.Contains(t, services.Service, &reflectionpb.ServiceResponse{ + Name: pb.SignerService_ServiceDesc.ServiceName, + }) +} + func TestNew_ReturnsRunnableWithServiceName(t *testing.T) { cfg := &grpcServer.Config{ Host: "127.0.0.1", diff --git a/internal/app/runtime_dev.go b/internal/app/runtime_dev.go new file mode 100644 index 0000000..42bec40 --- /dev/null +++ b/internal/app/runtime_dev.go @@ -0,0 +1,97 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !prod + +package app + +import ( + "context" + "errors" + + "github.com/aws/aws-sdk-go-v2/aws" + awsSdkConfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + enclaveProvider "github.com/circlefin/arc-remote-signer/internal/app/provider/enclave" + "github.com/circlefin/arc-remote-signer/internal/app/provider/secrets" + "github.com/circlefin/arc-remote-signer/internal/common/config" + "github.com/circlefin/arc-remote-signer/internal/common/logging" + "github.com/circlefin/arc-remote-signer/proto/pb" + "google.golang.org/grpc" +) + +func applyBuildPolicy(*Config) { + // The development build uses the supplied configuration. +} + +// NewAWSKMSConfig returns development KMS defaults. +func NewAWSKMSConfig() *AWSKMSConfig { + return &AWSKMSConfig{ + Localstack: &AWSKMSLocalstackConfig{Enabled: true}, + Arns: []string{ + "arn:aws:kms:us-east-1:000000000000:alias/dev-multi-region-crypto", + "arn:aws:kms:us-west-2:000000000000:alias/dev-multi-region-crypto", + }, + } +} + +// NewConfig returns development configuration. +func NewConfig() *Config { + return newConfig(config.NewBaseConfig(), secrets.NewConfig(), NewAWSKMSConfig()) +} + +func newRuntimeEnclave(cfg *enclaveProvider.ProviderConfig) (pb.EnclaveServiceClient, *grpc.ClientConn, error) { + return enclaveProvider.New(cfg) +} + +func loadRuntimeAWSConfig(ctx context.Context, cfg *Config, logger *logging.Logger) (aws.Config, error) { + return retrieveAWSConfig(ctx, cfg, logger) +} + +// MergeAwsConfigWithLocalstack creates an AWS configuration for LocalStack. +func MergeAwsConfigWithLocalstack(cfg *Config) (aws.Config, error) { + awsEndpoint := cfg.Provider.Secrets.Localstack.Endpoint + awsRegion := cfg.Provider.Secrets.Localstack.Region + + configOptions := []func(*awsSdkConfig.LoadOptions) error{} + if awsRegion != "" { + configOptions = append(configOptions, awsSdkConfig.WithRegion(awsRegion)) + } + if awsEndpoint != "" { + configOptions = append( + configOptions, + awsSdkConfig.WithBaseEndpoint(awsEndpoint), + awsSdkConfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider( + "test", + "test", + "dev-session-token-placeholder", + )), + ) + } + + return awsSdkConfig.LoadDefaultConfig(context.Background(), configOptions...) +} + +func retrieveAWSConfig(ctx context.Context, cfg *Config, logger *logging.Logger) (aws.Config, error) { + if cfg.BaseConfig == nil || cfg.Provider.Secrets == nil || cfg.Provider.Secrets.Localstack == nil { + return aws.Config{}, errors.New("secrets configuration is unavailable") + } + + if (cfg.Env == config.Dev || cfg.Env == config.QA) && + cfg.Provider.Secrets.Localstack.Enabled && cfg.Provider.Secrets.Localstack.Endpoint != "" { + logger.Info(ctx, "The signer uses the LocalStack AWS configuration.", nil) + return MergeAwsConfigWithLocalstack(cfg) + } + logger.Info(ctx, "The signer uses the standard AWS configuration.", nil) + return awsSdkConfig.LoadDefaultConfig(ctx) +} diff --git a/internal/app/runtime_dev_test.go b/internal/app/runtime_dev_test.go new file mode 100644 index 0000000..bf440f4 --- /dev/null +++ b/internal/app/runtime_dev_test.go @@ -0,0 +1,40 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !prod + +package app + +import ( + "testing" + + "github.com/circlefin/arc-remote-signer/internal/common/config" + "github.com/stretchr/testify/require" +) + +func TestApplyBuildPolicy_PreservesDevelopmentSettings(t *testing.T) { + cfg := NewConfig() + cfg.Env = config.QA + cfg.Provider.Enclave.NitroEnclave.Enabled = false + cfg.Provider.Enclave.Client.BaseURL = "localhost:20350" + cfg.Provider.Secrets.Localstack.Enabled = true + cfg.Provider.AWSKMS.Localstack.Enabled = true + + applyBuildPolicy(cfg) + + require.Equal(t, config.QA, cfg.Env) + require.False(t, cfg.Provider.Enclave.NitroEnclave.Enabled) + require.Equal(t, "localhost:20350", cfg.Provider.Enclave.Client.BaseURL) + require.True(t, cfg.Provider.Secrets.Localstack.Enabled) + require.True(t, cfg.Provider.AWSKMS.Localstack.Enabled) +} diff --git a/internal/app/runtime_prod.go b/internal/app/runtime_prod.go new file mode 100644 index 0000000..2f62378 --- /dev/null +++ b/internal/app/runtime_prod.go @@ -0,0 +1,83 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build prod + +package app + +import ( + "context" + "fmt" + + "github.com/aws/aws-sdk-go-v2/aws" + awsSdkConfig "github.com/aws/aws-sdk-go-v2/config" + enclaveProvider "github.com/circlefin/arc-remote-signer/internal/app/provider/enclave" + "github.com/circlefin/arc-remote-signer/internal/app/provider/secrets" + "github.com/circlefin/arc-remote-signer/internal/common/config" + "github.com/circlefin/arc-remote-signer/internal/common/logging" + "github.com/circlefin/arc-remote-signer/proto/pb" + "google.golang.org/grpc" +) + +const ( + // The production host uses a fixed CID and port. This rule prevents + // configuration from selecting a local VSOCK peer. + productionEnclaveCID uint32 = 16 + productionEnclavePort uint32 = 10350 +) + +func applyBuildPolicy(cfg *Config) { + // The environment value identifies the deployment only. It must not select runtime behavior. + if cfg.Env != config.Stg { + cfg.Env = config.Prod + } + cfg.Provider.Enclave.NitroEnclave.Enabled = true + cfg.Provider.Enclave.NitroEnclave.CID = productionEnclaveCID + cfg.Provider.Enclave.NitroEnclave.Port = productionEnclavePort + cfg.Provider.Enclave.Client.BaseURL = fmt.Sprintf("localhost:%d", productionEnclavePort) + cfg.Provider.Secrets.Localstack.Enabled = false + cfg.Provider.AWSKMS.Localstack.Enabled = false +} + +// NewConfig returns production configuration. +func NewConfig() *Config { + cfg := newConfig( + config.NewBaseConfig(), + &secrets.Config{Localstack: &secrets.LocalstackConfig{}}, + &AWSKMSConfig{Localstack: &AWSKMSLocalstackConfig{}}, + ) + applyBuildPolicy(cfg) + return cfg +} + +func newRuntimeEnclave(cfg *enclaveProvider.ProviderConfig) (pb.EnclaveServiceClient, *grpc.ClientConn, error) { + return enclaveProvider.NewNitro(cfg) +} + +func loadRuntimeAWSConfig(ctx context.Context, _ *Config, logger *logging.Logger) (aws.Config, error) { + logger.Info(ctx, "The signer uses the standard AWS configuration.", nil) + cfg, err := awsSdkConfig.LoadDefaultConfig(ctx) + if err != nil { + return aws.Config{}, err + } + cfg.BaseEndpoint = nil + cfg.ConfigSources = append([]interface{}{productionEndpointPolicy{}}, cfg.ConfigSources...) + return cfg, nil +} + +// productionEndpointPolicy disables SDK endpoint settings. +type productionEndpointPolicy struct{} + +func (productionEndpointPolicy) GetIgnoreConfiguredEndpoints(context.Context) (bool, bool, error) { + return true, true, nil +} diff --git a/internal/app/runtime_prod_test.go b/internal/app/runtime_prod_test.go new file mode 100644 index 0000000..8ffd98f --- /dev/null +++ b/internal/app/runtime_prod_test.go @@ -0,0 +1,178 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build prod + +package app + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + awsSdkConfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/secretsmanager" + "github.com/circlefin/arc-remote-signer/internal/common/config" + "github.com/circlefin/arc-remote-signer/internal/common/logging" + "github.com/stretchr/testify/require" +) + +func TestApplyBuildPolicy_ReappliesAfterEnvironmentLoad(t *testing.T) { + t.Setenv("APP_ENV", "dev") + t.Setenv("APP_PROVIDER_ENCLAVE_NITROENCLAVE_ENABLED", "false") + t.Setenv("APP_PROVIDER_ENCLAVE_NITROENCLAVE_CID", "1") + t.Setenv("APP_PROVIDER_ENCLAVE_NITROENCLAVE_PORT", "1") + t.Setenv("APP_PROVIDER_ENCLAVE_CLIENT_BASEURL", "127.0.0.1:1") + t.Setenv("APP_PROVIDER_SECRETS_LOCALSTACK_ENABLED", "true") + t.Setenv("APP_PROVIDER_AWSKMS_LOCALSTACK_ENABLED", "true") + + configFile := filepath.Join(t.TempDir(), "config.yaml") + require.NoError(t, os.WriteFile(configFile, []byte("{}\n"), 0o600)) + + cfg := NewConfig() + config.LoadConfig(cfg, configFile) + + require.Equal(t, config.Dev, cfg.Env) + require.False(t, cfg.Provider.Enclave.NitroEnclave.Enabled) + require.Equal(t, uint32(1), cfg.Provider.Enclave.NitroEnclave.CID) + require.Equal(t, uint32(1), cfg.Provider.Enclave.NitroEnclave.Port) + require.Equal(t, "127.0.0.1:1", cfg.Provider.Enclave.Client.BaseURL) + require.True(t, cfg.Provider.Secrets.Localstack.Enabled) + require.True(t, cfg.Provider.AWSKMS.Localstack.Enabled) + + applyBuildPolicy(cfg) + + require.Equal(t, config.Prod, cfg.Env) + require.True(t, cfg.Provider.Enclave.NitroEnclave.Enabled) + require.Equal(t, uint32(16), cfg.Provider.Enclave.NitroEnclave.CID) + require.Equal(t, uint32(10350), cfg.Provider.Enclave.NitroEnclave.Port) + require.Equal(t, "localhost:10350", cfg.Provider.Enclave.Client.BaseURL) + require.False(t, cfg.Provider.Secrets.Localstack.Enabled) + require.False(t, cfg.Provider.AWSKMS.Localstack.Enabled) +} + +func TestApplyBuildPolicy_PreservesStagingLabelAndEnforcesPolicy(t *testing.T) { + cfg := NewConfig() + cfg.Env = config.Stg + cfg.Provider.Enclave.NitroEnclave.Enabled = false + cfg.Provider.Enclave.NitroEnclave.CID = 1 + cfg.Provider.Enclave.NitroEnclave.Port = 1 + cfg.Provider.Enclave.Client.BaseURL = "127.0.0.1:1" + cfg.Provider.Secrets.Localstack.Enabled = true + cfg.Provider.AWSKMS.Localstack.Enabled = true + + applyBuildPolicy(cfg) + + require.Equal(t, config.Stg, cfg.Env) + require.True(t, cfg.Provider.Enclave.NitroEnclave.Enabled) + require.Equal(t, productionEnclaveCID, cfg.Provider.Enclave.NitroEnclave.CID) + require.Equal(t, productionEnclavePort, cfg.Provider.Enclave.NitroEnclave.Port) + require.Equal(t, "localhost:10350", cfg.Provider.Enclave.Client.BaseURL) + require.False(t, cfg.Provider.Secrets.Localstack.Enabled) + require.False(t, cfg.Provider.AWSKMS.Localstack.Enabled) +} + +func TestProductionConfig_PreservesEnclaveRequestTimeout(t *testing.T) { + cfg := NewConfig() + config.LoadConfig(cfg, filepath.Join("..", "..", "configs", "app.production.yaml")) + + require.Equal(t, 500, cfg.Provider.Enclave.Client.RequestTimeoutMS) + require.Equal(t, 30000, cfg.Provider.Enclave.Client.Methods["initialize"].TimeoutMS) + require.Equal(t, uint(1), cfg.Provider.Enclave.Client.Methods["initialize"].MaxAttempts) + require.Equal(t, 30000, cfg.Provider.Enclave.Client.Methods["generatekey"].TimeoutMS) + require.Equal(t, uint(1), cfg.Provider.Enclave.Client.Methods["generatekey"].MaxAttempts) + require.Equal(t, 30000, cfg.Provider.Enclave.Client.Methods["getpublickey"].TimeoutMS) + require.Equal(t, 500, cfg.Provider.Enclave.Client.Methods["signmessage"].TimeoutMS) + require.Equal(t, uint(1), cfg.Provider.Enclave.Client.Methods["signmessage"].MaxAttempts) + require.Equal(t, 30000, cfg.Provider.Enclave.StartupTimeoutMS) + require.Equal(t, 30*time.Second, cfg.Provider.Enclave.StartupTimeout()) +} + +func TestProductionConfig_MethodTimeoutEnvironmentOverride(t *testing.T) { + t.Setenv("APP_PROVIDER_ENCLAVE_CLIENT_METHODS_INITIALIZE_TIMEOUTMS", "1234") + cfg := NewConfig() + + config.LoadConfig(cfg, filepath.Join("..", "..", "configs", "app.production.yaml")) + + require.Equal(t, 1234, cfg.Provider.Enclave.Client.Methods["initialize"].TimeoutMS) +} + +func TestNewConfig_ExcludesDevelopmentDefaults(t *testing.T) { + cfg := NewConfig() + + require.Equal(t, config.Prod, cfg.Env) + require.True(t, cfg.Provider.Enclave.NitroEnclave.Enabled) + require.Empty(t, cfg.Provider.Secrets.Localstack.Endpoint) + require.Empty(t, cfg.Provider.Secrets.Localstack.Region) + require.False(t, cfg.Provider.Secrets.Localstack.Enabled) + require.Empty(t, cfg.Provider.AWSKMS.Arns) + require.False(t, cfg.Provider.AWSKMS.Localstack.Enabled) +} + +func TestNewRuntimeEnclave_CreatesNitroClient(t *testing.T) { + client, conn, err := newRuntimeEnclave(NewConfig().Provider.Enclave) + require.NoError(t, err) + require.NotNil(t, client) + require.NotNil(t, conn) + t.Cleanup(func() { + if err := conn.Close(); err != nil { + t.Errorf("close enclave connection: %v", err) + } + }) +} + +func TestLoadRuntimeAWSConfig_IgnoresEndpointOverrides(t *testing.T) { + t.Run("environment", func(t *testing.T) { + t.Setenv("AWS_REGION", "us-east-1") + t.Setenv("AWS_ENDPOINT_URL", "http://localhost:4566") + t.Setenv("AWS_ENDPOINT_URL_SECRETS_MANAGER", "http://localhost:4567") + t.Setenv("AWS_IGNORE_CONFIGURED_ENDPOINT_URLS", "false") + + assertProductionIgnoresEndpoint(t, "http://localhost:4567") + }) + + t.Run("shared configuration", func(t *testing.T) { + configFile := filepath.Join(t.TempDir(), "config") + require.NoError(t, os.WriteFile(configFile, []byte(`[profile production] +region = us-east-1 +endpoint_url = http://localhost:4568 +services = production-services + +[services production-services] +secrets_manager = + endpoint_url = http://localhost:4569 +`), 0o600)) + t.Setenv("AWS_CONFIG_FILE", configFile) + t.Setenv("AWS_PROFILE", "production") + t.Setenv("AWS_SDK_LOAD_CONFIG", "1") + t.Setenv("AWS_IGNORE_CONFIGURED_ENDPOINT_URLS", "false") + + assertProductionIgnoresEndpoint(t, "http://localhost:4569") + }) +} + +func assertProductionIgnoresEndpoint(t *testing.T, unsafeEndpoint string) { + t.Helper() + ctx := context.Background() + + unsafeConfig, err := awsSdkConfig.LoadDefaultConfig(ctx) + require.NoError(t, err) + require.Equal(t, unsafeEndpoint, *secretsmanager.NewFromConfig(unsafeConfig).Options().BaseEndpoint) + + safeConfig, err := loadRuntimeAWSConfig(ctx, NewConfig(), logging.Get("production-config-test")) + require.NoError(t, err) + require.Nil(t, safeConfig.BaseEndpoint) + require.Nil(t, secretsmanager.NewFromConfig(safeConfig).Options().BaseEndpoint) +} diff --git a/internal/app/service/signer/cache_test.go b/internal/app/service/signer/cache_test.go deleted file mode 100644 index cf3d3f0..0000000 --- a/internal/app/service/signer/cache_test.go +++ /dev/null @@ -1,164 +0,0 @@ -// Copyright (c) 2026, Circle Internet Group, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package signer - -import ( - "sync" - "testing" - - "github.com/circlefin/arc-remote-signer/proto/pb" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -const ( - testEncryptedPrivateKey = "test-encrypted-private-key" - testPlainDataKey = "test-plain-data-key" - testNonce = "test-nonce" - testPublicKey = "test-public-key" -) - -func TestCache_SetAndGet(t *testing.T) { - c := newCache() - - require.Nil(t, c.get()) - - testKey := &key{ - encryptedKeyMaterial: &pb.EncryptedKeyMaterial{ - EncryptedPrivateKey: []byte(testEncryptedPrivateKey), - EnclaveEncryptedDataKey: []byte(testPlainDataKey), - Nonce: []byte(testNonce), - }, - publicKey: []byte(testPublicKey), - } - - c.set(testKey) - - got := c.get() - require.NotNil(t, got) - assertKeyEqual(t, got, testKey) -} - -func TestCache_UpdateKey(t *testing.T) { - c := newCache() - - initialKey := &key{ - encryptedKeyMaterial: &pb.EncryptedKeyMaterial{ - EncryptedPrivateKey: []byte(testEncryptedPrivateKey), - EnclaveEncryptedDataKey: []byte(testPlainDataKey), - Nonce: []byte(testNonce), - }, - publicKey: []byte(testPublicKey), - } - c.set(initialKey) - - got := c.get() - assertKeyEqual(t, got, initialKey) - - updatedKey := &key{ - encryptedKeyMaterial: &pb.EncryptedKeyMaterial{ - EncryptedPrivateKey: []byte("updated-encrypted-private-key"), - EnclaveEncryptedDataKey: []byte("updated-plain-data-key"), - Nonce: []byte("updated-nonce"), - }, - publicKey: []byte("updated-public-key"), - } - c.set(updatedKey) - - got = c.get() - assertKeyEqual(t, got, updatedKey) -} - -func TestCache_ConcurrentAccess(_ *testing.T) { - c := newCache() - testKey := &key{ - encryptedKeyMaterial: &pb.EncryptedKeyMaterial{ - EncryptedPrivateKey: []byte(testEncryptedPrivateKey), - EnclaveEncryptedDataKey: []byte(testPlainDataKey), - Nonce: []byte(testNonce), - }, - publicKey: []byte(testPublicKey), - } - var wg sync.WaitGroup - - // Writer goroutine - wg.Add(1) - go func() { - defer wg.Done() - for i := 0; i < 1000; i++ { - c.set(testKey) - } - }() - - // Reader goroutine - wg.Add(1) - go func() { - defer wg.Done() - for i := 0; i < 1000; i++ { - _ = c.get() - } - }() - - wg.Wait() -} - -func TestCache_MultipleReaders(t *testing.T) { - c := newCache() - testKey := &key{ - encryptedKeyMaterial: &pb.EncryptedKeyMaterial{ - EncryptedPrivateKey: []byte(testEncryptedPrivateKey), - EnclaveEncryptedDataKey: []byte(testPlainDataKey), - Nonce: []byte(testNonce), - }, - publicKey: []byte(testPublicKey), - } - - c.set(testKey) - - var wg sync.WaitGroup - readerCount := 10 - - for i := 0; i < readerCount; i++ { - wg.Add(1) - go func() { - defer wg.Done() - for j := 0; j < 100; j++ { - got := c.get() - if assert.NotNil(t, got) { - assertKeyEqual(t, got, testKey) - } - } - }() - } - - wg.Wait() -} - -func assertKeyEqual(t *testing.T, got, want *key) { - t.Helper() - assert.NotNil(t, got) - assert.NotNil(t, got.encryptedKeyMaterial) - assert.NotNil(t, want.encryptedKeyMaterial) - assert.Equal(t, want.encryptedKeyMaterial.EncryptedPrivateKey, got.encryptedKeyMaterial.EncryptedPrivateKey) - assert.Equal(t, want.encryptedKeyMaterial.EnclaveEncryptedDataKey, got.encryptedKeyMaterial.EnclaveEncryptedDataKey) - assert.Equal(t, want.encryptedKeyMaterial.Nonce, got.encryptedKeyMaterial.Nonce) - assert.Equal(t, want.publicKey, got.publicKey) -} - -func TestCache_NewCache(t *testing.T) { - c := newCache() - - require.NotNil(t, c) - require.Nil(t, c.get()) -} diff --git a/internal/app/service/signer/crypto.go b/internal/app/service/signer/crypto.go deleted file mode 100644 index 1d6843a..0000000 --- a/internal/app/service/signer/crypto.go +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright (c) 2026, Circle Internet Group, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package signer - -import ( - "bytes" - "encoding/gob" -) - -// header represents the AES encrypted data. -// It contains the cipher key, cipher data, and nonce. -type header struct { - CipherKey []byte - CipherData []byte - Nonce []byte -} - -// MarshalBinary marshals the header to binary format. -func (h *header) MarshalBinary() ([]byte, error) { - type headerAlias header - var buf bytes.Buffer - if err := gob.NewEncoder(&buf).Encode((*headerAlias)(h)); err != nil { - return nil, err - } - return buf.Bytes(), nil -} - -// UnmarshalBinary unmarshals the header from binary format. -func (h *header) UnmarshalBinary(data []byte) error { - type headerAlias header - return gob.NewDecoder(bytes.NewReader(data)).Decode((*headerAlias)(h)) -} diff --git a/internal/app/service/signer/signer.go b/internal/app/service/signer/signer.go deleted file mode 100644 index 09d6c31..0000000 --- a/internal/app/service/signer/signer.go +++ /dev/null @@ -1,275 +0,0 @@ -// Copyright (c) 2026, Circle Internet Group, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//go:generate mockgen -source=signer.go -destination=signer_mock.go -package=signer . Service - -// Package signer manages logic for avalanche-go external BLS signer -package signer - -import ( - "context" - "encoding/hex" - "fmt" - "sync" - - "github.com/circlefin/arc-remote-signer/internal/app/provider/awskms" - "github.com/circlefin/arc-remote-signer/internal/app/provider/secrets" - "github.com/circlefin/arc-remote-signer/internal/common/crypto" - "github.com/circlefin/arc-remote-signer/internal/common/logging" - "github.com/circlefin/arc-remote-signer/proto/pb" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" -) - -var ( - _loggerLoadOnce sync.Once - _logger *logging.Logger -) - -func getLogger() *logging.Logger { - _loggerLoadOnce.Do(func() { - _logger = logging.Get("service.signer") - }) - return _logger -} - -const ( - errSignMessage = "failed to sign message" - errInvalidRequest = "invalid request" - errEmptyMessage = "message cannot be empty" - errServiceNotInitialized = "service is not initialized" -) - -// Service is the implementation of the Signer gRPC server. -type Service struct { - pb.UnimplementedSignerServiceServer - isNitroEnclaveEnabled bool - secretPvd secrets.Provider - enclavePvd pb.EnclaveServiceClient - awskmsPvd awskms.Provider - algorithm pb.Algorithm - cache *cache -} - -// New creates a new instance of the Signer gRPC server. -func New(ctx context.Context, isNitroEnclaveEnabled bool, keyCfg *Config, secretPvd secrets.Provider, enclavePvd pb.EnclaveServiceClient, awskmsPvd awskms.Provider) (*Service, error) { - pbAlgorithm, err := toPBAlgorithm(keyCfg.Algorithm) - if err != nil { - return nil, err - } - service := &Service{ - isNitroEnclaveEnabled: isNitroEnclaveEnabled, - secretPvd: secretPvd, - enclavePvd: enclavePvd, - awskmsPvd: awskmsPvd, - algorithm: pbAlgorithm, - cache: newCache(), - } - if err := service.initialize(ctx, keyCfg); err != nil { - return nil, err - } - return service, nil -} - -func (s *Service) initialize(ctx context.Context, keyCfg *Config) error { - // retrieve the secret from the secret provider - secret, err := s.secretPvd.Get(ctx, keyCfg.KeyID) - if err != nil { - return fmt.Errorf("failed to get secret: %w", err) - } - - // if secret is empty, generate a new key, marshal it and store it in the cache - if len(secret) == 0 { - return s.generateAndStoreKey(ctx, keyCfg) - } - - // if secret is not empty, derive the public key and store it in the cache - return s.loadKeyFromSecret(ctx, secret) -} - -func (s *Service) generateAndStoreKey(ctx context.Context, keyCfg *Config) error { - /* - Nitro is enabled in environments above CI/DEV, so the recipient parameter - is included when calling AWS KMS. As a result, plainDataKey will be nil, - so we don't need to zero it out here. - */ - plainDataKey, KMSEncryptedDataKey, enclaveEncryptedDataKey, err := s.awskmsPvd.GenerateDataKey(ctx) - if err != nil { - return fmt.Errorf("failed to generate data key: %w", err) - } - - pbAlgorithm, err := toPBAlgorithm(keyCfg.Algorithm) - if err != nil { - return err - } - - extGenerateRequest := &pb.GenerateKeyRequest{ - Algorithm: pbAlgorithm, - EnclaveEncryptedDataKey: plainDataKey, - } - - if enclaveEncryptedDataKey != nil { - extGenerateRequest.EnclaveEncryptedDataKey = enclaveEncryptedDataKey - } - - resp, err := s.enclavePvd.GenerateKey(ctx, extGenerateRequest) - if err != nil { - return fmt.Errorf("failed to generate key: %w", err) - } - - hdr := header{ - CipherKey: KMSEncryptedDataKey, - CipherData: resp.EncryptedPrivateKey, - Nonce: resp.Nonce, - } - - headerBytes, err := hdr.MarshalBinary() - if err != nil { - return fmt.Errorf("failed to marshal header: %w", err) - } - - if _, err := s.secretPvd.Update(ctx, keyCfg.KeyID, headerBytes); err != nil { - return fmt.Errorf("failed to update secret: %w", err) - } - - s.cache.set(&key{ - encryptedKeyMaterial: prepareEncryptedKeyMaterial(s.isNitroEnclaveEnabled, resp.EncryptedPrivateKey, plainDataKey, enclaveEncryptedDataKey, resp.Nonce), - publicKey: resp.PublicKey, - }) - - getLogger().Info(ctx, "loaded signer public key", logging.Entries{"public_key": "0x" + hex.EncodeToString(resp.PublicKey)}) - return nil -} - -func (s *Service) loadKeyFromSecret(ctx context.Context, secret []byte) error { - hdr := header{} - if err := hdr.UnmarshalBinary(secret); err != nil { - return fmt.Errorf("failed to unmarshal header: %w", err) - } - - /* - Nitro is enabled in environments above CI/DEV, so the recipient parameter - is included when calling AWS KMS. As a result, plainDataKey will be nil, - so we don't need to zero it out here. - */ - plainDataKey, enclaveEncryptedDataKey, err := s.awskmsPvd.Decrypt(ctx, hdr.CipherKey) - if err != nil { - return fmt.Errorf("failed to decrypt data key: %w", err) - } - - encryptedKeyMaterial := prepareEncryptedKeyMaterial(s.isNitroEnclaveEnabled, hdr.CipherData, plainDataKey, enclaveEncryptedDataKey, hdr.Nonce) - extGetPublicKeyRequest := &pb.GetPublicKeyRequest{ - Algorithm: s.algorithm, - EncryptedKeyMaterial: encryptedKeyMaterial, - } - - resp, err := s.enclavePvd.GetPublicKey(ctx, extGetPublicKeyRequest) - if err != nil { - return fmt.Errorf("failed to get public key: %w", err) - } - - s.cache.set(&key{ - encryptedKeyMaterial: encryptedKeyMaterial, - publicKey: resp.PublicKey, - }) - - getLogger().Info(ctx, "loaded signer public key", logging.Entries{"public_key": "0x" + hex.EncodeToString(resp.PublicKey)}) - return nil -} - -// PublicKey returns the cached signer public key. -// It returns a gRPC internal error when the service has not been initialized. -func (s *Service) PublicKey(_ context.Context, _ *pb.PublicKeyRequest) (*pb.PublicKeyResponse, error) { - cached := s.cache.get() - if cached == nil { - return nil, status.Error(codes.Internal, errServiceNotInitialized) - } - return &pb.PublicKeyResponse{ - PublicKey: cached.publicKey, - }, nil -} - -// Sign validates the request, signs the message through the enclave provider, -// and returns the generated signature. -// It returns gRPC status errors for invalid input and uninitialized service state. -func (s *Service) Sign(ctx context.Context, req *pb.SignRequest) (*pb.SignResponse, error) { - // Validate request - if req == nil { - return nil, status.Error(codes.InvalidArgument, errInvalidRequest) - } - if len(req.Message) == 0 { - return nil, status.Error(codes.InvalidArgument, errEmptyMessage) - } - - cached := s.cache.get() - if cached == nil { - return nil, status.Error(codes.Internal, errServiceNotInitialized) - } - - // call the enclave to sign the message - resp, err := s.enclavePvd.SignMessage(ctx, &pb.SignMessageRequest{ - Algorithm: s.algorithm, - EncryptedKeyMaterial: cached.encryptedKeyMaterial, - Message: req.Message, - }) - if err != nil { - getLogger().ErrorErr(ctx, errSignMessage, err, nil) - // Preserve the original gRPC status code from the enclave - if st, ok := status.FromError(err); ok { - return nil, st.Err() - } - // Fallback to Internal if not a gRPC status error - return nil, status.Error(codes.Internal, errSignMessage) - } - // Convert the response back to protobuf format - return &pb.SignResponse{ - Signature: resp.Signature, - }, nil -} - -func prepareEncryptedKeyMaterial(isNitroEnclaveEnabled bool, encryptedPrivateKey, plainDataKey, enclaveEncryptedDataKey, nonce []byte) *pb.EncryptedKeyMaterial { - // When we use the ricipient(attestation document) parameter, instead of returning the plaintext data, KMS - // encrypts the plaintext data with the public key in the attestation document, - // and returns the resulting ciphertext in the CiphertextForRecipient field - // in the response. This ciphertext can be decrypted only with the private key - // in the enclave. The Plaintext field in the response is null or empty. - // So we use the `isNitroEnclaveEnabled` flag to determine whether to use the enclave encrypted data key or the plaintext data key. - // If we use the plaintext data key with Nitro Enclave, it will cause the decryption failure. - // - // For more details, please refer to the following document: - // https://docs.aws.amazon.com/kms/latest/developerguide/services-nitro-enclaves.html - if isNitroEnclaveEnabled { - return newEncryptedKeyMaterial(encryptedPrivateKey, enclaveEncryptedDataKey, nonce) - } - return newEncryptedKeyMaterial(encryptedPrivateKey, plainDataKey, nonce) -} - -func newEncryptedKeyMaterial(encryptedPrivateKey, dataKey, nonce []byte) *pb.EncryptedKeyMaterial { - material := &pb.EncryptedKeyMaterial{ - EncryptedPrivateKey: encryptedPrivateKey, - EnclaveEncryptedDataKey: dataKey, - Nonce: nonce, - } - return material -} - -func toPBAlgorithm(algorithm crypto.Algorithm) (pb.Algorithm, error) { - switch algorithm { - case crypto.AlgorithmBLS: - return pb.Algorithm_ALGORITHM_BLS, nil - case crypto.AlgorithmEd25519: - return pb.Algorithm_ALGORITHM_ED25519, nil - default: - return pb.Algorithm_ALGORITHM_UNSPECIFIED, fmt.Errorf("unknown algorithm: %s", algorithm) - } -} diff --git a/internal/app/service/signer/signer_mock.go b/internal/app/service/signer/signer_mock.go deleted file mode 100644 index 0a961ae..0000000 --- a/internal/app/service/signer/signer_mock.go +++ /dev/null @@ -1,5 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: signer.go - -// Package signer is a generated GoMock package. -package signer diff --git a/internal/app/service/signer/signer_test.go b/internal/app/service/signer/signer_test.go deleted file mode 100644 index 719d9d3..0000000 --- a/internal/app/service/signer/signer_test.go +++ /dev/null @@ -1,522 +0,0 @@ -// Copyright (c) 2026, Circle Internet Group, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package signer - -import ( - "context" - "errors" - "testing" - - "github.com/circlefin/arc-remote-signer/internal/app/provider/awskms" - "github.com/circlefin/arc-remote-signer/internal/app/provider/enclave" - "github.com/circlefin/arc-remote-signer/internal/app/provider/secrets" - "github.com/circlefin/arc-remote-signer/internal/common/crypto" - commonAES "github.com/circlefin/arc-remote-signer/internal/common/crypto/aes" - "github.com/circlefin/arc-remote-signer/internal/common/crypto/rand" - "github.com/circlefin/arc-remote-signer/internal/enclave/common/crypto/ed25519" - pb "github.com/circlefin/arc-remote-signer/proto/pb" - "github.com/golang/mock/gomock" - "github.com/stretchr/testify/suite" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" -) - -const ( - testKeyID = "00000000-0000-0000-0000-000000000000" - testSecretID = "test-secret-id" -) - -// SignerServiceTestSuite contains the test suite for the signer service. -type SignerServiceTestSuite struct { - suite.Suite - ctrl *gomock.Controller - mockEnclavePvd *enclave.MockEnclaveServiceClient - mockSecretsPvd *secrets.MockProvider - mockAWSKMSPvd *awskms.MockProvider - - initializedService *Service - uninitializedService *Service - testMessage []byte - testPublicKey []byte - testSignature []byte - testEncryptedKeyMaterial pb.EncryptedKeyMaterial -} - -// SetupTest runs before each test method. -func (s *SignerServiceTestSuite) SetupTest() { - s.ctrl = gomock.NewController(s.T()) - s.mockEnclavePvd = enclave.NewMockEnclaveServiceClient(s.ctrl) - s.mockSecretsPvd = secrets.NewMockProvider(s.ctrl) - s.mockAWSKMSPvd = awskms.NewMockProvider(s.ctrl) - - s.testMessage = []byte("test message") - - key, err := ed25519.New() - s.Require().NoError(err) - pub, err := key.PublicKey() - s.Require().NoError(err) - - keyBytes, err := key.Serialize() - s.Require().NoError(err) - - dataKey := rand.MustGenerateRandomBytes(32) - - encryptedDataKey, nonce, err := commonAES.EncryptGCM(dataKey, keyBytes) - s.Require().NoError(err) - - s.testEncryptedKeyMaterial = pb.EncryptedKeyMaterial{ - EncryptedPrivateKey: encryptedDataKey, - EnclaveEncryptedDataKey: dataKey, - Nonce: nonce, - } - - s.testPublicKey = pub - sig, err := key.SignMessage(s.testMessage) - s.Require().NoError(err) - s.testSignature = sig - - s.setupUninitializedService() - s.setupInitializedService() -} - -// TearDownTest runs after each test method. -func (s *SignerServiceTestSuite) TearDownTest() { - s.ctrl.Finish() -} - -func (s *SignerServiceTestSuite) TestLogger() { - logger := getLogger() - s.Require().NotNil(logger) -} - -// TestSign tests the Sign RPC method. -func (s *SignerServiceTestSuite) TestSign() { - tests := []struct { - name string - request *pb.SignRequest - mockSetup func() - wantError bool - wantCode codes.Code - wantMessage string - wantSignature []byte - service *Service - }{ - { - name: "successful sign", - request: &pb.SignRequest{ - Message: []byte(s.testMessage), - }, - mockSetup: func() { - s.mockEnclavePvd.EXPECT(). - SignMessage(gomock.Any(), &pb.SignMessageRequest{ - Algorithm: pb.Algorithm_ALGORITHM_ED25519, - EncryptedKeyMaterial: &s.testEncryptedKeyMaterial, - Message: s.testMessage, - }). - Return(&pb.SignMessageResponse{ - Signature: s.testSignature, - }, nil). - Times(1) - }, - wantError: false, - wantCode: codes.OK, - wantMessage: "", - wantSignature: s.testSignature, - service: s.initializedService, - }, - { - name: "nil request", - request: nil, - mockSetup: func() {}, - wantError: true, - wantCode: codes.InvalidArgument, - wantMessage: errInvalidRequest, - wantSignature: nil, - service: s.initializedService, - }, - { - name: "empty message", - request: &pb.SignRequest{Message: []byte{}}, - mockSetup: func() {}, - wantError: true, - wantCode: codes.InvalidArgument, - wantMessage: errEmptyMessage, - wantSignature: nil, - service: s.initializedService, - }, - { - name: "nil message", - request: &pb.SignRequest{Message: nil}, - mockSetup: func() {}, - wantError: true, - wantCode: codes.InvalidArgument, - wantMessage: errEmptyMessage, - wantSignature: nil, - service: s.initializedService, - }, - { - name: "enclave provider error", - request: &pb.SignRequest{ - Message: s.testMessage, - }, - mockSetup: func() { - s.mockEnclavePvd.EXPECT(). - SignMessage(gomock.Any(), &pb.SignMessageRequest{ - Algorithm: pb.Algorithm_ALGORITHM_ED25519, - EncryptedKeyMaterial: &s.testEncryptedKeyMaterial, - Message: s.testMessage, - }). - Return(nil, errors.New("enclave error")). - Times(1) - }, - wantError: true, - wantCode: codes.Unknown, - wantMessage: "failed to sign message", - wantSignature: nil, - service: s.initializedService, - }, - { - name: "sign message with uninitialized service", - request: &pb.SignRequest{ - Message: s.testMessage, - }, - mockSetup: func() {}, - wantError: true, - wantCode: codes.Internal, - wantMessage: "service is not initialized", - wantSignature: nil, - service: s.uninitializedService, - }, - } - - for _, tt := range tests { - s.Run(tt.name, func() { - tt.mockSetup() - resp, err := tt.service.Sign(context.Background(), tt.request) - if tt.wantError { - s.Require().Error(err) - s.Require().Nil(resp) - if tt.wantCode == codes.Unknown { - s.Require().Contains(err.Error(), tt.wantMessage) - } else { - grpcErr, ok := status.FromError(err) - s.Require().True(ok) - s.Require().Equal(tt.wantCode, grpcErr.Code()) - s.Require().Contains(grpcErr.Message(), tt.wantMessage) - } - } else { - s.Require().NoError(err) - s.Require().NotNil(resp) - s.Require().Equal([]byte(tt.wantSignature), resp.Signature) - } - }) - } -} - -// TestPublicKey tests the PublicKey RPC method. -func (s *SignerServiceTestSuite) TestPublicKey() { - tests := []struct { - name string - wantError bool - wantCode codes.Code - wantMessage string - wantPublic []byte - service *Service - }{ - { - name: "successful public key retrieval", - wantError: false, - wantCode: codes.OK, - wantMessage: "", - wantPublic: s.testPublicKey, - service: s.initializedService, - }, - { - name: "retrieval with uninitialized service", - wantError: true, - wantCode: codes.Internal, - wantMessage: "service is not initialized", - wantPublic: nil, - service: s.uninitializedService, - }, - } - - for _, tt := range tests { - s.Run(tt.name, func() { - ctrl := gomock.NewController(s.T()) - defer ctrl.Finish() - - resp, err := tt.service.PublicKey(context.Background(), &pb.PublicKeyRequest{}) - if tt.wantError { - s.Require().Error(err) - s.Require().Nil(resp) - grpcErr, ok := status.FromError(err) - s.Require().True(ok) - s.Require().Equal(tt.wantCode, grpcErr.Code()) - s.Require().Contains(grpcErr.Message(), tt.wantMessage) - } else { - s.Require().NoError(err) - s.Require().NotNil(resp) - s.Require().Equal(tt.wantPublic, resp.PublicKey) - } - }) - } -} - -// TestServiceInitialization tests the service initialization logic. -func (s *SignerServiceTestSuite) TestServiceInitialization() { - hdr := header{ - CipherKey: s.testEncryptedKeyMaterial.EnclaveEncryptedDataKey, - CipherData: s.testEncryptedKeyMaterial.EncryptedPrivateKey, - Nonce: s.testEncryptedKeyMaterial.Nonce, - } - - headerBytes, err := hdr.MarshalBinary() - if err != nil { - s.Require().NoError(err) - } - - tests := []struct { - name string - secretValue []byte - mockSetup func() - wantError bool - wantMessage string - }{ - { - name: "initialize with existing secret", - secretValue: headerBytes, - mockSetup: func() { - s.mockSecretsPvd.EXPECT(). - Get(gomock.Any(), testKeyID). - Return(headerBytes, nil). - Times(1) - - s.mockAWSKMSPvd.EXPECT().Decrypt(gomock.Any(), s.testEncryptedKeyMaterial.EnclaveEncryptedDataKey).Return(s.testEncryptedKeyMaterial.EnclaveEncryptedDataKey, s.testEncryptedKeyMaterial.EnclaveEncryptedDataKey, nil) - - s.mockEnclavePvd.EXPECT(). - GetPublicKey(gomock.Any(), &pb.GetPublicKeyRequest{ - Algorithm: pb.Algorithm_ALGORITHM_ED25519, - EncryptedKeyMaterial: &s.testEncryptedKeyMaterial, - }). - Return(&pb.GetPublicKeyResponse{ - PublicKey: s.testPublicKey, - }, nil). - Times(1) - }, - wantError: false, - wantMessage: "", - }, - { - name: "get secret error", - secretValue: []byte{}, - mockSetup: func() { - s.mockSecretsPvd.EXPECT(). - Get(gomock.Any(), testKeyID). - Return(nil, errors.New("secrets error")). - Times(1) - }, - wantError: true, - wantMessage: "failed to get secret", - }, - { - name: "get public key error", - secretValue: headerBytes, - mockSetup: func() { - s.mockSecretsPvd.EXPECT(). - Get(gomock.Any(), testKeyID). - Return(headerBytes, nil). - Times(1) - - s.mockAWSKMSPvd.EXPECT().Decrypt(gomock.Any(), s.testEncryptedKeyMaterial.EnclaveEncryptedDataKey).Return(s.testEncryptedKeyMaterial.EnclaveEncryptedDataKey, s.testEncryptedKeyMaterial.EnclaveEncryptedDataKey, nil) - - s.mockEnclavePvd.EXPECT(). - GetPublicKey(gomock.Any(), &pb.GetPublicKeyRequest{ - Algorithm: pb.Algorithm_ALGORITHM_ED25519, - EncryptedKeyMaterial: &s.testEncryptedKeyMaterial, - }). - Return(nil, errors.New("get public key error")). - Times(1) - }, - wantError: true, - wantMessage: "failed to get public key", - }, - { - name: "decrypt data key error", - secretValue: headerBytes, - mockSetup: func() { - s.mockSecretsPvd.EXPECT(). - Get(gomock.Any(), testKeyID). - Return(headerBytes, nil). - Times(1) - - s.mockAWSKMSPvd.EXPECT().Decrypt(gomock.Any(), s.testEncryptedKeyMaterial.EnclaveEncryptedDataKey).Return(nil, nil, errors.New("decrypt data key error")) - }, - wantError: true, - wantMessage: "failed to decrypt data key", - }, - { - name: "key generation error", - secretValue: []byte{}, - mockSetup: func() { - s.mockSecretsPvd.EXPECT(). - Get(gomock.Any(), testKeyID). - Return([]byte{}, nil). - Times(1) - - s.mockAWSKMSPvd.EXPECT(). - GenerateDataKey(gomock.Any()). - Return(s.testEncryptedKeyMaterial.EnclaveEncryptedDataKey, s.testEncryptedKeyMaterial.EnclaveEncryptedDataKey, s.testEncryptedKeyMaterial.EnclaveEncryptedDataKey, nil). - Times(1) - - s.mockEnclavePvd.EXPECT(). - GenerateKey(gomock.Any(), &pb.GenerateKeyRequest{ - Algorithm: pb.Algorithm_ALGORITHM_ED25519, - EnclaveEncryptedDataKey: s.testEncryptedKeyMaterial.EnclaveEncryptedDataKey, - }). - Return(nil, errors.New("generation error")). - Times(1) - }, - wantError: true, - wantMessage: "failed to generate key", - }, - { - name: "update secret error", - secretValue: []byte{}, - mockSetup: func() { - s.mockSecretsPvd.EXPECT(). - Get(gomock.Any(), testKeyID). - Return([]byte{}, nil). - Times(1) - - s.mockAWSKMSPvd.EXPECT(). - GenerateDataKey(gomock.Any()). - Return(s.testEncryptedKeyMaterial.EnclaveEncryptedDataKey, s.testEncryptedKeyMaterial.EnclaveEncryptedDataKey, s.testEncryptedKeyMaterial.EnclaveEncryptedDataKey, nil). - Times(1) - - s.mockEnclavePvd.EXPECT(). - GenerateKey(gomock.Any(), &pb.GenerateKeyRequest{ - Algorithm: pb.Algorithm_ALGORITHM_ED25519, - EnclaveEncryptedDataKey: s.testEncryptedKeyMaterial.EnclaveEncryptedDataKey, - }). - Return(&pb.GenerateKeyResponse{ - EncryptedPrivateKey: s.testEncryptedKeyMaterial.EncryptedPrivateKey, - PublicKey: s.testPublicKey, - Nonce: s.testEncryptedKeyMaterial.Nonce, - }, nil) - - s.mockSecretsPvd.EXPECT(). - Update(gomock.Any(), testKeyID, headerBytes). - Return("", errors.New("update secret error")). - Times(1) - }, - wantError: true, - wantMessage: "failed to update secret", - }, - { - name: "generate data key error", - secretValue: []byte{}, - mockSetup: func() { - s.mockSecretsPvd.EXPECT(). - Get(gomock.Any(), testKeyID). - Return([]byte{}, nil). - Times(1) - - s.mockAWSKMSPvd.EXPECT(). - GenerateDataKey(gomock.Any()). - Return(nil, nil, nil, errors.New("generate data key error")). - Times(1) - }, - wantError: true, - wantMessage: "failed to generate data key", - }, - } - - for _, tt := range tests { - s.Run(tt.name, func() { - ctrl := gomock.NewController(s.T()) - defer ctrl.Finish() - config := &Config{ - Algorithm: crypto.AlgorithmEd25519, - KeyID: testKeyID, - } - tt.mockSetup() - service, err := New(context.Background(), false, config, s.mockSecretsPvd, s.mockEnclavePvd, s.mockAWSKMSPvd) - if tt.wantError { - s.Require().Error(err) - s.Require().Nil(service) - s.Require().Contains(err.Error(), tt.wantMessage) - } else { - s.Require().NoError(err) - s.Require().NotNil(service) - } - }) - } -} - -func (s *SignerServiceTestSuite) setupUninitializedService() { - s.uninitializedService = &Service{ - secretPvd: s.mockSecretsPvd, - enclavePvd: s.mockEnclavePvd, - awskmsPvd: s.mockAWSKMSPvd, - algorithm: pb.Algorithm_ALGORITHM_ED25519, - cache: newCache(), - } -} - -func (s *SignerServiceTestSuite) setupInitializedService() { - s.mockSecretsPvd.EXPECT(). - Get(gomock.Any(), testKeyID). - Return(nil, nil) - - s.mockAWSKMSPvd.EXPECT(). - GenerateDataKey(gomock.Any()). - Return(s.testEncryptedKeyMaterial.EnclaveEncryptedDataKey, s.testEncryptedKeyMaterial.EncryptedPrivateKey, s.testEncryptedKeyMaterial.EnclaveEncryptedDataKey, nil) - - s.mockEnclavePvd.EXPECT(). - GenerateKey(gomock.Any(), &pb.GenerateKeyRequest{ - Algorithm: pb.Algorithm_ALGORITHM_ED25519, - EnclaveEncryptedDataKey: s.testEncryptedKeyMaterial.EnclaveEncryptedDataKey, - }). - Return(&pb.GenerateKeyResponse{ - EncryptedPrivateKey: s.testEncryptedKeyMaterial.EncryptedPrivateKey, - PublicKey: s.testPublicKey, - Nonce: s.testEncryptedKeyMaterial.Nonce, - }, nil) - - hdr := header{ - CipherKey: s.testEncryptedKeyMaterial.EncryptedPrivateKey, - CipherData: s.testEncryptedKeyMaterial.EncryptedPrivateKey, - Nonce: s.testEncryptedKeyMaterial.Nonce, - } - - headerBytes, err := hdr.MarshalBinary() - if err != nil { - s.Require().NoError(err) - } - - s.mockSecretsPvd.EXPECT(). - Update(gomock.Any(), testKeyID, headerBytes). - Return(testSecretID, nil) - - cfg := NewConfig() - initializedService, err := New(context.Background(), false, cfg, s.mockSecretsPvd, s.mockEnclavePvd, s.mockAWSKMSPvd) - s.Require().NoError(err) - s.initializedService = initializedService -} - -// Run the test suite. -func TestSignerServiceTestSuite(t *testing.T) { - suite.Run(t, new(SignerServiceTestSuite)) -} diff --git a/internal/app/service/signer/v1/attestation.go b/internal/app/service/signer/v1/attestation.go new file mode 100644 index 0000000..2ee222c --- /dev/null +++ b/internal/app/service/signer/v1/attestation.go @@ -0,0 +1,125 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +import ( + "crypto/x509" + "errors" + "fmt" + "time" + + "github.com/fxamacker/cbor/v2" +) + +const ( + coseSign1Tag = 18 + attestationRefreshRetryFloor = 30 * time.Second + attestationRefreshRetryCeiling = 5 * time.Minute + attestationRefreshNumerator = 4 + attestationRefreshDivisor = 5 +) + +type attestationValidity struct { + notBefore time.Time + notAfter time.Time +} + +func (v attestationValidity) validateAt(now time.Time) error { + if now.Before(v.notBefore) { + return errors.New(errAttestationNotValidYet) + } + if !now.Before(v.notAfter) { + return errors.New(errAttestationExpired) + } + return nil +} + +type attestationPayload struct { + Certificate []byte `cbor:"certificate"` + CABundle [][]byte `cbor:"cabundle"` +} + +func parseAttestationValidity(document []byte) (attestationValidity, error) { + var tag cbor.RawTag + err := cbor.Unmarshal(document, &tag) + if err == nil { + if tag.Number != coseSign1Tag { + return attestationValidity{}, fmt.Errorf("unexpected COSE tag: %d", tag.Number) + } + document = tag.Content + } else { + var typeError *cbor.UnmarshalTypeError + if !errors.As(err, &typeError) { + return attestationValidity{}, fmt.Errorf("failed to decode COSE tag: %w", err) + } + } + + var sign1 []cbor.RawMessage + if err := cbor.Unmarshal(document, &sign1); err != nil { + return attestationValidity{}, fmt.Errorf("failed to decode COSE Sign1: %w", err) + } + if len(sign1) != 4 { + return attestationValidity{}, errors.New("COSE Sign1 must contain four fields") + } + + var payloadBytes []byte + if err := cbor.Unmarshal(sign1[2], &payloadBytes); err != nil { + return attestationValidity{}, fmt.Errorf("failed to decode attestation payload: %w", err) + } + var payload attestationPayload + if err := cbor.Unmarshal(payloadBytes, &payload); err != nil { + return attestationValidity{}, fmt.Errorf("failed to decode attestation fields: %w", err) + } + if len(payload.Certificate) == 0 { + return attestationValidity{}, errors.New("attestation certificate is empty") + } + + certificateData := make([][]byte, 0, len(payload.CABundle)+1) + certificateData = append(certificateData, payload.Certificate) + certificateData = append(certificateData, payload.CABundle...) + + var validity attestationValidity + for _, data := range certificateData { + certificate, err := x509.ParseCertificate(data) + if err != nil { + return attestationValidity{}, fmt.Errorf("failed to parse attestation certificate: %w", err) + } + if validity.notBefore.IsZero() || certificate.NotBefore.After(validity.notBefore) { + validity.notBefore = certificate.NotBefore + } + if validity.notAfter.IsZero() || certificate.NotAfter.Before(validity.notAfter) { + validity.notAfter = certificate.NotAfter + } + } + return validity, nil +} + +func attestationRefreshDelay(now time.Time, validity attestationValidity, retry bool) time.Duration { + if !retry { + lifetime := validity.notAfter.Sub(validity.notBefore) + refreshAt := validity.notBefore.Add(lifetime * attestationRefreshNumerator / attestationRefreshDivisor) + if now.Before(refreshAt) { + return refreshAt.Sub(now) + } + } + + delay := validity.notAfter.Sub(now) / 2 + if delay < attestationRefreshRetryFloor { + return attestationRefreshRetryFloor + } + if delay > attestationRefreshRetryCeiling { + return attestationRefreshRetryCeiling + } + return delay +} diff --git a/internal/app/service/signer/v1/attestation_test.go b/internal/app/service/signer/v1/attestation_test.go new file mode 100644 index 0000000..0cf6de2 --- /dev/null +++ b/internal/app/service/signer/v1/attestation_test.go @@ -0,0 +1,128 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +import ( + "crypto/ed25519" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "math/big" + "testing" + "time" + + "github.com/fxamacker/cbor/v2" + "github.com/stretchr/testify/require" +) + +func TestParseAttestationValidityUsesCertificateChainIntersection(t *testing.T) { + now := time.Date(2026, time.August, 11, 10, 0, 0, 0, time.UTC) + target := testCertificate(t, now.Add(-time.Hour), now.Add(12*time.Hour), 1) + intermediate := testCertificate(t, now.Add(-2*time.Hour), now.Add(6*time.Hour), 2) + root := testCertificate(t, now.Add(-24*time.Hour), now.Add(365*24*time.Hour), 3) + document := testAttestationDocument(t, target, intermediate, root) + + validity, err := parseAttestationValidity(document) + + require.NoError(t, err) + require.Equal(t, target.NotBefore, validity.notBefore) + require.Equal(t, intermediate.NotAfter, validity.notAfter) +} + +func TestParseAttestationValidityAcceptsUntaggedCOSESign1(t *testing.T) { + now := time.Date(2026, time.August, 11, 10, 0, 0, 0, time.UTC) + certificate := testCertificate(t, now.Add(-time.Hour), now.Add(12*time.Hour), 4) + taggedDocument := testAttestationDocument(t, certificate) + var tag cbor.RawTag + require.NoError(t, cbor.Unmarshal(taggedDocument, &tag)) + + validity, err := parseAttestationValidity(tag.Content) + + require.NoError(t, err) + require.Equal(t, certificate.NotBefore, validity.notBefore) + require.Equal(t, certificate.NotAfter, validity.notAfter) +} + +func TestParseAttestationValidityRejectsMalformedDocument(t *testing.T) { + _, err := parseAttestationValidity([]byte("not-cbor")) + + require.ErrorContains(t, err, "failed to decode COSE tag") +} + +func TestParseAttestationValidityRejectsUnexpectedCOSETag(t *testing.T) { + document, err := cbor.Marshal(cbor.RawTag{Number: 17, Content: []byte{}}) + require.NoError(t, err) + + _, err = parseAttestationValidity(document) + + require.ErrorContains(t, err, "unexpected COSE tag: 17") +} + +func TestAttestationRefreshDelayUsesCertificateLifetime(t *testing.T) { + validity := attestationValidity{ + notBefore: time.Date(2026, time.August, 11, 0, 0, 0, 0, time.UTC), + notAfter: time.Date(2026, time.August, 11, 10, 0, 0, 0, time.UTC), + } + + delay := attestationRefreshDelay(validity.notBefore.Add(7*time.Hour), validity, false) + + require.Equal(t, time.Hour, delay) +} + +func TestAttestationRefreshDelayBoundsRetries(t *testing.T) { + validity := attestationValidity{ + notBefore: time.Date(2026, time.August, 11, 0, 0, 0, 0, time.UTC), + notAfter: time.Date(2026, time.August, 11, 10, 0, 0, 0, time.UTC), + } + + require.Equal(t, attestationRefreshRetryCeiling, attestationRefreshDelay(validity.notAfter.Add(-time.Hour), validity, true)) + require.Equal(t, attestationRefreshRetryFloor, attestationRefreshDelay(validity.notAfter.Add(-time.Second), validity, true)) +} + +func testCertificate(t *testing.T, notBefore, notAfter time.Time, serial int64) *x509.Certificate { + t.Helper() + + publicKey, privateKey, err := ed25519.GenerateKey(rand.Reader) + require.NoError(t, err) + template := &x509.Certificate{ + SerialNumber: big.NewInt(serial), + Subject: pkix.Name{CommonName: "test-attestation"}, + NotBefore: notBefore, + NotAfter: notAfter, + } + der, err := x509.CreateCertificate(rand.Reader, template, template, publicKey, privateKey) + require.NoError(t, err) + certificate, err := x509.ParseCertificate(der) + require.NoError(t, err) + return certificate +} + +func testAttestationDocument(t *testing.T, certificate *x509.Certificate, caBundle ...*x509.Certificate) []byte { + t.Helper() + + encodedBundle := make([][]byte, 0, len(caBundle)) + for _, cert := range caBundle { + encodedBundle = append(encodedBundle, cert.Raw) + } + payload, err := cbor.Marshal(map[string]any{ + "certificate": certificate.Raw, + "cabundle": encodedBundle, + }) + require.NoError(t, err) + sign1, err := cbor.Marshal([]any{[]byte{}, map[string]any{}, payload, []byte("signature")}) + require.NoError(t, err) + document, err := cbor.Marshal(cbor.RawTag{Number: 18, Content: sign1}) + require.NoError(t, err) + return document +} diff --git a/internal/app/service/signer/config.go b/internal/app/service/signer/v1/config.go similarity index 98% rename from internal/app/service/signer/config.go rename to internal/app/service/signer/v1/config.go index cc5c0c0..ca48d39 100644 --- a/internal/app/service/signer/config.go +++ b/internal/app/service/signer/v1/config.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package signer +package v1 import "github.com/circlefin/arc-remote-signer/internal/common/crypto" diff --git a/internal/app/service/signer/v1/crypto.go b/internal/app/service/signer/v1/crypto.go new file mode 100644 index 0000000..934cabf --- /dev/null +++ b/internal/app/service/signer/v1/crypto.go @@ -0,0 +1,78 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +import ( + "bytes" + "encoding/gob" + + "github.com/circlefin/arc-remote-signer/proto/pb" +) + +// header represents the AES encrypted data. +// It contains the algorithm, cipher key, cipher data, and nonce. +// +// The stored-secret format is preserved across the KMS-in-enclave migration: +// CipherKey/CipherData/Nonce map 1:1 onto the SecretEnvelope's +// kms_encrypted_data_key/encrypted_private_key/nonce, so existing validator +// secrets load without re-registration. +// +// Algorithm records the algorithm the key was generated under so a later reload +// can fail cleanly on a config mismatch instead of silently signing under the +// wrong one. It is gob-backward-compatible: secrets written before this field +// decode as ALGORITHM_UNSPECIFIED. +type header struct { + Algorithm pb.Algorithm + CipherKey []byte + CipherData []byte + Nonce []byte +} + +// toSecretEnvelope maps a stored header to the proto SecretEnvelope the +// enclave decrypts in-enclave. +func (h *header) toSecretEnvelope(alg pb.Algorithm) *pb.SecretEnvelope { + return &pb.SecretEnvelope{ + Algorithm: alg, + KmsEncryptedDataKey: h.CipherKey, + EncryptedPrivateKey: h.CipherData, + Nonce: h.Nonce, + } +} + +// headerFromSecretEnvelope maps an enclave-produced SecretEnvelope back to +// the stored header format for persistence in Secrets Manager. +func headerFromSecretEnvelope(env *pb.SecretEnvelope) *header { + return &header{ + Algorithm: env.GetAlgorithm(), + CipherKey: env.GetKmsEncryptedDataKey(), + CipherData: env.GetEncryptedPrivateKey(), + Nonce: env.GetNonce(), + } +} + +// MarshalBinary marshals the header to binary format. +func (h *header) MarshalBinary() ([]byte, error) { + type headerAlias header + var buf bytes.Buffer + if err := gob.NewEncoder(&buf).Encode((*headerAlias)(h)); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// UnmarshalBinary unmarshals the header from binary format. +func (h *header) UnmarshalBinary(data []byte) error { + type headerAlias header + return gob.NewDecoder(bytes.NewReader(data)).Decode((*headerAlias)(h)) +} diff --git a/internal/app/service/signer/crypto_test.go b/internal/app/service/signer/v1/crypto_test.go similarity index 99% rename from internal/app/service/signer/crypto_test.go rename to internal/app/service/signer/v1/crypto_test.go index 6762fdd..b23ed59 100644 --- a/internal/app/service/signer/crypto_test.go +++ b/internal/app/service/signer/v1/crypto_test.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package signer +package v1 import ( "bytes" diff --git a/internal/app/service/signer/v1/signer.go b/internal/app/service/signer/v1/signer.go new file mode 100644 index 0000000..83c10ee --- /dev/null +++ b/internal/app/service/signer/v1/signer.go @@ -0,0 +1,384 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package v1 implements the version 1 signer service and key lifecycle. +package v1 + +import ( + "bytes" + "context" + "encoding/hex" + "errors" + "fmt" + "sync" + "time" + + "github.com/circlefin/arc-remote-signer/internal/app/provider/secrets" + "github.com/circlefin/arc-remote-signer/internal/common/crypto" + "github.com/circlefin/arc-remote-signer/internal/common/logging" + "github.com/circlefin/arc-remote-signer/proto/pb" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +var ( + _loggerLoadOnce sync.Once + _logger *logging.Logger +) + +func getLogger() *logging.Logger { + _loggerLoadOnce.Do(func() { + _logger = logging.Get("service.signer") + }) + return _logger +} + +const ( + errSignMessage = "failed to sign message" + errInvalidRequest = "invalid request" + errEmptyMessage = "message cannot be empty" + errServiceNotInitialized = "service is not initialized" + errAttestationExpired = "attestation document is expired" + errAttestationNotValidYet = "attestation certificate is not valid yet" +) + +// Service is the implementation of the Signer gRPC server. +type Service struct { + pb.UnimplementedSignerServiceServer + enclavePvd pb.EnclaveServiceClient + loadedKeyMu sync.RWMutex + loadedKey *keyState +} + +// InitializeKeyFunc installs or recovers the enclave key. +type InitializeKeyFunc func( + context.Context, + *pb.SecretEnvelope, + pb.Algorithm, +) (*pb.InitializeResponse, error) + +// keyState contains public response data for the installed enclave key. +type keyState struct { + publicKey []byte + attestationDocument []byte + attestationValidity attestationValidity +} + +// New creates a new instance of the Signer gRPC server. +// +// The enclave creates an ephemeral KMS client for each initialization attempt. +// It completes key generation or recovery before it publishes the Ready state. +// The host persists a generated SecretEnvelope and does not call KMS directly. +func New( + ctx context.Context, + keyCfg *Config, + secretPvd secrets.Provider, + enclavePvd pb.EnclaveServiceClient, + initializeKey InitializeKeyFunc, +) (*Service, error) { + pbAlgorithm, err := toPBAlgorithm(keyCfg.Algorithm) + if err != nil { + return nil, err + } + if initializeKey == nil { + return nil, errors.New("enclave initializer is required") + } + loadedKey, err := initializeKeyState(ctx, keyCfg, secretPvd, pbAlgorithm, initializeKey) + if err != nil { + return nil, err + } + service := &Service{ + enclavePvd: enclavePvd, + loadedKey: loadedKey, + } + service.startAttestationRefresh(ctx) + return service, nil +} + +func initializeKeyState( + ctx context.Context, + keyCfg *Config, + secretPvd secrets.Provider, + algorithm pb.Algorithm, + initializeKey InitializeKeyFunc, +) (*keyState, error) { + secret, err := secretPvd.Get(ctx, keyCfg.KeyID) + if err != nil { + return nil, fmt.Errorf("failed to get secret: %w", err) + } + + if len(secret) == 0 { + return generateAndStoreKey(ctx, keyCfg, secretPvd, algorithm, initializeKey) + } + + return loadKeyFromSecret(ctx, secret, algorithm, initializeKey) +} + +func generateAndStoreKey( + ctx context.Context, + keyCfg *Config, + secretPvd secrets.Provider, + algorithm pb.Algorithm, + initializeKey InitializeKeyFunc, +) (*keyState, error) { + resp, err := initializeKey(ctx, nil, algorithm) + if err != nil { + return nil, fmt.Errorf("failed to initialize generated key: %w", err) + } + if resp == nil || resp.SecretEnvelope == nil { + return nil, errors.New("enclave Initialize returned no secret envelope") + } + if len(resp.PublicKey) == 0 { + return nil, errors.New("enclave Initialize returned empty public key") + } + env := resp.SecretEnvelope + if env.Algorithm != algorithm { + return nil, fmt.Errorf("generated key algorithm %s does not match configured algorithm %s", env.Algorithm, algorithm) + } + if len(env.GetKmsEncryptedDataKey()) == 0 || len(env.GetEncryptedPrivateKey()) == 0 || len(env.GetNonce()) == 0 { + return nil, errors.New("enclave Initialize returned incomplete secret envelope") + } + + headerBytes, err := headerFromSecretEnvelope(resp.SecretEnvelope).MarshalBinary() + if err != nil { + return nil, fmt.Errorf("failed to marshal header: %w", err) + } + loadedKey, err := newKeyState(resp) + if err != nil { + return nil, err + } + + if _, err := secretPvd.Update(ctx, keyCfg.KeyID, headerBytes); err != nil { + return nil, fmt.Errorf("failed to update secret: %w", err) + } + + getLogger().Info(ctx, "loaded signer public key", logging.Entries{"public_key": "0x" + hex.EncodeToString(resp.PublicKey)}) + return loadedKey, nil +} + +func loadKeyFromSecret( + ctx context.Context, + secret []byte, + algorithm pb.Algorithm, + initializeKey InitializeKeyFunc, +) (*keyState, error) { + hdr := header{} + if err := hdr.UnmarshalBinary(secret); err != nil { + return nil, fmt.Errorf("failed to unmarshal header: %w", err) + } + + // The stored header records the algorithm the key was generated under. Fail + // cleanly on a config/key mismatch instead of silently signing under the + // wrong algorithm. Secrets written before this field decode as + // ALGORITHM_UNSPECIFIED and fall through to the configured algorithm. + if hdr.Algorithm != pb.Algorithm_ALGORITHM_UNSPECIFIED && hdr.Algorithm != algorithm { + return nil, fmt.Errorf("stored key algorithm %s does not match configured algorithm %s", hdr.Algorithm, algorithm) + } + + envelope := hdr.toSecretEnvelope(algorithm) + resp, err := initializeKey(ctx, envelope, pb.Algorithm_ALGORITHM_UNSPECIFIED) + if err != nil { + return nil, fmt.Errorf("failed to initialize existing key: %w", err) + } + if resp == nil { + return nil, errors.New("enclave Initialize returned no response") + } + if resp.SecretEnvelope != nil { + return nil, errors.New("enclave Initialize returned an envelope for an existing key") + } + + loadedKey, err := newKeyState(resp) + if err != nil { + return nil, err + } + + getLogger().Info(ctx, "loaded signer public key", logging.Entries{"public_key": "0x" + hex.EncodeToString(resp.PublicKey)}) + return loadedKey, nil +} + +func newKeyState(resp *pb.InitializeResponse) (*keyState, error) { + if resp == nil || len(resp.PublicKey) == 0 { + return nil, errors.New("enclave Initialize returned empty public key") + } + var validity attestationValidity + if len(resp.AttestationDocument) > 0 { + parsedValidity, err := parseAttestationValidity(resp.AttestationDocument) + if err != nil { + return nil, fmt.Errorf("failed to parse attestation document: %w", err) + } + if err := parsedValidity.validateAt(time.Now()); err != nil { + return nil, err + } + validity = parsedValidity + } + + return &keyState{ + publicKey: bytes.Clone(resp.PublicKey), + attestationDocument: bytes.Clone(resp.AttestationDocument), + attestationValidity: validity, + }, nil +} + +func (s *Service) refreshAttestation(ctx context.Context) (attestationValidity, error) { + s.loadedKeyMu.RLock() + if s.loadedKey == nil { + s.loadedKeyMu.RUnlock() + return attestationValidity{}, errors.New(errServiceNotInitialized) + } + publicKey := bytes.Clone(s.loadedKey.publicKey) + currentValidity := s.loadedKey.attestationValidity + s.loadedKeyMu.RUnlock() + + resp, err := s.getPublicKey(ctx) + if err != nil { + return currentValidity, err + } + if !bytes.Equal(publicKey, resp.PublicKey) { + return currentValidity, errors.New("enclave returned a different public key") + } + if len(resp.AttestationDocument) == 0 { + return currentValidity, errors.New("enclave returned an empty attestation document") + } + validity, err := parseAttestationValidity(resp.AttestationDocument) + if err != nil { + return currentValidity, fmt.Errorf("failed to parse attestation document: %w", err) + } + if err := validity.validateAt(time.Now()); err != nil { + return currentValidity, err + } + if !validity.notAfter.After(currentValidity.notAfter) { + return currentValidity, nil + } + + s.loadedKeyMu.Lock() + if s.loadedKey != nil && bytes.Equal(s.loadedKey.publicKey, publicKey) { + s.loadedKey.attestationDocument = bytes.Clone(resp.AttestationDocument) + s.loadedKey.attestationValidity = validity + } + s.loadedKeyMu.Unlock() + return validity, nil +} + +func (s *Service) startAttestationRefresh(ctx context.Context) { + s.loadedKeyMu.RLock() + if s.loadedKey == nil || s.loadedKey.attestationValidity.notAfter.IsZero() { + s.loadedKeyMu.RUnlock() + return + } + validity := s.loadedKey.attestationValidity + s.loadedKeyMu.RUnlock() + + go s.runAttestationRefresh(ctx, validity) +} + +func (s *Service) runAttestationRefresh(ctx context.Context, validity attestationValidity) { + retry := false + for { + delay := attestationRefreshDelay(time.Now(), validity, retry) + timer := time.NewTimer(delay) + select { + case <-ctx.Done(): + if !timer.Stop() { + select { + case <-timer.C: + default: + } + } + return + case <-timer.C: + } + + refreshedValidity, err := s.refreshAttestation(ctx) + if err != nil { + getLogger().ErrorErr(ctx, "failed to refresh attestation document", err, nil) + retry = true + continue + } + retry = !refreshedValidity.notAfter.After(validity.notAfter) + validity = refreshedValidity + } +} + +func (s *Service) getPublicKey(ctx context.Context) (*pb.GetPublicKeyResponse, error) { + resp, err := s.enclavePvd.GetPublicKey(ctx, &pb.GetPublicKeyRequest{}) + if err != nil { + return nil, fmt.Errorf("failed to get public key: %w", err) + } + if resp == nil || len(resp.PublicKey) == 0 { + return nil, errors.New("enclave GetPublicKey returned empty public key") + } + return resp, nil +} + +// PublicKey returns the cached key and a valid attestation document. +func (s *Service) PublicKey(_ context.Context, _ *pb.PublicKeyRequest) (*pb.PublicKeyResponse, error) { + s.loadedKeyMu.RLock() + defer s.loadedKeyMu.RUnlock() + if s.loadedKey == nil { + return nil, status.Error(codes.Internal, errServiceNotInitialized) + } + if !s.loadedKey.attestationValidity.notAfter.IsZero() { + if err := s.loadedKey.attestationValidity.validateAt(time.Now()); err != nil { + return nil, status.Error(codes.Unavailable, err.Error()) + } + } + + return &pb.PublicKeyResponse{ + PublicKey: bytes.Clone(s.loadedKey.publicKey), + AttestationDocument: bytes.Clone(s.loadedKey.attestationDocument), + }, nil +} + +// Sign validates the request, signs the message through the enclave provider, +// and returns the generated signature. +// It returns gRPC status errors for invalid input and uninitialized service state. +func (s *Service) Sign(ctx context.Context, req *pb.SignRequest) (*pb.SignResponse, error) { + // Validate request + if req == nil { + return nil, status.Error(codes.InvalidArgument, errInvalidRequest) + } + if len(req.Message) == 0 { + return nil, status.Error(codes.InvalidArgument, errEmptyMessage) + } + + s.loadedKeyMu.RLock() + if s.loadedKey == nil { + s.loadedKeyMu.RUnlock() + return nil, status.Error(codes.Internal, errServiceNotInitialized) + } + s.loadedKeyMu.RUnlock() + + // call the enclave to sign the message + resp, err := s.enclavePvd.SignMessage(ctx, &pb.SignMessageRequest{ + Message: req.Message, + }) + if err != nil { + getLogger().ErrorErr(ctx, errSignMessage, err, nil) + return nil, status.Error(codes.Internal, errSignMessage) + } + // Convert the response back to protobuf format + return &pb.SignResponse{ + Signature: resp.Signature, + }, nil +} + +func toPBAlgorithm(algorithm crypto.Algorithm) (pb.Algorithm, error) { + switch algorithm { + case crypto.AlgorithmBLS: + return pb.Algorithm_ALGORITHM_BLS, nil + case crypto.AlgorithmEd25519: + return pb.Algorithm_ALGORITHM_ED25519, nil + default: + return pb.Algorithm_ALGORITHM_UNSPECIFIED, fmt.Errorf("unknown algorithm: %s", algorithm) + } +} diff --git a/internal/app/service/signer/v1/signer_test.go b/internal/app/service/signer/v1/signer_test.go new file mode 100644 index 0000000..cc7bd03 --- /dev/null +++ b/internal/app/service/signer/v1/signer_test.go @@ -0,0 +1,467 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +import ( + "bytes" + "context" + "errors" + "testing" + "time" + + enclaveProvider "github.com/circlefin/arc-remote-signer/internal/app/provider/enclave" + "github.com/circlefin/arc-remote-signer/internal/app/provider/secrets" + "github.com/circlefin/arc-remote-signer/internal/common/crypto" + "github.com/circlefin/arc-remote-signer/proto/pb" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func testEnvelope() *pb.SecretEnvelope { + return &pb.SecretEnvelope{ + Algorithm: pb.Algorithm_ALGORITHM_ED25519, + KmsEncryptedDataKey: []byte("kms-ciphertext"), + EncryptedPrivateKey: []byte("private-ciphertext"), + Nonce: []byte("nonce"), + } +} + +func TestNew_GeneratesAndPersistsThroughInitialize(t *testing.T) { + ctrl := gomock.NewController(t) + secretPvd := secrets.NewMockProvider(ctrl) + enclavePvd := enclaveProvider.NewMockEnclaveServiceClient(ctrl) + envelope := testEnvelope() + publicKey := []byte("public-key") + cfg := &Config{Algorithm: crypto.AlgorithmEd25519, KeyID: "key-id"} + headerBytes, err := headerFromSecretEnvelope(envelope).MarshalBinary() + require.NoError(t, err) + secretPvd.EXPECT().Get(gomock.Any(), cfg.KeyID).Return(nil, nil) + secretPvd.EXPECT().Update(gomock.Any(), cfg.KeyID, headerBytes).Return("secret-id", nil) + initialize := func( + _ context.Context, + existing *pb.SecretEnvelope, + generate pb.Algorithm, + ) (*pb.InitializeResponse, error) { + require.Nil(t, existing) + require.Equal(t, pb.Algorithm_ALGORITHM_ED25519, generate) + return &pb.InitializeResponse{PublicKey: publicKey, SecretEnvelope: envelope}, nil + } + + service, err := New(context.Background(), cfg, secretPvd, enclavePvd, initialize) + + require.NoError(t, err) + resp, err := service.PublicKey(context.Background(), &pb.PublicKeyRequest{}) + require.NoError(t, err) + require.Equal(t, publicKey, resp.GetPublicKey()) +} + +func TestNew_RecoversStoredKeyThroughInitialize(t *testing.T) { + ctrl := gomock.NewController(t) + secretPvd := secrets.NewMockProvider(ctrl) + enclavePvd := enclaveProvider.NewMockEnclaveServiceClient(ctrl) + envelope := testEnvelope() + stored, err := headerFromSecretEnvelope(envelope).MarshalBinary() + require.NoError(t, err) + cfg := &Config{Algorithm: crypto.AlgorithmEd25519, KeyID: "key-id"} + secretPvd.EXPECT().Get(gomock.Any(), cfg.KeyID).Return(stored, nil) + initialize := func( + _ context.Context, + existing *pb.SecretEnvelope, + generate pb.Algorithm, + ) (*pb.InitializeResponse, error) { + require.Equal(t, envelope, existing) + require.Equal(t, pb.Algorithm_ALGORITHM_UNSPECIFIED, generate) + return &pb.InitializeResponse{PublicKey: []byte("public-key")}, nil + } + + service, err := New(context.Background(), cfg, secretPvd, enclavePvd, initialize) + + require.NoError(t, err) + require.NotNil(t, service) +} + +func TestNew_InitializationFailureDoesNotPublishService(t *testing.T) { + ctrl := gomock.NewController(t) + secretPvd := secrets.NewMockProvider(ctrl) + enclavePvd := enclaveProvider.NewMockEnclaveServiceClient(ctrl) + cfg := &Config{Algorithm: crypto.AlgorithmEd25519, KeyID: "key-id"} + secretPvd.EXPECT().Get(gomock.Any(), cfg.KeyID).Return(nil, nil) + initialize := func( + context.Context, + *pb.SecretEnvelope, + pb.Algorithm, + ) (*pb.InitializeResponse, error) { + return nil, errors.New("initialize failed") + } + + service, err := New(context.Background(), cfg, secretPvd, enclavePvd, initialize) + + require.Nil(t, service) + require.ErrorContains(t, err, "initialize failed") +} + +func TestSign_ForwardsOnlyMessageAfterInitialization(t *testing.T) { + ctrl := gomock.NewController(t) + enclavePvd := enclaveProvider.NewMockEnclaveServiceClient(ctrl) + service := &Service{ + enclavePvd: enclavePvd, + loadedKey: &keyState{publicKey: []byte("public-key")}, + } + message := []byte("message") + signature := []byte("signature") + enclavePvd.EXPECT(). + SignMessage(gomock.Any(), &pb.SignMessageRequest{Message: message}). + Return(&pb.SignMessageResponse{Signature: signature}, nil) + + resp, err := service.Sign(context.Background(), &pb.SignRequest{Message: message}) + + require.NoError(t, err) + require.Equal(t, signature, resp.GetSignature()) +} + +func TestSign_ValidatesRequestAndReadyState(t *testing.T) { + service := &Service{} + _, err := service.Sign(context.Background(), nil) + require.Equal(t, codes.InvalidArgument, status.Code(err)) + _, err = service.Sign(context.Background(), &pb.SignRequest{}) + require.Equal(t, codes.InvalidArgument, status.Code(err)) + _, err = service.Sign(context.Background(), &pb.SignRequest{Message: []byte("message")}) + require.Equal(t, codes.Internal, status.Code(err)) +} + +func TestRefreshAttestation_UpdatesCachedDocument(t *testing.T) { + ctrl := gomock.NewController(t) + enclavePvd := enclaveProvider.NewMockEnclaveServiceClient(ctrl) + now := time.Now() + currentCertificate := testCertificate(t, now.Add(-time.Hour), now.Add(12*time.Hour), 20) + newCertificate := testCertificate(t, now.Add(-time.Hour), now.Add(24*time.Hour), 21) + currentDocument := testAttestationDocument(t, currentCertificate) + newDocument := testAttestationDocument(t, newCertificate) + currentValidity, err := parseAttestationValidity(currentDocument) + require.NoError(t, err) + publicKey := []byte("public-key") + service := &Service{ + enclavePvd: enclavePvd, + loadedKey: &keyState{ + publicKey: publicKey, + attestationDocument: currentDocument, + attestationValidity: currentValidity, + }, + } + enclavePvd.EXPECT(). + GetPublicKey(gomock.Any(), &pb.GetPublicKeyRequest{}). + Return(&pb.GetPublicKeyResponse{ + PublicKey: publicKey, + AttestationDocument: newDocument, + }, nil) + + validity, err := service.refreshAttestation(context.Background()) + + require.NoError(t, err) + require.Equal(t, newCertificate.NotAfter, validity.notAfter) + response, err := service.PublicKey(context.Background(), &pb.PublicKeyRequest{}) + require.NoError(t, err) + require.Equal(t, newDocument, response.GetAttestationDocument()) +} + +func TestNew_RejectsMalformedAttestation(t *testing.T) { + ctrl := gomock.NewController(t) + secretPvd := secrets.NewMockProvider(ctrl) + enclavePvd := enclaveProvider.NewMockEnclaveServiceClient(ctrl) + cfg := &Config{Algorithm: crypto.AlgorithmEd25519, KeyID: "key-id"} + secretPvd.EXPECT().Get(gomock.Any(), cfg.KeyID).Return(nil, nil) + initialize := func( + context.Context, + *pb.SecretEnvelope, + pb.Algorithm, + ) (*pb.InitializeResponse, error) { + return &pb.InitializeResponse{ + PublicKey: []byte("public-key"), + AttestationDocument: []byte("not-cbor"), + SecretEnvelope: testEnvelope(), + }, nil + } + + service, err := New(context.Background(), cfg, secretPvd, enclavePvd, initialize) + + require.Nil(t, service) + require.ErrorContains(t, err, "failed to parse attestation document") +} + +func TestNew_RejectsExpiredAttestation(t *testing.T) { + ctrl := gomock.NewController(t) + secretPvd := secrets.NewMockProvider(ctrl) + enclavePvd := enclaveProvider.NewMockEnclaveServiceClient(ctrl) + cfg := &Config{Algorithm: crypto.AlgorithmEd25519, KeyID: "key-id"} + now := time.Now() + expiredCertificate := testCertificate(t, now.Add(-2*time.Hour), now.Add(-time.Hour), 22) + expiredDocument := testAttestationDocument(t, expiredCertificate) + secretPvd.EXPECT().Get(gomock.Any(), cfg.KeyID).Return(nil, nil) + initialize := func( + context.Context, + *pb.SecretEnvelope, + pb.Algorithm, + ) (*pb.InitializeResponse, error) { + return &pb.InitializeResponse{ + PublicKey: []byte("public-key"), + AttestationDocument: expiredDocument, + SecretEnvelope: testEnvelope(), + }, nil + } + + service, err := New(context.Background(), cfg, secretPvd, enclavePvd, initialize) + + require.Nil(t, service) + require.ErrorContains(t, err, errAttestationExpired) +} + +func TestNew_RejectsStoredAlgorithmMismatchBeforeInitialize(t *testing.T) { + ctrl := gomock.NewController(t) + secretPvd := secrets.NewMockProvider(ctrl) + enclavePvd := enclaveProvider.NewMockEnclaveServiceClient(ctrl) + envelope := testEnvelope() + mismatched, err := (&header{ + Algorithm: pb.Algorithm_ALGORITHM_BLS, + CipherKey: envelope.KmsEncryptedDataKey, + CipherData: envelope.EncryptedPrivateKey, + Nonce: envelope.Nonce, + }).MarshalBinary() + require.NoError(t, err) + cfg := &Config{Algorithm: crypto.AlgorithmEd25519, KeyID: "key-id"} + secretPvd.EXPECT().Get(gomock.Any(), cfg.KeyID).Return(mismatched, nil) + initializeCalled := false + initialize := func( + context.Context, + *pb.SecretEnvelope, + pb.Algorithm, + ) (*pb.InitializeResponse, error) { + initializeCalled = true + return nil, nil + } + + service, err := New(context.Background(), cfg, secretPvd, enclavePvd, initialize) + + require.Nil(t, service) + require.ErrorContains(t, err, "does not match configured algorithm") + require.False(t, initializeCalled) +} + +func TestNew_RejectsIncompleteGeneratedEnvelope(t *testing.T) { + ctrl := gomock.NewController(t) + secretPvd := secrets.NewMockProvider(ctrl) + enclavePvd := enclaveProvider.NewMockEnclaveServiceClient(ctrl) + cfg := &Config{Algorithm: crypto.AlgorithmEd25519, KeyID: "key-id"} + secretPvd.EXPECT().Get(gomock.Any(), cfg.KeyID).Return(nil, nil) + incomplete := testEnvelope() + incomplete.Nonce = nil + initialize := func( + context.Context, + *pb.SecretEnvelope, + pb.Algorithm, + ) (*pb.InitializeResponse, error) { + return &pb.InitializeResponse{ + PublicKey: []byte("public-key"), + SecretEnvelope: incomplete, + }, nil + } + + service, err := New(context.Background(), cfg, secretPvd, enclavePvd, initialize) + + require.Nil(t, service) + require.ErrorContains(t, err, "incomplete secret envelope") +} + +func TestNew_SecretsManagerUpdateFailureDoesNotPublishService(t *testing.T) { + ctrl := gomock.NewController(t) + secretPvd := secrets.NewMockProvider(ctrl) + enclavePvd := enclaveProvider.NewMockEnclaveServiceClient(ctrl) + envelope := testEnvelope() + stored, err := headerFromSecretEnvelope(envelope).MarshalBinary() + require.NoError(t, err) + cfg := &Config{Algorithm: crypto.AlgorithmEd25519, KeyID: "key-id"} + secretPvd.EXPECT().Get(gomock.Any(), cfg.KeyID).Return(nil, nil) + secretPvd.EXPECT(). + Update(gomock.Any(), cfg.KeyID, stored). + Return("", errors.New("update failed")) + initialize := func( + context.Context, + *pb.SecretEnvelope, + pb.Algorithm, + ) (*pb.InitializeResponse, error) { + return &pb.InitializeResponse{ + PublicKey: []byte("public-key"), + SecretEnvelope: envelope, + }, nil + } + + service, err := New(context.Background(), cfg, secretPvd, enclavePvd, initialize) + + require.Nil(t, service) + require.ErrorContains(t, err, "failed to update secret") +} + +func TestPublicKey_ReturnsImmutableCachedAttestation(t *testing.T) { + publicKey := []byte("public-key") + document := []byte("attestation") + service := &Service{ + loadedKey: &keyState{ + publicKey: bytes.Clone(publicKey), + attestationDocument: bytes.Clone(document), + }, + } + + first, err := service.PublicKey(context.Background(), &pb.PublicKeyRequest{}) + require.NoError(t, err) + first.PublicKey[0] ^= 0xff + first.AttestationDocument[0] ^= 0xff + second, err := service.PublicKey(context.Background(), &pb.PublicKeyRequest{}) + + require.NoError(t, err) + require.Equal(t, publicKey, second.GetPublicKey()) + require.Equal(t, document, second.GetAttestationDocument()) +} + +func TestPublicKey_RejectsUninitializedService(t *testing.T) { + response, err := (&Service{}).PublicKey(context.Background(), &pb.PublicKeyRequest{}) + + require.Nil(t, response) + require.Equal(t, codes.Internal, status.Code(err)) + require.Equal(t, errServiceNotInitialized, status.Convert(err).Message()) +} + +func TestPublicKey_RejectsExpiredAttestation(t *testing.T) { + service := &Service{ + loadedKey: &keyState{ + publicKey: []byte("public-key"), + attestationValidity: attestationValidity{ + notBefore: time.Now().Add(-2 * time.Hour), + notAfter: time.Now().Add(-time.Second), + }, + }, + } + + response, err := service.PublicKey(context.Background(), &pb.PublicKeyRequest{}) + + require.Nil(t, response) + require.Equal(t, codes.Unavailable, status.Code(err)) + require.Equal(t, errAttestationExpired, status.Convert(err).Message()) +} + +func TestSign_SanitizesEnclaveError(t *testing.T) { + ctrl := gomock.NewController(t) + enclavePvd := enclaveProvider.NewMockEnclaveServiceClient(ctrl) + service := &Service{ + enclavePvd: enclavePvd, + loadedKey: &keyState{publicKey: []byte("public-key")}, + } + message := []byte("message") + enclavePvd.EXPECT(). + SignMessage(gomock.Any(), &pb.SignMessageRequest{Message: message}). + Return(nil, status.Error(codes.PermissionDenied, "sensitive enclave details")) + + response, err := service.Sign(context.Background(), &pb.SignRequest{Message: message}) + + require.Nil(t, response) + require.Equal(t, codes.Internal, status.Code(err)) + require.Equal(t, errSignMessage, status.Convert(err).Message()) + require.NotContains(t, status.Convert(err).Message(), "sensitive enclave details") +} + +func TestRefreshAttestation_KeepsCacheOnPublicKeyMismatch(t *testing.T) { + ctrl := gomock.NewController(t) + enclavePvd := enclaveProvider.NewMockEnclaveServiceClient(ctrl) + document := []byte("current-attestation") + currentValidity := attestationValidity{notAfter: time.Now().Add(24 * time.Hour)} + service := &Service{ + enclavePvd: enclavePvd, + loadedKey: &keyState{ + publicKey: []byte("public-key"), + attestationDocument: document, + attestationValidity: currentValidity, + }, + } + enclavePvd.EXPECT(). + GetPublicKey(gomock.Any(), &pb.GetPublicKeyRequest{}). + Return(&pb.GetPublicKeyResponse{ + PublicKey: []byte("different-public-key"), + AttestationDocument: document, + }, nil) + + validity, err := service.refreshAttestation(context.Background()) + + require.ErrorContains(t, err, "different public key") + require.Equal(t, currentValidity, validity) + response, responseErr := service.PublicKey(context.Background(), &pb.PublicKeyRequest{}) + require.NoError(t, responseErr) + require.Equal(t, document, response.GetAttestationDocument()) +} + +func TestRefreshAttestation_KeepsCurrentExpiryWhenNewExpiryIsEarlier(t *testing.T) { + ctrl := gomock.NewController(t) + enclavePvd := enclaveProvider.NewMockEnclaveServiceClient(ctrl) + now := time.Now() + currentCertificate := testCertificate(t, now.Add(-time.Hour), now.Add(24*time.Hour), 23) + shorterCertificate := testCertificate(t, now.Add(-time.Hour), now.Add(12*time.Hour), 24) + currentDocument := testAttestationDocument(t, currentCertificate) + shorterDocument := testAttestationDocument(t, shorterCertificate) + currentValidity, err := parseAttestationValidity(currentDocument) + require.NoError(t, err) + service := &Service{ + enclavePvd: enclavePvd, + loadedKey: &keyState{ + publicKey: []byte("public-key"), + attestationDocument: currentDocument, + attestationValidity: currentValidity, + }, + } + enclavePvd.EXPECT(). + GetPublicKey(gomock.Any(), &pb.GetPublicKeyRequest{}). + Return(&pb.GetPublicKeyResponse{ + PublicKey: []byte("public-key"), + AttestationDocument: shorterDocument, + }, nil) + + validity, err := service.refreshAttestation(context.Background()) + + require.NoError(t, err) + require.Equal(t, currentValidity, validity) + response, responseErr := service.PublicKey(context.Background(), &pb.PublicKeyRequest{}) + require.NoError(t, responseErr) + require.Equal(t, currentDocument, response.GetAttestationDocument()) +} + +func TestGetPublicKey_RejectsEmptyResponse(t *testing.T) { + tests := map[string]*pb.GetPublicKeyResponse{ + "nil response": nil, + "empty public key": {}, + } + + for name, response := range tests { + t.Run(name, func(t *testing.T) { + ctrl := gomock.NewController(t) + enclavePvd := enclaveProvider.NewMockEnclaveServiceClient(ctrl) + service := &Service{enclavePvd: enclavePvd} + enclavePvd.EXPECT(). + GetPublicKey(gomock.Any(), &pb.GetPublicKeyRequest{}). + Return(response, nil) + + result, err := service.getPublicKey(context.Background()) + + require.Nil(t, result) + require.ErrorContains(t, err, "enclave GetPublicKey returned empty public key") + }) + } +} diff --git a/internal/common/byteproxy/awsproxy.go b/internal/common/byteproxy/awsproxy.go new file mode 100644 index 0000000..bce462d --- /dev/null +++ b/internal/common/byteproxy/awsproxy.go @@ -0,0 +1,56 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package byteproxy + +import "fmt" + +// AWSServiceBinding is what a wrapper supplies per service: an informational +// Endpoint (logged on dial failure) plus the Listen and Dial closures that +// the framework will invoke. Wrappers close over the port and any other +// transport-specific state when constructing the binding. +type AWSServiceBinding struct { + Endpoint string + Listen ListenFunc + Dial DialFunc +} + +// BindingFactory is the wrapper-supplied callback that produces one binding +// per service. NewAWSProxy iterates AWSServiceNames and calls this for each +// (name, port) pair. Returning an error aborts construction. +type BindingFactory func(name string, port uint32) (AWSServiceBinding, error) + +// NewAWSProxy builds a Proxy over AWSServiceNames by asking the wrapper for +// one binding per service. Wrappers (awsproxy on the enclave, vsockproxy on +// the host) share this so the services loop, port assignment, and Config +// assembly cannot drift between the two sides of the bridge. +// +// Ports are assigned as basePort + index-in-AWSServiceNames, matching the +// hand-rolled implementations that this helper replaces. +func NewAWSProxy(name string, basePort uint32, maxConns int, bf BindingFactory) (Proxy, error) { + services := make([]ServiceConfig, 0, len(AWSServiceNames)) + for i, svc := range AWSServiceNames { + port := basePort + uint32(i) //nolint:gosec // bounded by len(AWSServiceNames) + b, err := bf(svc, port) + if err != nil { + return nil, fmt.Errorf("byteproxy: binding for service %q: %w", svc, err) + } + services = append(services, ServiceConfig{ + Name: svc, + Endpoint: b.Endpoint, + Listen: b.Listen, + Dial: b.Dial, + }) + } + return New(&Config{Name: name, Services: services, MaxConns: maxConns}) +} diff --git a/internal/common/byteproxy/awsproxy_test.go b/internal/common/byteproxy/awsproxy_test.go new file mode 100644 index 0000000..493f516 --- /dev/null +++ b/internal/common/byteproxy/awsproxy_test.go @@ -0,0 +1,111 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package byteproxy + +import ( + "context" + "errors" + "net" + "testing" + + "github.com/stretchr/testify/suite" +) + +// AWSProxyTestSuite covers NewAWSProxy's per-service iteration, port +// assignment, binding plumbing, and factory-error propagation. The full +// proxy lifecycle (accept/bridge/shutdown) is covered by ByteProxyTestSuite; +// here we only verify the helper that constructs the per-service Config. +type AWSProxyTestSuite struct { + suite.Suite +} + +// newLocalListener returns a TCP listener bound to a random loopback port. +func (s *AWSProxyTestSuite) newLocalListener() net.Listener { + s.T().Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + s.Require().NoError(err) + s.T().Cleanup(func() { _ = ln.Close() }) + return ln +} + +// TestNewAWSProxy_PortAssignmentAndIteration verifies that the factory is +// invoked once per AWSServiceNames entry, in order, with port = basePort + +// index. Endpoint, Listen, and Dial flow through to the underlying proxy. +func (s *AWSProxyTestSuite) TestNewAWSProxy_PortAssignmentAndIteration() { + s.Require().NotEmpty(AWSServiceNames, "AWSServiceNames must declare at least one service") + + const base uint32 = 50000 + + var seenNames []string + var seenPorts []uint32 + bf := func(name string, port uint32) (AWSServiceBinding, error) { + seenNames = append(seenNames, name) + seenPorts = append(seenPorts, port) + return AWSServiceBinding{ + Endpoint: "stub-endpoint", + Listen: func() (net.Listener, error) { return s.newLocalListener(), nil }, + Dial: func(_ context.Context, _ net.Conn) (net.Conn, error) { return nil, errors.New("dial unused") }, + }, nil + } + + p, err := NewAWSProxy("unit-test", base, 4, bf) + s.Require().NoError(err) + s.T().Cleanup(func() { _ = p.Shutdown(context.Background()) }) + s.Equal("unit-test", p.Name()) + + wantPorts := make([]uint32, len(AWSServiceNames)) + for i := range AWSServiceNames { + wantPorts[i] = base + uint32(i) //nolint:gosec // bounded by len(AWSServiceNames) + } + s.Equal(AWSServiceNames, seenNames, "factory must be called once per service in order") + s.Equal(wantPorts, seenPorts, "ports must be basePort + index-in-AWSServiceNames") +} + +// TestNewAWSProxy_FactoryError verifies the factory's error is wrapped with +// the service name and surfaced β€” and that the underlying proxy is not +// constructed (no listeners leak). +func (s *AWSProxyTestSuite) TestNewAWSProxy_FactoryError() { + s.Require().NotEmpty(AWSServiceNames) + + sentinel := errors.New("binding failed for this test") + bf := func(_ string, _ uint32) (AWSServiceBinding, error) { + return AWSServiceBinding{}, sentinel + } + + p, err := NewAWSProxy("unit-test", 50000, 4, bf) + s.Require().Error(err) + s.Require().ErrorIs(err, sentinel) + s.Contains(err.Error(), AWSServiceNames[0], "error must identify which service's binding failed") + s.Nil(p) +} + +// TestNewAWSProxy_PropagatesNewError verifies that downstream New errors +// (e.g. invalid MaxConns) surface from NewAWSProxy unchanged. +func (s *AWSProxyTestSuite) TestNewAWSProxy_PropagatesNewError() { + bf := func(_ string, _ uint32) (AWSServiceBinding, error) { + return AWSServiceBinding{ + Endpoint: "stub", + Listen: func() (net.Listener, error) { return s.newLocalListener(), nil }, + Dial: func(_ context.Context, _ net.Conn) (net.Conn, error) { return nil, errors.New("dial unused") }, + }, nil + } + p, err := NewAWSProxy("unit-test", 50000, 0, bf) // MaxConns=0 must fail + s.Require().Error(err) + s.Nil(p) +} + +// TestAWSProxyTestSuite is the entry point for the suite. +func TestAWSProxyTestSuite(t *testing.T) { + suite.Run(t, &AWSProxyTestSuite{Suite: suite.Suite{}}) +} diff --git a/internal/common/byteproxy/awsroute.go b/internal/common/byteproxy/awsroute.go new file mode 100644 index 0000000..594b83d --- /dev/null +++ b/internal/common/byteproxy/awsroute.go @@ -0,0 +1,89 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package byteproxy + +import ( + "bytes" + "errors" + "fmt" + "io" +) + +const ( + awsRouteMagic = "ARSP" + awsRouteVersion = byte(1) + awsRoutePrefix = len(awsRouteMagic) + 3 +) + +// AWSRoute identifies the AWS service and region selected by an enclave SDK +// client. It is sent before the untouched TLS or HTTP stream. +type AWSRoute struct { + Service string + Region string +} + +// WriteAWSRoute writes one bounded routing header to w. +func WriteAWSRoute(w io.Writer, route AWSRoute) error { + if route.Service == "" || len(route.Service) > 255 { + return errors.New("byteproxy: AWS route service must contain 1-255 bytes") + } + if route.Region == "" || len(route.Region) > 255 { + return errors.New("byteproxy: AWS route region must contain 1-255 bytes") + } + header := make([]byte, 0, awsRoutePrefix+len(route.Service)+len(route.Region)) + header = append(header, awsRouteMagic...) + header = append(header, awsRouteVersion, byte(len(route.Service)), byte(len(route.Region))) + header = append(header, route.Service...) + header = append(header, route.Region...) + n, err := w.Write(header) + if err != nil { + return fmt.Errorf("byteproxy: write AWS route: %w", err) + } + if n != len(header) { + return fmt.Errorf( + "byteproxy: write AWS route: wrote %d of %d bytes: %w", + n, len(header), io.ErrShortWrite, + ) + } + return nil +} + +// ReadAWSRoute reads exactly one routing header from r and leaves the +// following TLS or HTTP bytes unread. On error, bytes already read from r are +// not restored. +func ReadAWSRoute(r io.Reader) (AWSRoute, error) { + prefix := make([]byte, awsRoutePrefix) + if _, err := io.ReadFull(r, prefix); err != nil { + return AWSRoute{}, fmt.Errorf("byteproxy: read AWS route prefix: %w", err) + } + if !bytes.Equal(prefix[:len(awsRouteMagic)], []byte(awsRouteMagic)) { + return AWSRoute{}, errors.New("byteproxy: invalid magic in AWS route header") + } + if prefix[len(awsRouteMagic)] != awsRouteVersion { + return AWSRoute{}, fmt.Errorf("byteproxy: unsupported AWS route version %d", prefix[len(awsRouteMagic)]) + } + serviceLen := int(prefix[len(awsRouteMagic)+1]) + regionLen := int(prefix[len(awsRouteMagic)+2]) + if serviceLen == 0 || regionLen == 0 { + return AWSRoute{}, errors.New("byteproxy: AWS route service and region must be non-empty") + } + payload := make([]byte, serviceLen+regionLen) + if _, err := io.ReadFull(r, payload); err != nil { + return AWSRoute{}, fmt.Errorf("byteproxy: read AWS route payload: %w", err) + } + return AWSRoute{ + Service: string(payload[:serviceLen]), + Region: string(payload[serviceLen:]), + }, nil +} diff --git a/internal/common/byteproxy/awsroute_test.go b/internal/common/byteproxy/awsroute_test.go new file mode 100644 index 0000000..bef331c --- /dev/null +++ b/internal/common/byteproxy/awsroute_test.go @@ -0,0 +1,120 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package byteproxy + +import ( + "bytes" + "errors" + "io" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestAWSRouteRoundTripPreservesPayload(t *testing.T) { + var wire bytes.Buffer + want := AWSRoute{Service: "kms", Region: "us-west-2"} + + require.NoError(t, WriteAWSRoute(&wire, want)) + wire.WriteString("tls-client-hello") + + got, err := ReadAWSRoute(&wire) + require.NoError(t, err) + require.Equal(t, want, got) + require.Equal(t, "tls-client-hello", wire.String()) +} + +func TestReadAWSRouteRejectsInvalidMagic(t *testing.T) { + _, err := ReadAWSRoute(strings.NewReader("NOPE\x01\x03\x09kmsus-west-2")) + + require.ErrorContains(t, err, "invalid magic") +} + +func TestReadAWSRouteRejectsMalformedHeaders(t *testing.T) { + tests := map[string]struct { + wire string + message string + }{ + "truncated prefix": { + wire: "ARSP", + message: "read AWS route prefix", + }, + "unsupported version": { + wire: "ARSP\x02\x03\x09kmsus-west-2", + message: "unsupported AWS route version", + }, + "empty field": { + wire: "ARSP\x01\x00\x09", + message: "must be non-empty", + }, + "truncated payload": { + wire: "ARSP\x01\x03\x09kmsus-west", + message: "read AWS route payload", + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + _, err := ReadAWSRoute(strings.NewReader(tc.wire)) + require.ErrorContains(t, err, tc.message) + }) + } +} + +func TestWriteAWSRouteRejectsInvalidFields(t *testing.T) { + tests := map[string]AWSRoute{ + "empty service": {Region: "us-west-2"}, + "oversized service": {Service: strings.Repeat("a", 256), Region: "us-west-2"}, + "empty region": {Service: "kms"}, + "oversized region": {Service: "kms", Region: strings.Repeat("a", 256)}, + } + + for name, route := range tests { + t.Run(name, func(t *testing.T) { + err := WriteAWSRoute(&bytes.Buffer{}, route) + require.Error(t, err) + }) + } +} + +func TestWriteAWSRoutePropagatesWriterFailure(t *testing.T) { + err := WriteAWSRoute(errorWriter{err: errors.New("write failed")}, AWSRoute{ + Service: "kms", + Region: "us-west-2", + }) + + require.ErrorContains(t, err, "write failed") +} + +func TestWriteAWSRouteRejectsShortWrite(t *testing.T) { + err := WriteAWSRoute(shortWriter{}, AWSRoute{Service: "kms", Region: "us-west-2"}) + + require.ErrorIs(t, err, io.ErrShortWrite) + require.ErrorContains(t, err, "byteproxy: write AWS route") +} + +type shortWriter struct{} + +type errorWriter struct { + err error +} + +func (w errorWriter) Write([]byte) (int, error) { + return 0, w.err +} + +func (shortWriter) Write(p []byte) (int, error) { + return len(p) - 1, nil +} diff --git a/internal/common/byteproxy/byteproxy.go b/internal/common/byteproxy/byteproxy.go new file mode 100644 index 0000000..af69c66 --- /dev/null +++ b/internal/common/byteproxy/byteproxy.go @@ -0,0 +1,387 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package byteproxy provides a small framework for spawning per-service +// listeners that bidirectionally bridge each accepted connection to an +// upstream dialed by the caller. The accept loop, per-connection bridge, +// bounded concurrency, and shutdown semantics live here so that +// transport-specific wrappers only declare how to bind and how to dial. +package byteproxy + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "sync" + "sync/atomic" + + "github.com/circlefin/arc-remote-signer/internal/common/lifecycle" + "github.com/circlefin/arc-remote-signer/internal/common/logging" +) + +// ListenFunc returns the inbound listener for one service slot. +type ListenFunc func() (net.Listener, error) + +// DialFunc opens an upstream connection for one bridged session. inbound is +// the accepted connection and may be read before bridging to consume routing +// metadata owned by this hop. Bytes read from inbound are consumed and are not +// copied upstream, so intermediate hops must leave downstream metadata unread. +// The ctx is cancelled when the proxy starts shutting down; transports that +// can be cancelled mid-dial (e.g. net.Dialer.DialContext) MUST honor it so a +// hung upstream does not pin Shutdown past its deadline. Transports whose Dial +// cannot accept a ctx (e.g. mdlayher/vsock at v1.2.1) SHOULD race the dial +// against ctx.Done() in the wrapper and close any resulting conn if ctx wins. +type DialFunc func(ctx context.Context, inbound net.Conn) (net.Conn, error) + +// ServiceConfig describes one bridge slot. +type ServiceConfig struct { + // Name is informational; used as a log field. + Name string + + // Endpoint is informational; logged on dial failure so operators can + // distinguish endpoint misconfiguration from transport failures. The + // framework does not consume it for routing β€” Dial alone produces the + // upstream connection. + Endpoint string + + // Listen returns the inbound listener. + Listen ListenFunc + + // Dial opens the upstream connection for each accepted session. + Dial DialFunc +} + +// Config configures a multi-service proxy. +type Config struct { + // Name is the identifier returned by lifecycle.Runnable.Name and used + // as the base for the logger namespace. + Name string + + // Services declares one slot per service. Order is informational. + Services []ServiceConfig + + // MaxConns caps in-flight bridged connections per service. Must be > 0. + MaxConns int +} + +// Proxy is the lifecycle.Runnable returned by New. The extra DroppedCount +// method is exposed for observability hooks and tests; consumers that only +// want lifecycle semantics can assign to a lifecycle.Runnable. +// +// Lifecycle contract: Run must complete before Shutdown is invoked, and +// Run is called at most once. lifecycle.Manager satisfies this by calling +// Run synchronously for all runnables before waiting on signals, then +// calling Shutdown sequentially. Direct callers must follow the same +// ordering; interleaving Run and Shutdown from separate goroutines is +// undefined. +type Proxy interface { + lifecycle.Runnable + + // DroppedCount returns the cumulative number of inbound connections + // dropped for `service` because its per-service concurrency cap was + // already at MaxConns. Returns 0 for unknown service names. + DroppedCount(service string) int64 +} + +// proxy holds one listener per service and supervises their accept loops. +type proxy struct { + name string + logger *logging.Logger + listeners []*serviceListener + wg sync.WaitGroup + + // shutdownCh is closed first in Shutdown to gate acceptLoop's post-Accept + // check before any kernel-queued conn is dispatched. + shutdownCh chan struct{} + shutOnce sync.Once + + // dialCtx is handed to every DialFunc invocation; dialCancel is fired by + // Shutdown so transports that honor ctx (e.g. net.Dialer.DialContext) and + // wrappers that race vsock.Dial against ctx.Done() can abort in-flight + // dials instead of holding the WaitGroup open past Shutdown's deadline. + dialCtx context.Context + dialCancel context.CancelFunc +} + +type serviceListener struct { + ServiceConfig + ln net.Listener + sem chan struct{} + dropped atomic.Int64 +} + +// New creates a multi-service byte proxy. The caller is responsible for +// gating the call on platform availability (vsock requires Linux); New does +// not inspect runtime capabilities. +func New(cfg *Config) (Proxy, error) { + if cfg == nil { + return nil, errors.New("byteproxy: nil config") + } + if cfg.Name == "" { + return nil, errors.New("byteproxy: Name must be set") + } + if len(cfg.Services) == 0 { + return nil, errors.New("byteproxy: at least one service required") + } + if cfg.MaxConns <= 0 { + return nil, errors.New("byteproxy: MaxConns must be > 0") + } + + dialCtx, dialCancel := context.WithCancel(context.Background()) + p := &proxy{ + name: cfg.Name, + logger: logging.Get(cfg.Name + ".byteproxy"), + shutdownCh: make(chan struct{}), + dialCtx: dialCtx, + dialCancel: dialCancel, + } + + for _, svc := range cfg.Services { + if svc.Listen == nil || svc.Dial == nil { + p.closeListeners() + dialCancel() + return nil, fmt.Errorf("byteproxy: service %q missing Listen or Dial", svc.Name) + } + ln, err := svc.Listen() + if err != nil { + p.closeListeners() + dialCancel() + return nil, fmt.Errorf("byteproxy: bind for service %s: %w", svc.Name, err) + } + p.listeners = append(p.listeners, &serviceListener{ + ServiceConfig: svc, + ln: ln, + sem: make(chan struct{}, cfg.MaxConns), + }) + } + return p, nil +} + +// closeListeners closes every listener currently held by p. Used both by +// New to roll back partially-bound listeners on a construction error, and +// by Shutdown to tear down once the lifecycle ends. +// +// Caller is responsible for any ordering against shutdownCh; this method +// does not signal shutdown on its own. +func (p *proxy) closeListeners() { + for _, sl := range p.listeners { + _ = sl.ln.Close() + } +} + +// Name implements lifecycle.Runnable. +func (p *proxy) Name() string { return p.name } + +// Run spawns accept loops in background goroutines and returns immediately. +// This matches the lifecycle.Runnable contract used by other services in +// this repo (gRPC server, metrics server) where lifecycle.Manager calls Run +// synchronously and the actual serving runs in a goroutine. Callers wait +// for full drain via Shutdown. +// +// Run is not safe to call concurrently with Shutdown; see the Proxy +// interface godoc for the required ordering. +func (p *proxy) Run() error { + for _, sl := range p.listeners { + p.wg.Add(1) + go p.acceptLoop(context.Background(), sl) + } + return nil +} + +// Shutdown closes listeners, cancels in-flight dials, and waits for bridge +// goroutines to drain. If ctx expires first, Shutdown returns ctx.Err() β€” +// bridge goroutines whose DialFunc ignores ctx will outlive this call. +// Idempotent; must not run concurrently with Run. +func (p *proxy) Shutdown(ctx context.Context) error { + p.shutOnce.Do(func() { + // Close shutdownCh first so acceptLoop's post-Accept gate observes + // the signal before any kernel-queued conn slips through. The + // listener close below races against in-flight Accept calls; both + // shutdown checks in acceptLoop depend on this ordering. + close(p.shutdownCh) + p.closeListeners() + // Cancel the dial context so in-flight DialFunc calls can abort. + // Bridge goroutines that have already moved past Dial are unblocked + // by the listener close above; this signal is for the dial gap. + p.dialCancel() + }) + + done := make(chan struct{}) + go func() { + p.wg.Wait() + close(done) + }() + select { + case <-done: + return nil + case <-ctx.Done(): + return fmt.Errorf("byteproxy %s: shutdown deadline reached before all goroutines drained: %w", p.name, ctx.Err()) + } +} + +// DroppedCount returns the cumulative drop count for the named service. +func (p *proxy) DroppedCount(service string) int64 { + for _, sl := range p.listeners { + if sl.Name == service { + return sl.dropped.Load() + } + } + return 0 +} + +func (p *proxy) acceptLoop(ctx context.Context, sl *serviceListener) { + defer p.wg.Done() + p.logger.Info(ctx, "listener started", logging.Entries{ + "service": sl.Name, + "endpoint": sl.Endpoint, + }) + + for { + conn, err := sl.ln.Accept() + if err != nil { + // First shutdown check: Accept returned an error because + // Shutdown closed the listener. Exit silently. + select { + case <-p.shutdownCh: + return + default: + } + // Accept failures outside shutdown are terminal for this + // service loop β€” the listener will no longer accept new + // connections until the process restarts. Log at error level so + // the condition surfaces in alerting; transient-error retry + // can be revisited once CCHAIN-2027 brings a real consumer + // and we have observability on actual accept failure modes. + p.logger.ErrorErr(ctx, "accept failed (service loop exiting)", err, logging.Entries{ + "service": sl.Name, + }) + return + } + + // Second shutdown check: Accept returned a real conn but Shutdown + // has already fired (close(shutdownCh) happens before listener + // Close in Shutdown, so this catches conns that were queued in the + // kernel backlog at the moment Shutdown started). + select { + case <-p.shutdownCh: + _ = conn.Close() + return + default: + } + + select { + case sl.sem <- struct{}{}: + default: + count := sl.dropped.Add(1) + p.logger.Warn(ctx, "dropping connection: too many in-flight", logging.Entries{ + "service": sl.Name, + "dropped_total": count, + }) + _ = conn.Close() + continue + } + + // Safe to wg.Add here: acceptLoop itself holds a wg ticket + // (defer p.wg.Done above), so the counter is β‰₯ 1 throughout and + // Add-while-positive is always legal per sync.WaitGroup docs. + p.wg.Add(1) + go func(c net.Conn) { + defer p.wg.Done() + defer func() { + <-sl.sem + _ = c.Close() + }() + p.bridge(ctx, sl, c) + }(conn) + } +} + +// bridge dials the upstream for sl and bidirectionally copies bytes between +// inbound and upstream until either side closes or shutdown is signalled. +// The dial uses p.dialCtx so Shutdown can abort dialers that are still +// waiting on the upstream β€” see the DialFunc godoc for the wrapper contract. +func (p *proxy) bridge(ctx context.Context, sl *serviceListener, inbound net.Conn) { + upstream, err := sl.Dial(p.dialCtx, inbound) + if err != nil { + p.logger.WarnErr(ctx, "upstream dial failed", err, logging.Entries{ + "service": sl.Name, + "endpoint": sl.Endpoint, + }) + return + } + defer func() { _ = upstream.Close() }() + + p.pipe(ctx, sl, inbound, upstream) +} + +// pipe runs two io.Copy goroutines and returns when both copies have +// exited, either naturally (one side hits EOF, its close cascades to the +// other) or because shutdown was signalled (forcing both sides closed). +// The function always waits for the io.Copy goroutines to finish before +// returning, so it never leaks them past pipe's lifetime. +// +// Non-nil io.Copy errors are logged at Debug level with the direction +// (inbound->upstream or upstream->inbound) so mid-stream resets or write +// failures are diagnosable on demand without flooding production logs. +// io.Copy normalises EOF to nil, and the shutdown path closes both sides +// so a clean shutdown surfaces as net.ErrClosed at Debug too β€” acceptable +// since the level is off by default in prod. +// +// Close may be called multiple times across the bridge stack (here in pipe, +// in the goroutines' own deferred close-other-side, in bridge's defer, and +// in acceptLoop's outer defer). All callers discard the error and our +// supported transports (TCP, vsock) are idempotent on Close. A future +// transport that panics on double Close would need this assumption revisited. +func (p *proxy) pipe(ctx context.Context, sl *serviceListener, inbound, upstream net.Conn) { + copyDone := make(chan struct{}, 2) + go func() { + _, err := io.Copy(upstream, inbound) + if err != nil { + p.logger.DebugErr(ctx, "copy failed", err, logging.Entries{ + "service": sl.Name, + "direction": "inbound->upstream", + }) + } + _ = upstream.Close() + copyDone <- struct{}{} + }() + go func() { + _, err := io.Copy(inbound, upstream) + if err != nil { + p.logger.DebugErr(ctx, "copy failed", err, logging.Entries{ + "service": sl.Name, + "direction": "upstream->inbound", + }) + } + _ = inbound.Close() + copyDone <- struct{}{} + }() + + shutdown := p.shutdownCh + finished := 0 + for finished < 2 { + select { + case <-copyDone: + finished++ + case <-shutdown: + // Force both sides closed so the io.Copy goroutines unblock, + // then fall through to drain copyDone. Nil out shutdown so we + // only do this once even if it stays closed. + _ = inbound.Close() + _ = upstream.Close() + shutdown = nil + } + } +} diff --git a/internal/common/byteproxy/byteproxy_test.go b/internal/common/byteproxy/byteproxy_test.go new file mode 100644 index 0000000..71501b9 --- /dev/null +++ b/internal/common/byteproxy/byteproxy_test.go @@ -0,0 +1,617 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package byteproxy + +import ( + "context" + "errors" + "io" + "net" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/suite" +) + +// ByteProxyTestSuite tests the shared byte-proxy framework using in-memory +// TCP listeners as both the inbound and upstream sides, so the suite has no +// dependency on vsock or any other transport. +type ByteProxyTestSuite struct { + suite.Suite +} + +// newLocalListener returns a TCP listener bound to a random loopback port. +func (s *ByteProxyTestSuite) newLocalListener() net.Listener { + s.T().Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + s.Require().NoError(err) + s.T().Cleanup(func() { _ = ln.Close() }) + return ln +} + +// startEcho accepts a single connection from ln and echoes everything back. +func (s *ByteProxyTestSuite) startEcho(ln net.Listener) { + s.T().Helper() + go func() { + conn, err := ln.Accept() + if err != nil { + return + } + defer func() { _ = conn.Close() }() + _, _ = io.Copy(conn, conn) + }() +} + +// startProxy creates a proxy, calls Run(), and registers Shutdown with +// t.Cleanup. It also asserts that Run is non-blocking by failing fast if +// it does not return promptly β€” guarding the lifecycle.Runnable contract. +func (s *ByteProxyTestSuite) startProxy(cfg *Config) Proxy { + s.T().Helper() + p, err := New(cfg) + s.Require().NoError(err) + + runDone := make(chan struct{}) + go func() { + _ = p.Run() + close(runDone) + }() + select { + case <-runDone: + case <-time.After(time.Second): + s.FailNow("Run() did not return promptly; it must be non-blocking per lifecycle.Runnable") + } + + s.T().Cleanup(func() { _ = p.Shutdown(context.Background()) }) + return p +} + +// TestNew covers constructor validation in a table-driven form. +func (s *ByteProxyTestSuite) TestNew() { + okListen := func() (net.Listener, error) { + return s.newLocalListener(), nil + } + okDial := func(_ context.Context, _ net.Conn) (net.Conn, error) { return nil, errors.New("not used") } + failingListen := func() (net.Listener, error) { return nil, errors.New("bind failed") } + + tests := map[string]struct { + cfg *Config + wantErr bool + }{ + "NilConfig": { + cfg: nil, + wantErr: true, + }, + "MissingName": { + cfg: &Config{ + Services: []ServiceConfig{{Name: "kms", Listen: okListen, Dial: okDial}}, + MaxConns: 4, + }, + wantErr: true, + }, + "NoServices": { + cfg: &Config{Name: "test", MaxConns: 4}, + wantErr: true, + }, + "InvalidMaxConns": { + cfg: &Config{ + Name: "test", + Services: []ServiceConfig{{Name: "kms", Listen: okListen, Dial: okDial}}, + MaxConns: 0, + }, + wantErr: true, + }, + "MissingListen": { + cfg: &Config{ + Name: "test", + Services: []ServiceConfig{{Name: "kms", Dial: okDial}}, + MaxConns: 4, + }, + wantErr: true, + }, + "MissingDial": { + cfg: &Config{ + Name: "test", + Services: []ServiceConfig{{Name: "kms", Listen: okListen}}, + MaxConns: 4, + }, + wantErr: true, + }, + "BindFailure": { + cfg: &Config{ + Name: "test", + Services: []ServiceConfig{{Name: "kms", Listen: failingListen, Dial: okDial}}, + MaxConns: 4, + }, + wantErr: true, + }, + } + for name, tt := range tests { + s.Run(name, func() { + p, err := New(tt.cfg) + if tt.wantErr { + s.Require().Error(err) + return + } + s.Require().NoError(err) + s.NotNil(p) + }) + } +} + +// TestRun_IsNonBlocking explicitly verifies that Run() returns immediately, +// since lifecycle.Manager calls each Runnable's Run synchronously and would +// stall on the next service if Run blocked. +func (s *ByteProxyTestSuite) TestRun_IsNonBlocking() { + inbound := s.newLocalListener() + p, err := New(&Config{ + Name: "test", + MaxConns: 4, + Services: []ServiceConfig{{ + Name: "kms", + Listen: func() (net.Listener, error) { return inbound, nil }, + Dial: func(_ context.Context, _ net.Conn) (net.Conn, error) { return nil, errors.New("not used") }, + }}, + }) + s.Require().NoError(err) + s.T().Cleanup(func() { _ = p.Shutdown(context.Background()) }) + + returned := make(chan error, 1) + go func() { returned <- p.Run() }() + select { + case err := <-returned: + s.Require().NoError(err) + case <-time.After(500 * time.Millisecond): + s.FailNow("Run() did not return; it must be non-blocking") + } +} + +// TestProxy_BidirectionalCopy verifies bytes flow both ways between inbound +// and upstream. +func (s *ByteProxyTestSuite) TestProxy_BidirectionalCopy() { + upstream := s.newLocalListener() + s.startEcho(upstream) + + inbound := s.newLocalListener() + p := s.startProxy(&Config{ + Name: "test", + MaxConns: 4, + Services: []ServiceConfig{{ + Name: "kms", + Listen: func() (net.Listener, error) { return inbound, nil }, + Dial: func(_ context.Context, _ net.Conn) (net.Conn, error) { + return net.Dial("tcp", upstream.Addr().String()) + }, + }}, + }) + _ = p + + conn, err := net.Dial("tcp", inbound.Addr().String()) + s.Require().NoError(err) + defer func() { _ = conn.Close() }() + + payload := []byte("hello-from-sdk") + _, err = conn.Write(payload) + s.Require().NoError(err) + + buf := make([]byte, len(payload)) + s.Require().NoError(conn.SetReadDeadline(time.Now().Add(2 * time.Second))) + _, err = io.ReadFull(conn, buf) + s.Require().NoError(err) + s.Equal(payload, buf) +} + +func (s *ByteProxyTestSuite) TestProxy_DialCanReadInboundRouteBeforeBridge() { + upstream := s.newLocalListener() + s.startEcho(upstream) + + inbound := s.newLocalListener() + routes := make(chan AWSRoute, 1) + s.startProxy(&Config{ + Name: "test", + MaxConns: 4, + Services: []ServiceConfig{{ + Name: "kms", + Listen: func() (net.Listener, error) { return inbound, nil }, + Dial: func(_ context.Context, accepted net.Conn) (net.Conn, error) { + route, err := ReadAWSRoute(accepted) + if err != nil { + return nil, err + } + routes <- route + return net.Dial("tcp", upstream.Addr().String()) + }, + }}, + }) + + conn, err := net.Dial("tcp", inbound.Addr().String()) + s.Require().NoError(err) + defer func() { _ = conn.Close() }() + + wantRoute := AWSRoute{Service: "kms", Region: "us-west-2"} + s.Require().NoError(WriteAWSRoute(conn, wantRoute)) + payload := []byte("hello-after-route") + _, err = conn.Write(payload) + s.Require().NoError(err) + + buf := make([]byte, len(payload)) + s.Require().NoError(conn.SetReadDeadline(time.Now().Add(2 * time.Second))) + _, err = io.ReadFull(conn, buf) + s.Require().NoError(err) + s.Equal(payload, buf) + s.Equal(wantRoute, <-routes) +} + +// TestProxy_HalfClose verifies the proxy closes the peer side when one side +// ends. Synchronisation uses a signal channel from the upstream-side echo +// rather than time.Sleep, so the test is deterministic under CI load. +func (s *ByteProxyTestSuite) TestProxy_HalfClose() { + upstream := s.newLocalListener() + peerClosed := make(chan struct{}) + go func() { + conn, err := upstream.Accept() + if err != nil { + return + } + defer func() { _ = conn.Close() }() + _, _ = io.Copy(io.Discard, conn) + close(peerClosed) + }() + + inbound := s.newLocalListener() + s.startProxy(&Config{ + Name: "test", + MaxConns: 4, + Services: []ServiceConfig{{ + Name: "kms", + Listen: func() (net.Listener, error) { return inbound, nil }, + Dial: func(_ context.Context, _ net.Conn) (net.Conn, error) { + return net.Dial("tcp", upstream.Addr().String()) + }, + }}, + }) + + conn, err := net.Dial("tcp", inbound.Addr().String()) + s.Require().NoError(err) + _, err = conn.Write([]byte("payload")) + s.Require().NoError(err) + s.Require().NoError(conn.Close()) + + select { + case <-peerClosed: + case <-time.After(2 * time.Second): + s.FailNow("upstream did not observe EOF after inbound close; half-close did not propagate") + } +} + +// TestProxy_DialFailureClosesConnection verifies that a failed upstream dial +// does not leave the inbound connection open. +func (s *ByteProxyTestSuite) TestProxy_DialFailureClosesConnection() { + inbound := s.newLocalListener() + s.startProxy(&Config{ + Name: "test", + MaxConns: 4, + Services: []ServiceConfig{{ + Name: "kms", + Endpoint: "kms.example.com:443", + Listen: func() (net.Listener, error) { return inbound, nil }, + Dial: func(_ context.Context, _ net.Conn) (net.Conn, error) { return nil, errors.New("dial failed") }, + }}, + }) + + conn, err := net.Dial("tcp", inbound.Addr().String()) + s.Require().NoError(err) + defer func() { _ = conn.Close() }() + + s.Require().NoError(conn.SetReadDeadline(time.Now().Add(2 * time.Second))) + buf := make([]byte, 16) + _, err = conn.Read(buf) + s.Require().ErrorIs(err, io.EOF) +} + +// TestProxy_DroppedCount verifies that connections rejected because the +// per-service concurrency cap is full are counted. +func (s *ByteProxyTestSuite) TestProxy_DroppedCount() { + // release is closed early β€” before Shutdown β€” so the upstream goroutines + // that hold connections can exit cleanly without depending on pipe's + // shutdown branch to unblock them. defer runs in LIFO before t.Cleanup, + // so this fires first. + release := make(chan struct{}) + defer close(release) + + // Upstream accepts and holds connections until release is closed. + upstream := s.newLocalListener() + accepted := make(chan struct{}, 16) + go func() { + for { + conn, err := upstream.Accept() + if err != nil { + return + } + accepted <- struct{}{} + go func(c net.Conn) { + <-release + _ = c.Close() + }(conn) + } + }() + + inbound := s.newLocalListener() + const maxConns = 2 + p := s.startProxy(&Config{ + Name: "test", + MaxConns: maxConns, + Services: []ServiceConfig{{ + Name: "kms", + Listen: func() (net.Listener, error) { return inbound, nil }, + Dial: func(_ context.Context, _ net.Conn) (net.Conn, error) { + return net.Dial("tcp", upstream.Addr().String()) + }, + }}, + }) + + // Dial up to the cap and wait until upstream has accepted them, so the + // per-service semaphore is provably full before we send the extras. + holdConns := make([]net.Conn, 0, maxConns) + for range maxConns { + c, derr := net.Dial("tcp", inbound.Addr().String()) + s.Require().NoError(derr) + holdConns = append(holdConns, c) + } + for range maxConns { + select { + case <-accepted: + case <-time.After(2 * time.Second): + s.FailNow("upstream did not accept the held connections in time") + } + } + defer func() { + for _, c := range holdConns { + _ = c.Close() + } + }() + + // Now extras should be dropped. We dial and immediately observe EOF + // because the proxy closes the inbound conn after sl.sem is full. + const extras = 3 + for range extras { + c, derr := net.Dial("tcp", inbound.Addr().String()) + s.Require().NoError(derr) + s.Require().NoError(c.SetReadDeadline(time.Now().Add(2 * time.Second))) + buf := make([]byte, 1) + _, err := c.Read(buf) + s.Require().ErrorIs(err, io.EOF) + _ = c.Close() + } + + s.Equal(int64(extras), p.DroppedCount("kms")) + s.Equal(int64(0), p.DroppedCount("nonexistent")) +} + +// TestProxy_ShutdownDuringAccept exercises the "two shutdown checks" race +// path in acceptLoop: a listener whose Accept returns a real conn while +// shutdownCh is already closed. The conn must be closed by acceptLoop and +// not dispatched to a bridge. +func (s *ByteProxyTestSuite) TestProxy_ShutdownDuringAccept() { + // We control the listener directly so we can race Shutdown against + // Accept. acceptCh has one queued conn pre-shutdown. + queued := make(chan net.Conn, 1) + acceptErr := make(chan error, 1) + closeCh := make(chan struct{}) + ln := &controllableListener{ + queued: queued, + acceptErr: acceptErr, + closeCh: closeCh, + } + + dialed := make(chan struct{}, 1) + p, err := New(&Config{ + Name: "test", + MaxConns: 4, + Services: []ServiceConfig{{ + Name: "kms", + Listen: func() (net.Listener, error) { return ln, nil }, + Dial: func(_ context.Context, _ net.Conn) (net.Conn, error) { + dialed <- struct{}{} + return nil, errors.New("dial should not be called after shutdown") + }, + }}, + }) + s.Require().NoError(err) + + s.Require().NoError(p.Run()) + + // Pre-shutdown: queue a conn that acceptLoop will pick up. + clientConn, serverConn := net.Pipe() + defer func() { _ = clientConn.Close() }() + // Now shut down BEFORE feeding the queued conn. This guarantees + // shutdownCh is closed when Accept eventually returns. + shutdownDone := make(chan struct{}) + go func() { + _ = p.Shutdown(context.Background()) + close(shutdownDone) + }() + // Wait for shutdownCh to actually be closed before feeding the conn, + // so the post-Accept gate sees it. + // Shutdown's wg.Wait blocks on acceptLoop which is blocked on Accept, + // so we drive the listener manually to feed the conn after a brief + // moment for close(shutdownCh) to have happened. We use a sync + // channel: controllableListener.closeCh fires when Close is called by + // Shutdown, which happens AFTER close(shutdownCh). Wait for that. + select { + case <-closeCh: + // listener.Close was called β†’ shutdownCh is already closed. + // Feed the queued conn so Accept returns it. + queued <- serverConn + case <-time.After(2 * time.Second): + s.FailNow("listener Close was not invoked by Shutdown") + } + + // acceptLoop should see the post-Accept shutdown check fire, close + // serverConn, and exit without dialing. Wait briefly to confirm dial + // is NOT called. + select { + case <-dialed: + s.FailNow("Dial was called for a conn accepted post-shutdown; the second shutdown check failed") + case <-time.After(100 * time.Millisecond): + // expected: no dial + } + + // Final Accept returns ErrClosed to wake acceptLoop. + acceptErr <- net.ErrClosed + + select { + case <-shutdownDone: + case <-time.After(2 * time.Second): + s.FailNow("Shutdown did not return") + } +} + +// controllableListener is a net.Listener whose Accept is driven by the test +// via channels. Used by TestProxy_ShutdownDuringAccept to interleave +// Accept return with shutdown signalling. +type controllableListener struct { + queued chan net.Conn + acceptErr chan error + closeCh chan struct{} + closeOnce sync.Once +} + +func (l *controllableListener) Accept() (net.Conn, error) { + select { + case c := <-l.queued: + return c, nil + case err := <-l.acceptErr: + return nil, err + } +} + +func (l *controllableListener) Close() error { + l.closeOnce.Do(func() { close(l.closeCh) }) + return nil +} + +func (l *controllableListener) Addr() net.Addr { + return &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 0} +} + +// TestProxy_ShutdownCancelsInFlightDial verifies that Shutdown cancels the +// ctx handed to DialFunc, so cooperative dialers (those that honor ctx) +// abort instead of holding the WaitGroup open. This guards the liveness +// contract: a stalled upstream must not prevent lifecycle shutdown. +func (s *ByteProxyTestSuite) TestProxy_ShutdownCancelsInFlightDial() { + dialing := make(chan struct{}) + dialReturned := make(chan error, 1) + inbound := s.newLocalListener() + p, err := New(&Config{ + Name: "test", + MaxConns: 4, + Services: []ServiceConfig{{ + Name: "kms", + Listen: func() (net.Listener, error) { return inbound, nil }, + Dial: func(ctx context.Context, _ net.Conn) (net.Conn, error) { + close(dialing) + <-ctx.Done() + dialReturned <- ctx.Err() + return nil, ctx.Err() + }, + }}, + }) + s.Require().NoError(err) + s.Require().NoError(p.Run()) + + // Drive one connection through the inbound listener so bridge starts + // the dial and parks inside DialFunc waiting on ctx. + conn, derr := net.Dial("tcp", inbound.Addr().String()) + s.Require().NoError(derr) + defer func() { _ = conn.Close() }() + select { + case <-dialing: + case <-time.After(2 * time.Second): + s.FailNow("DialFunc was not invoked") + } + + // Shutdown must observe the ctx cancellation propagating to the parked + // dialer and drain promptly. + shutdownErr := make(chan error, 1) + go func() { shutdownErr <- p.Shutdown(context.Background()) }() + + select { + case err := <-dialReturned: + s.Require().ErrorIs(err, context.Canceled, + "DialFunc ctx must be cancelled by Shutdown so cooperative dialers unblock") + case <-time.After(2 * time.Second): + s.FailNow("Shutdown did not cancel the in-flight dial ctx") + } + + select { + case err := <-shutdownErr: + s.Require().NoError(err) + case <-time.After(2 * time.Second): + s.FailNow("Shutdown did not return after dial unblocked") + } +} + +// TestProxy_ShutdownHonorsContextDeadline verifies that Shutdown returns +// when its ctx expires even if a bridge goroutine refuses to honor the +// dial ctx (simulating a truly hung syscall). The dialer here ignores ctx +// entirely; Shutdown's wg.Wait would otherwise block forever. +func (s *ByteProxyTestSuite) TestProxy_ShutdownHonorsContextDeadline() { + dialing := make(chan struct{}) + release := make(chan struct{}) + defer close(release) + + inbound := s.newLocalListener() + p, err := New(&Config{ + Name: "test", + MaxConns: 4, + Services: []ServiceConfig{{ + Name: "kms", + Listen: func() (net.Listener, error) { return inbound, nil }, + Dial: func(_ context.Context, _ net.Conn) (net.Conn, error) { + close(dialing) + <-release // ignore ctx, simulate uncancellable dial + return nil, errors.New("released") + }, + }}, + }) + s.Require().NoError(err) + s.Require().NoError(p.Run()) + + conn, derr := net.Dial("tcp", inbound.Addr().String()) + s.Require().NoError(derr) + defer func() { _ = conn.Close() }() + select { + case <-dialing: + case <-time.After(2 * time.Second): + s.FailNow("DialFunc was not invoked") + } + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + start := time.Now() + err = p.Shutdown(ctx) + elapsed := time.Since(start) + + s.Require().Error(err) + s.Require().ErrorIs(err, context.DeadlineExceeded, + "Shutdown must surface ctx.Err() when the deadline expires before drain") + s.Less(elapsed, time.Second, + "Shutdown must return promptly once ctx expires, not wait for the hung dialer") +} + +// TestByteProxyTestSuite is the entry point for the suite. +func TestByteProxyTestSuite(t *testing.T) { + suite.Run(t, &ByteProxyTestSuite{Suite: suite.Suite{}}) +} diff --git a/internal/common/byteproxy/services.go b/internal/common/byteproxy/services.go new file mode 100644 index 0000000..2541dcc --- /dev/null +++ b/internal/common/byteproxy/services.go @@ -0,0 +1,29 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package byteproxy + +// AWSServiceNames is the ordered list of AWS services the enclave-side and +// host-side proxies must agree on. Indices determine the port offset from +// each side's BasePort, so adding or reordering entries here is the single +// edit point β€” wrappers derive their per-service config from this slice. +// +// Order mirrors gateway-tee-signer's hardcoded ["KMS", "SecretsManager", +// "STS"]. Entries beyond the first are reserved for later subtasks and are +// inert until they have AWS SDK consumers in the enclave. +// +// DO NOT mutate this slice at runtime. Callers (awsproxy on the enclave, +// vsockproxy on the host) coordinate port assignments by iterating this +// list and would observe inconsistent ports if append/reorder happened +// after either wrapper has been constructed. +var AWSServiceNames = []string{"kms"} diff --git a/internal/common/byteproxy/transport.go b/internal/common/byteproxy/transport.go new file mode 100644 index 0000000..c8e87cf --- /dev/null +++ b/internal/common/byteproxy/transport.go @@ -0,0 +1,32 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package byteproxy + +// TransportKind identifies which transport a proxy wrapper has selected. +// The enclave-side awsproxy and the host-side vsockproxy wrappers share +// this enum so the two sides of the bridge name transports identically. +// The zero value is TransportVsock, the production default; the wrapper +// packages (awsproxy, vsockproxy) expose a WithTCPTransport option to select +// TCP for dev/CI. A transport option installs the matching listen/dial +// unconditionally, so when a caller passes more than one transport option +// the last one wins. +type TransportKind int + +const ( + // TransportVsock selects the vsock transport (Linux only). It is the + // zero value, so a caller that passes no transport option gets vsock. + TransportVsock TransportKind = iota + // TransportTCP selects the TCP transport (dev/CI). + TransportTCP +) diff --git a/internal/common/config/loader.go b/internal/common/config/loader.go index 5d533f4..9b2e970 100644 --- a/internal/common/config/loader.go +++ b/internal/common/config/loader.go @@ -18,6 +18,7 @@ import ( "log" "strings" + "github.com/go-viper/mapstructure/v2" "github.com/spf13/viper" "sigs.k8s.io/yaml" ) @@ -60,7 +61,7 @@ func LoadConfig(cfg ApplicationConfig, cfgFile string) { log.Fatalf("Unable to parse config file: %v", err) } if cfgFile != "" { - log.Printf("Warning: specified config file not found: %v", err) + log.Fatalf("Specified config file could not be loaded: %v", err) } } @@ -70,7 +71,11 @@ func LoadConfig(cfg ApplicationConfig, cfgFile string) { v.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) // Unmarshal into config struct - if err := v.Unmarshal(cfg); err != nil { + if err := v.Unmarshal(cfg, func(decoderConfig *mapstructure.DecoderConfig) { + // Clear maps before decode to prevent duplicate normalized method-map + // keys from defaults and file or environment input. + decoderConfig.ZeroFields = true + }); err != nil { log.Fatalf("Unable to decode config: %v", err) } diff --git a/internal/common/config/loader_test.go b/internal/common/config/loader_test.go index 2a86bb0..fcf11a1 100644 --- a/internal/common/config/loader_test.go +++ b/internal/common/config/loader_test.go @@ -15,6 +15,7 @@ package config import ( "os" + "os/exec" "path/filepath" "testing" @@ -25,6 +26,15 @@ import ( type testAppConfig struct { *BaseConfig `mapstructure:",squash"` TestField string `mapstructure:"test_field"` + Nested testNestedConfig +} + +type testNestedConfig struct { + Methods map[string]testMethodConfig +} + +type testMethodConfig struct { + TimeoutMS int `json:"timeoutMS" mapstructure:"timeoutMS"` } func (c *testAppConfig) GetName() string { @@ -139,16 +149,6 @@ test_field: "file-value" assertLoadedConfig(t, cfg, Dev, "file-value") }) - t.Run("keeps initial values when config file does not exist", func(t *testing.T) { - t.Setenv("APP_ENV", "") - t.Setenv("APP_TEST_FIELD", "") - - cfg := newTestAppConfig() - LoadConfig(cfg, "/tmp/non-existent-config.yaml") - - assertLoadedConfig(t, cfg, Dev, "") - }) - t.Run("keeps initial values when default config path has no config file", func(t *testing.T) { t.Setenv("APP_ENV", "") t.Setenv("APP_TEST_FIELD", "") @@ -163,9 +163,47 @@ test_field: "file-value" }) } +func TestLoadConfig_ExitsWhenExplicitConfigFileMissing(t *testing.T) { + const ( + helperEnv = "TEST_LOAD_CONFIG_FATAL" + pathEnv = "TEST_LOAD_CONFIG_PATH" + ) + + if os.Getenv(helperEnv) == "1" { + LoadConfig(newTestAppConfig(), os.Getenv(pathEnv)) + return + } + + missingPath := filepath.Join(t.TempDir(), "missing.yaml") + cmd := exec.Command(os.Args[0], "-test.run=^TestLoadConfig_ExitsWhenExplicitConfigFileMissing$") + cmd.Env = append(os.Environ(), helperEnv+"=1", pathEnv+"="+missingPath) + output, err := cmd.CombinedOutput() + + var exitErr *exec.ExitError + require.ErrorAs(t, err, &exitErr) + require.Equal(t, 1, exitErr.ExitCode()) + require.Contains(t, string(output), "Specified config file could not be loaded:") + require.Contains(t, string(output), missingPath) +} + +func TestLoadConfig_EnvironmentOverridesNestedMapValue(t *testing.T) { + t.Setenv("APP_NESTED_METHODS_INITIALIZE_TIMEOUTMS", "123") + + cfg := newTestAppConfig() + LoadConfig(cfg, "") + + require.Len(t, cfg.Nested.Methods, 1) + require.Equal(t, 123, cfg.Nested.Methods["initialize"].TimeoutMS) +} + func newTestAppConfig() *testAppConfig { return &testAppConfig{ BaseConfig: NewBaseConfig(), + Nested: testNestedConfig{ + Methods: map[string]testMethodConfig{ + "Initialize": {TimeoutMS: 30000}, + }, + }, } } diff --git a/internal/common/grpc/client/client.go b/internal/common/grpc/client/client.go index a74a987..1d04d75 100644 --- a/internal/common/grpc/client/client.go +++ b/internal/common/grpc/client/client.go @@ -15,12 +15,13 @@ package client import ( + "context" "fmt" "net" + "strings" "time" grpcRetry "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/retry" - grpcTimeout "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/timeout" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials/insecure" @@ -40,17 +41,18 @@ func InsecureDialOptions(cfg Config, extraInterceptors ...grpc.UnaryClientInterc if len(retryCodes) == 0 { retryCodes = defaultRetryCodes } - if maxAttempts > 0 { + if cfg.RequestTimeoutMS > 0 || len(cfg.Methods) > 0 { + unaryInterceptors = append(unaryInterceptors, unaryMethodTimeoutInterceptor(cfg)) + } + if len(cfg.Methods) > 0 { + unaryInterceptors = append(unaryInterceptors, unaryMethodRetryInterceptor(cfg)) + } + if maxAttempts > 0 || hasMethodMaxAttempts(cfg.Methods) { unaryInterceptors = append(unaryInterceptors, grpcRetry.UnaryClientInterceptor( grpcRetry.WithMax(maxAttempts), grpcRetry.WithCodes(retryCodes...), )) } - if cfg.RequestTimeoutMS > 0 { - unaryInterceptors = append([]grpc.UnaryClientInterceptor{ - grpcTimeout.UnaryClientInterceptor(time.Duration(cfg.RequestTimeoutMS) * time.Millisecond), - }, unaryInterceptors...) - } unaryInterceptors = append(unaryInterceptors, extraInterceptors...) return []grpc.DialOption{ @@ -59,6 +61,85 @@ func InsecureDialOptions(cfg Config, extraInterceptors ...grpc.UnaryClientInterc } } +// unaryMethodTimeoutInterceptor applies the configured per-method timeout or +// the global request timeout to each unary RPC. +func unaryMethodTimeoutInterceptor(cfg Config) grpc.UnaryClientInterceptor { + return func( + ctx context.Context, + method string, + req, reply any, + cc *grpc.ClientConn, + invoker grpc.UnaryInvoker, + opts ...grpc.CallOption, + ) error { + timeout := resolveMethodTimeout(cfg, method) + if timeout <= 0 { + return invoker(ctx, method, req, reply, cc, opts...) + } + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + return invoker(ctx, method, req, reply, cc, opts...) + } +} + +func unaryMethodRetryInterceptor(cfg Config) grpc.UnaryClientInterceptor { + return func( + ctx context.Context, + method string, + req, reply any, + cc *grpc.ClientConn, + invoker grpc.UnaryInvoker, + opts ...grpc.CallOption, + ) error { + if methodCfg, ok := lookupMethodConfig(cfg, method); ok && methodCfg.MaxAttempts > 0 { + opts = append(opts, grpcRetry.WithMax(methodCfg.MaxAttempts)) + } + return invoker(ctx, method, req, reply, cc, opts...) + } +} + +func hasMethodMaxAttempts(methods map[string]MethodConfig) bool { + for _, methodCfg := range methods { + if methodCfg.MaxAttempts > 0 { + return true + } + } + return false +} + +func resolveMethodTimeout(cfg Config, fullMethod string) time.Duration { + if methodCfg, ok := lookupMethodConfig(cfg, fullMethod); ok { + if methodCfg.TimeoutMS < 0 { + return 0 + } + if methodCfg.TimeoutMS > 0 { + return time.Duration(methodCfg.TimeoutMS) * time.Millisecond + } + } + if cfg.RequestTimeoutMS > 0 { + return time.Duration(cfg.RequestTimeoutMS) * time.Millisecond + } + return 0 +} + +func lookupMethodConfig(cfg Config, fullMethod string) (MethodConfig, bool) { + if methodCfg, ok := cfg.Methods[fullMethod]; ok { + return methodCfg, true + } + if methodCfg, ok := cfg.Methods[strings.ToLower(fullMethod)]; ok { + return methodCfg, true + } + if idx := strings.LastIndex(fullMethod, "/"); idx >= 0 { + shortMethod := fullMethod[idx+1:] + if methodCfg, ok := cfg.Methods[shortMethod]; ok { + return methodCfg, true + } + methodCfg, ok := cfg.Methods[strings.ToLower(shortMethod)] + return methodCfg, ok + } + return MethodConfig{}, false +} + // NewInsecureClientConn creates an insecure grpc client connection from config. func NewInsecureClientConn(rawTarget string, cfg Config, extraDialOptions ...grpc.DialOption) (*grpc.ClientConn, error) { if _, _, err := net.SplitHostPort(rawTarget); err != nil { diff --git a/internal/common/grpc/client/client_test.go b/internal/common/grpc/client/client_test.go index b0fd56f..35d9672 100644 --- a/internal/common/grpc/client/client_test.go +++ b/internal/common/grpc/client/client_test.go @@ -18,6 +18,7 @@ import ( "net" "sync" "testing" + "time" "github.com/stretchr/testify/require" "google.golang.org/grpc" @@ -32,17 +33,25 @@ type testHealthServer struct { calls int failUntil int blockUntilCtxTimeout bool + responseDelay time.Duration } func (s *testHealthServer) Check(ctx context.Context, _ *grpcHealthV1.HealthCheckRequest) (*grpcHealthV1.HealthCheckResponse, error) { - if s.blockUntilCtxTimeout { - <-ctx.Done() - return nil, status.Error(codes.DeadlineExceeded, ctx.Err().Error()) - } s.mu.Lock() s.calls++ current := s.calls s.mu.Unlock() + if s.responseDelay > 0 { + select { + case <-time.After(s.responseDelay): + case <-ctx.Done(): + return nil, status.Error(codes.DeadlineExceeded, ctx.Err().Error()) + } + } + if s.blockUntilCtxTimeout { + <-ctx.Done() + return nil, status.Error(codes.DeadlineExceeded, ctx.Err().Error()) + } if current <= s.failUntil { return nil, status.Error(codes.Unavailable, "temporary unavailable") } @@ -165,3 +174,129 @@ func TestNewInsecureClientConnCustomRetryCodes(t *testing.T) { require.Equal(t, codes.Unavailable, status.Code(err), "expected unavailable") require.Equal(t, 1, svc.Calls(), "expected 1 attempt") } + +func TestNewInsecureClientConn_MethodTimeoutOverridesDefault(t *testing.T) { + svc := &testHealthServer{blockUntilCtxTimeout: true} + addr, stop := startHealthServer(t, svc) + defer stop() + + conn, err := NewInsecureClientConn(addr, Config{ + RequestTimeoutMS: 5000, + Methods: map[string]MethodConfig{ + "Check": {TimeoutMS: 10}, + }, + Retry: &RetryConfig{MaxAttempts: 0}, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = conn.Close() }) + + client := grpcHealthV1.NewHealthClient(conn) + start := time.Now() + _, err = client.Check(context.Background(), &grpcHealthV1.HealthCheckRequest{}) + + require.Equal(t, codes.DeadlineExceeded, status.Code(err)) + require.Less(t, time.Since(start), time.Second) +} + +func TestNewInsecureClientConn_PreservesShorterCallerDeadline(t *testing.T) { + svc := &testHealthServer{blockUntilCtxTimeout: true} + addr, stop := startHealthServer(t, svc) + defer stop() + + conn, err := NewInsecureClientConn(addr, Config{ + RequestTimeoutMS: 5000, + Methods: map[string]MethodConfig{ + "Check": {TimeoutMS: 5000}, + }, + Retry: &RetryConfig{MaxAttempts: 0}, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = conn.Close() }) + + client := grpcHealthV1.NewHealthClient(conn) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + defer cancel() + start := time.Now() + _, err = client.Check(ctx, &grpcHealthV1.HealthCheckRequest{}) + + require.Equal(t, codes.DeadlineExceeded, status.Code(err)) + require.Less(t, time.Since(start), time.Second) +} + +func TestNewInsecureClientConn_MethodTimeoutCanExceedDefault(t *testing.T) { + svc := &testHealthServer{responseDelay: 30 * time.Millisecond} + addr, stop := startHealthServer(t, svc) + defer stop() + + conn, err := NewInsecureClientConn(addr, Config{ + RequestTimeoutMS: 10, + Methods: map[string]MethodConfig{ + "Check": {TimeoutMS: 100}, + }, + Retry: &RetryConfig{MaxAttempts: 1}, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = conn.Close() }) + + client := grpcHealthV1.NewHealthClient(conn) + _, err = client.Check(context.Background(), &grpcHealthV1.HealthCheckRequest{}) + + require.NoError(t, err) + require.Equal(t, 1, svc.Calls()) +} + +func TestNewInsecureClientConn_RetriesShareMethodTimeout(t *testing.T) { + svc := &testHealthServer{failUntil: 10, responseDelay: 20 * time.Millisecond} + addr, stop := startHealthServer(t, svc) + defer stop() + + conn, err := NewInsecureClientConn(addr, Config{ + RequestTimeoutMS: 1000, + Methods: map[string]MethodConfig{ + "Check": {TimeoutMS: 80}, + }, + Retry: &RetryConfig{MaxAttempts: 3}, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = conn.Close() }) + + client := grpcHealthV1.NewHealthClient(conn) + start := time.Now() + _, err = client.Check(context.Background(), &grpcHealthV1.HealthCheckRequest{}) + + require.Equal(t, codes.DeadlineExceeded, status.Code(err)) + require.Less(t, time.Since(start), 150*time.Millisecond) +} + +func TestNewInsecureClientConn_MethodMaxAttemptsOverridesDefault(t *testing.T) { + svc := &testHealthServer{failUntil: 10} + addr, stop := startHealthServer(t, svc) + defer stop() + + conn, err := NewInsecureClientConn(addr, Config{ + RequestTimeoutMS: 1000, + Methods: map[string]MethodConfig{ + "Check": {TimeoutMS: 1000, MaxAttempts: 1}, + }, + Retry: &RetryConfig{MaxAttempts: 3}, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = conn.Close() }) + + client := grpcHealthV1.NewHealthClient(conn) + _, err = client.Check(context.Background(), &grpcHealthV1.HealthCheckRequest{}) + + require.Equal(t, codes.Unavailable, status.Code(err)) + require.Equal(t, 1, svc.Calls()) +} + +func TestResolveMethodTimeout_MatchesLowercaseConfigKey(t *testing.T) { + cfg := Config{ + RequestTimeoutMS: 10, + Methods: map[string]MethodConfig{ + "check": {TimeoutMS: 100}, + }, + } + + require.Equal(t, 100*time.Millisecond, resolveMethodTimeout(cfg, "/grpc.health.v1.Health/Check")) +} diff --git a/internal/common/grpc/client/config.go b/internal/common/grpc/client/config.go index 7f6499f..cb701f5 100644 --- a/internal/common/grpc/client/config.go +++ b/internal/common/grpc/client/config.go @@ -26,12 +26,24 @@ var defaultRetryCodes = []codes.Code{ codes.Internal, } +// MethodConfig contains per-RPC outbound gRPC settings. +// TimeoutMS > 0 sets the RPC timeout in milliseconds. +// TimeoutMS == 0 inherits the global RequestTimeoutMS. +// TimeoutMS < 0 disables automatic timeout injection for this RPC. +// MaxAttempts > 0 overrides the global retry attempt limit. +// MaxAttempts == 0 inherits the global retry attempt limit. +type MethodConfig struct { + TimeoutMS int `json:"timeoutMS" mapstructure:"timeoutMS"` + MaxAttempts uint `json:"maxAttempts" mapstructure:"maxAttempts"` +} + // Config contains outbound gRPC settings. type Config struct { - Name string - BaseURL string - RequestTimeoutMS int - Retry *RetryConfig + Name string `json:"name" mapstructure:"name"` + BaseURL string `json:"baseURL" mapstructure:"baseURL"` + RequestTimeoutMS int `json:"requestTimeoutMS" mapstructure:"requestTimeoutMS"` + Retry *RetryConfig `json:"retry" mapstructure:"retry"` + Methods map[string]MethodConfig `json:"methods" mapstructure:"methods"` } // RetryConfig contains outbound gRPC retry settings. diff --git a/internal/common/grpc/client/config_test.go b/internal/common/grpc/client/config_test.go index e67ef72..baa9a56 100644 --- a/internal/common/grpc/client/config_test.go +++ b/internal/common/grpc/client/config_test.go @@ -43,3 +43,23 @@ func TestNewClientConfigReturnsIndependentRetrySlices(t *testing.T) { first.Retry.RetryCodes[0] = codes.Aborted require.Equal(t, codes.Unavailable, second.Retry.RetryCodes[0]) } + +func TestConfig_MethodsConfiguration(t *testing.T) { + cfg := &Config{ + RequestTimeoutMS: 500, + Methods: map[string]MethodConfig{ + "Initialize": {TimeoutMS: 30000}, + "GenerateKey": {TimeoutMS: 30000, MaxAttempts: 1}, + "GetPublicKey": {TimeoutMS: 30000}, + "SignMessage": {TimeoutMS: 500, MaxAttempts: 1}, + }, + } + + require.Equal(t, 500, cfg.RequestTimeoutMS) + require.Equal(t, 30000, cfg.Methods["Initialize"].TimeoutMS) + require.Equal(t, 30000, cfg.Methods["GenerateKey"].TimeoutMS) + require.Equal(t, uint(1), cfg.Methods["GenerateKey"].MaxAttempts) + require.Equal(t, 30000, cfg.Methods["GetPublicKey"].TimeoutMS) + require.Equal(t, 500, cfg.Methods["SignMessage"].TimeoutMS) + require.Equal(t, uint(1), cfg.Methods["SignMessage"].MaxAttempts) +} diff --git a/internal/common/grpc/server/config.go b/internal/common/grpc/server/config.go index 7d8d8f8..d04e375 100644 --- a/internal/common/grpc/server/config.go +++ b/internal/common/grpc/server/config.go @@ -15,14 +15,27 @@ package server // Config represents the configuration for the gRPC server. type Config struct { - Host string `mapstructure:"host"` - Port int `mapstructure:"port"` - TLS *TLSConfig `mapstructure:"tls"` + Host string `mapstructure:"host"` + Port int `mapstructure:"port"` + TLS *TLSConfig `mapstructure:"tls"` + Reflection ReflectionConfig `mapstructure:"reflection"` +} + +// ReflectionConfig represents the gRPC server reflection configuration. +type ReflectionConfig struct { + Enabled bool `mapstructure:"enabled"` } // TLSConfig represents the TLS configuration to the gRPC server. type TLSConfig struct { + Enabled bool `mapstructure:"enabled"` + Cert string `mapstructure:"cert"` + Key string `mapstructure:"key"` + ClientAuth ClientAuthConfig `mapstructure:"clientAuth"` +} + +// ClientAuthConfig represents TLS client certificate authentication. +type ClientAuthConfig struct { Enabled bool `mapstructure:"enabled"` - Cert string `mapstructure:"cert"` - Key string `mapstructure:"key"` + CA string `mapstructure:"ca"` } diff --git a/internal/common/grpc/server/interceptor/recovery.go b/internal/common/grpc/server/interceptor/recovery.go index 4a064c0..2d80c4e 100644 --- a/internal/common/grpc/server/interceptor/recovery.go +++ b/internal/common/grpc/server/interceptor/recovery.go @@ -65,7 +65,7 @@ func recoveryWithLogger(logger *logging.Logger) grpc.UnaryServerInterceptor { err = status.Error(codes.Unavailable, "connection closed") } } else { - err = status.Errorf(codes.Internal, "something went wrong: %v", r) + err = status.Error(codes.Internal, "something went wrong") } } }() diff --git a/internal/common/grpc/server/interceptor/recovery_test.go b/internal/common/grpc/server/interceptor/recovery_test.go index 620cc3f..e8b8064 100644 --- a/internal/common/grpc/server/interceptor/recovery_test.go +++ b/internal/common/grpc/server/interceptor/recovery_test.go @@ -37,7 +37,6 @@ func TestRecoveryWithLogger(t *testing.T) { expectedResp interface{} expectedError error expectPanic bool - panicValue interface{} checkGRPCError bool expectCode codes.Code }{ @@ -61,7 +60,6 @@ func TestRecoveryWithLogger(t *testing.T) { name: "Panic Recovery with String", req: "test request", expectPanic: true, - panicValue: "test panic", handler: func(_ context.Context, _ interface{}) (interface{}, error) { panic("test panic") }, @@ -72,7 +70,6 @@ func TestRecoveryWithLogger(t *testing.T) { name: "Panic Recovery with Nil", req: "test request", expectPanic: true, - panicValue: nil, handler: func(_ context.Context, _ interface{}) (interface{}, error) { panic(nil) }, @@ -83,13 +80,6 @@ func TestRecoveryWithLogger(t *testing.T) { name: "Panic Recovery with Struct", req: "test request", expectPanic: true, - panicValue: struct { - Message string - Code int - }{ - Message: "structured panic", - Code: 500, - }, handler: func(_ context.Context, _ interface{}) (interface{}, error) { panic(struct { Message string @@ -106,7 +96,6 @@ func TestRecoveryWithLogger(t *testing.T) { name: "Panic Recovery with Error", req: "test request", expectPanic: true, - panicValue: errors.New("panic error"), handler: func(_ context.Context, _ interface{}) (interface{}, error) { panic(errors.New("panic error")) }, @@ -117,14 +106,6 @@ func TestRecoveryWithLogger(t *testing.T) { name: "Broken Pipe Panic", req: "test request", expectPanic: true, - panicValue: &net.OpError{ - Op: "write", - Net: "tcp", - Err: &os.SyscallError{ - Syscall: "write", - Err: errors.New("broken pipe"), - }, - }, handler: func(_ context.Context, _ interface{}) (interface{}, error) { panic(&net.OpError{ Op: "write", @@ -145,14 +126,6 @@ func TestRecoveryWithLogger(t *testing.T) { name: "Connection Reset Panic", req: "test request", expectPanic: true, - panicValue: &net.OpError{ - Op: "write", - Net: "tcp", - Err: &os.SyscallError{ - Syscall: "write", - Err: errors.New("connection reset by peer"), - }, - }, handler: func(_ context.Context, _ interface{}) (interface{}, error) { panic(&net.OpError{ Op: "write", @@ -229,12 +202,7 @@ func TestRecoveryWithLogger(t *testing.T) { if tc.expectCode == codes.Unavailable { assert.Contains(t, st.Message(), "connection closed") } else { - assert.Contains(t, st.Message(), "something went wrong") - - // Check panic value in error message if it's a string - if panicStr, ok := tc.panicValue.(string); ok { - assert.Contains(t, st.Message(), panicStr) - } + assert.Equal(t, "something went wrong", st.Message()) } } } else if tc.expectedError != nil { diff --git a/internal/common/grpc/server/option.go b/internal/common/grpc/server/option.go index 8be0e06..befe6ee 100644 --- a/internal/common/grpc/server/option.go +++ b/internal/common/grpc/server/option.go @@ -14,9 +14,14 @@ package server import ( + "bytes" + "crypto/tls" + "crypto/x509" + "encoding/pem" "fmt" "log" "net" + "os" "github.com/mdlayher/vsock" "google.golang.org/grpc" @@ -84,16 +89,77 @@ func WithHealthServer(serviceNames ...string) RunnableOption { // WithTLS creates a TLS server option from the given TLS configuration. func WithTLS(cfg *TLSConfig) ([]grpc.ServerOption, error) { - if cfg != nil && cfg.Enabled { - if cfg.Cert == "" || cfg.Key == "" { - return nil, fmt.Errorf("TLS is enabled but cert/key paths are not configured") + if cfg == nil { + return []grpc.ServerOption{}, nil + } + if !cfg.Enabled { + if cfg.ClientAuth.Enabled { + return nil, fmt.Errorf("TLS client authentication requires TLS to be enabled") + } + return []grpc.ServerOption{}, nil + } + if cfg.Cert == "" || cfg.Key == "" { + return nil, fmt.Errorf("TLS is enabled but cert/key paths are not configured") + } + if cfg.ClientAuth.Enabled && cfg.ClientAuth.CA == "" { + return nil, fmt.Errorf("TLS client authentication is enabled but CA path is not configured") + } + + certificate, err := tls.LoadX509KeyPair(cfg.Cert, cfg.Key) + if err != nil { + return nil, fmt.Errorf("failed to load TLS credentials: %w", err) + } + tlsConfig := &tls.Config{ + Certificates: []tls.Certificate{certificate}, + MinVersion: tls.VersionTLS12, + } + + if cfg.ClientAuth.Enabled { + clientCAPEM, err := os.ReadFile(cfg.ClientAuth.CA) + if err != nil { + return nil, fmt.Errorf("failed to read TLS client CA: %w", err) } - log.Printf("loading TLS config for gRPC server") - creds, err := credentials.NewServerTLSFromFile(cfg.Cert, cfg.Key) + clientCAs, err := certPoolFromPEM(clientCAPEM) if err != nil { - return nil, fmt.Errorf("failed to load TLS credentials: %w", err) + return nil, fmt.Errorf("failed to parse TLS client CA %q: %w", cfg.ClientAuth.CA, err) } - return []grpc.ServerOption{grpc.Creds(creds)}, nil + tlsConfig.ClientCAs = clientCAs + tlsConfig.ClientAuth = tls.RequireAndVerifyClientCert + } + + log.Printf( + "loaded TLS config for gRPC server (client authentication enabled=%t)", + cfg.ClientAuth.Enabled, + ) + return []grpc.ServerOption{grpc.Creds(credentials.NewTLS(tlsConfig))}, nil +} + +func certPoolFromPEM(data []byte) (*x509.CertPool, error) { + certPool := x509.NewCertPool() + remaining := bytes.TrimSpace(data) + certificateCount := 0 + + for len(remaining) > 0 { + if !bytes.HasPrefix(remaining, []byte("-----BEGIN CERTIFICATE-----")) { + return nil, fmt.Errorf("bundle contains invalid PEM data") + } + + block, rest := pem.Decode(remaining) + if block == nil || block.Type != "CERTIFICATE" || len(block.Headers) != 0 { + return nil, fmt.Errorf("bundle contains an invalid certificate PEM block") + } + certificate, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return nil, fmt.Errorf("bundle contains an invalid certificate: %w", err) + } + + certPool.AddCert(certificate) + certificateCount++ + remaining = bytes.TrimSpace(rest) + } + + if certificateCount == 0 { + return nil, fmt.Errorf("bundle contains no certificates") } - return []grpc.ServerOption{}, nil + return certPool, nil } diff --git a/internal/common/grpc/server/option_test.go b/internal/common/grpc/server/option_test.go index 4c62f57..4ed0efe 100644 --- a/internal/common/grpc/server/option_test.go +++ b/internal/common/grpc/server/option_test.go @@ -14,13 +14,17 @@ package server import ( + "bytes" "context" "crypto/ecdsa" "crypto/elliptic" "crypto/rand" + "crypto/tls" "crypto/x509" "crypto/x509/pkix" "encoding/pem" + "fmt" + "log" "math/big" "net" "os" @@ -34,6 +38,7 @@ import ( "google.golang.org/grpc/credentials/insecure" grpcHealth "google.golang.org/grpc/health" grpcHealthV1 "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/test/bufconn" ) func TestWithListener_TCPSuccess(t *testing.T) { @@ -132,12 +137,24 @@ func TestWithTLS(t *testing.T) { }) t.Run("disabled config returns no options", func(t *testing.T) { - certPath, keyPath := writeSelfSignedKeyPair(t, t.TempDir()) + certPath, keyPath := writeServerKeyPair(t, t.TempDir()) opts, err := WithTLS(&TLSConfig{Enabled: false, Cert: certPath, Key: keyPath}) require.NoError(t, err) require.Empty(t, opts) }) + t.Run("client authentication requires TLS", func(t *testing.T) { + opts, err := WithTLS(&TLSConfig{ + Enabled: false, + ClientAuth: ClientAuthConfig{ + Enabled: true, + CA: "/unused", + }, + }) + require.Error(t, err) + require.Nil(t, opts) + }) + t.Run("enabled with empty cert or key returns error", func(t *testing.T) { opts, err := WithTLS(&TLSConfig{Enabled: true, Cert: "", Key: ""}) require.Error(t, err) @@ -167,77 +184,375 @@ func TestWithTLS(t *testing.T) { require.Nil(t, opts) }) - t.Run("enabled secures the server with a working TLS handshake", func(t *testing.T) { - certPath, keyPath := writeSelfSignedKeyPair(t, t.TempDir()) + t.Run("client authentication requires CA path", func(t *testing.T) { + opts, err := WithTLS(&TLSConfig{ + Enabled: true, + Cert: "/unused/server-cert.pem", + Key: "/unused/server-key.pem", + ClientAuth: ClientAuthConfig{ + Enabled: true, + }, + }) + require.Error(t, err) + require.Nil(t, opts) + }) - opts, err := WithTLS(&TLSConfig{Enabled: true, Cert: certPath, Key: keyPath}) + t.Run("client authentication rejects missing CA file", func(t *testing.T) { + dir := t.TempDir() + certPath, keyPath := writeServerKeyPair(t, dir) + opts, err := WithTLS(&TLSConfig{ + Enabled: true, + Cert: certPath, + Key: keyPath, + ClientAuth: ClientAuthConfig{ + Enabled: true, + CA: filepath.Join(dir, "absent-client-ca.pem"), + }, + }) + require.Error(t, err) + require.Nil(t, opts) + }) + + t.Run("client authentication rejects malformed CA bundle", func(t *testing.T) { + dir := t.TempDir() + certPath, keyPath := writeServerKeyPair(t, dir) + clientCAPath := filepath.Join(dir, "client-ca.pem") + require.NoError(t, os.WriteFile(clientCAPath, []byte("not a PEM certificate"), 0o600)) + + opts, err := WithTLS(&TLSConfig{ + Enabled: true, + Cert: certPath, + Key: keyPath, + ClientAuth: ClientAuthConfig{ + Enabled: true, + CA: clientCAPath, + }, + }) + require.Error(t, err) + require.ErrorContains(t, err, clientCAPath) + require.Nil(t, opts) + }) + + t.Run("client authentication rejects partially corrupted CA bundle", func(t *testing.T) { + dir := t.TempDir() + certPath, keyPath := writeServerKeyPair(t, dir) + clientCAPath, _ := writeCertificateAuthority(t, dir, "client-ca") + clientCAPEM, err := os.ReadFile(clientCAPath) require.NoError(t, err) - require.Len(t, opts, 1) + clientCAPEM = append(clientCAPEM, []byte("\ncorrupted trailing content")...) + require.NoError(t, os.WriteFile(clientCAPath, clientCAPEM, 0o600)) - server := grpc.NewServer(opts...) - healthServer := grpcHealth.NewServer() - grpcHealthV1.RegisterHealthServer(server, healthServer) - healthServer.SetServingStatus("", grpcHealthV1.HealthCheckResponse_SERVING) + opts, err := WithTLS(&TLSConfig{ + Enabled: true, + Cert: certPath, + Key: keyPath, + ClientAuth: ClientAuthConfig{ + Enabled: true, + CA: clientCAPath, + }, + }) + require.Error(t, err) + require.ErrorContains(t, err, clientCAPath) + require.Nil(t, opts) + }) - lis, err := net.Listen("tcp", "127.0.0.1:0") - require.NoError(t, err, "failed to create listener") - go func() { _ = server.Serve(lis) }() - t.Cleanup(server.Stop) + t.Run("client authentication requires trusted client certificate", func(t *testing.T) { + dir := t.TempDir() + serverCAPath, serverCA := writeCertificateAuthority(t, dir, "server-ca") + serverCertPath, serverKeyPath := writeSignedKeyPair( + t, + dir, + "server", + serverCA, + []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + []string{"localhost"}, + ) + clientCAPath, clientCA := writeCertificateAuthority(t, dir, "client-ca") + trustedClientCertPath, trustedClientKeyPath := writeSignedKeyPair( + t, + dir, + "trusted-client", + clientCA, + []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + nil, + ) + _, untrustedClientCA := writeCertificateAuthority(t, dir, "untrusted-client-ca") + untrustedClientCertPath, untrustedClientKeyPath := writeSignedKeyPair( + t, + dir, + "untrusted-client", + untrustedClientCA, + []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + nil, + ) - // A client trusting the server cert completes the TLS handshake. - clientCreds, err := credentials.NewClientTLSFromFile(certPath, "localhost") + opts, err := WithTLS(&TLSConfig{ + Enabled: true, + Cert: serverCertPath, + Key: serverKeyPath, + ClientAuth: ClientAuthConfig{ + Enabled: true, + CA: clientCAPath, + }, + }) require.NoError(t, err) - conn, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(clientCreds)) - require.NoError(t, err, "failed to create grpc client") - t.Cleanup(func() { _ = conn.Close() }) + listener := startHealthServer(t, opts) + + trustedClientCredentials := newClientTLSCredentials( + t, + serverCAPath, + trustedClientCertPath, + trustedClientKeyPath, + ) + require.NoError(t, checkHealth(listener, trustedClientCredentials)) + + serverTrustOnlyCredentials := newClientTLSCredentials(t, serverCAPath, "", "") + require.Error(t, checkHealth(listener, serverTrustOnlyCredentials)) + + untrustedClientCredentials := newClientTLSCredentials( + t, + serverCAPath, + untrustedClientCertPath, + untrustedClientKeyPath, + ) + require.Error(t, checkHealth(listener, untrustedClientCredentials)) + }) - client := grpcHealthV1.NewHealthClient(conn) - waitForStatus(t, client, "", grpcHealthV1.HealthCheckResponse_SERVING) + t.Run("server authenticated TLS accepts client without identity", func(t *testing.T) { + dir := t.TempDir() + serverCAPath, serverCA := writeCertificateAuthority(t, dir, "server-ca") + serverCertPath, serverKeyPath := writeSignedKeyPair( + t, + dir, + "server", + serverCA, + []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + []string{"localhost"}, + ) - // A plaintext client cannot talk to the TLS-secured server. - insecureConn, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + opts, err := WithTLS(&TLSConfig{ + Enabled: true, + Cert: serverCertPath, + Key: serverKeyPath, + }) require.NoError(t, err) - t.Cleanup(func() { _ = insecureConn.Close() }) + listener := startHealthServer(t, opts) - ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) - defer cancel() - _, err = grpcHealthV1.NewHealthClient(insecureConn).Check(ctx, &grpcHealthV1.HealthCheckRequest{}) - require.Error(t, err, "plaintext client must not reach the TLS server") + clientCredentials := newClientTLSCredentials(t, serverCAPath, "", "") + require.NoError(t, checkHealth(listener, clientCredentials)) + + require.Error(t, checkHealth(listener, insecure.NewCredentials())) + }) + + t.Run("logs the successfully loaded client authentication mode", func(t *testing.T) { + for _, clientAuthEnabled := range []bool{false, true} { + t.Run(fmt.Sprintf("enabled=%t", clientAuthEnabled), func(t *testing.T) { + dir := t.TempDir() + certPath, keyPath := writeServerKeyPair(t, dir) + cfg := &TLSConfig{ + Enabled: true, + Cert: certPath, + Key: keyPath, + } + if clientAuthEnabled { + clientCAPath, _ := writeCertificateAuthority(t, dir, "client-ca") + cfg.ClientAuth = ClientAuthConfig{ + Enabled: true, + CA: clientCAPath, + } + } + + var output bytes.Buffer + originalOutput := log.Writer() + log.SetOutput(&output) + t.Cleanup(func() { + log.SetOutput(originalOutput) + }) + + opts, err := WithTLS(cfg) + require.NoError(t, err) + require.NotEmpty(t, opts) + require.Contains( + t, + output.String(), + fmt.Sprintf("client authentication enabled=%t", clientAuthEnabled), + ) + }) + } }) } -// writeSelfSignedKeyPair generates an ECDSA self-signed certificate valid for -// localhost/127.0.0.1 and writes the PEM-encoded cert and key into dir, -// returning their paths. -func writeSelfSignedKeyPair(t *testing.T, dir string) (certPath, keyPath string) { +type testCertificateAuthority struct { + certificate *x509.Certificate + privateKey *ecdsa.PrivateKey +} + +func writeServerKeyPair(t *testing.T, dir string) (string, string) { t.Helper() - priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + _, certificateAuthority := writeCertificateAuthority(t, dir, "server-ca") + return writeSignedKeyPair( + t, + dir, + "server", + certificateAuthority, + []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + []string{"localhost"}, + ) +} + +func writeCertificateAuthority( + t *testing.T, + dir string, + name string, +) (string, testCertificateAuthority) { + t.Helper() + + privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) require.NoError(t, err) - template := x509.Certificate{ - SerialNumber: big.NewInt(1), - Subject: pkix.Name{CommonName: "localhost"}, + template := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: name}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature, + BasicConstraintsValid: true, + IsCA: true, + } + certificateDER, err := x509.CreateCertificate( + rand.Reader, + template, + template, + &privateKey.PublicKey, + privateKey, + ) + require.NoError(t, err) + + certificatePath := filepath.Join(dir, name+".pem") + certificatePEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certificateDER}) + require.NoError(t, os.WriteFile(certificatePath, certificatePEM, 0o600)) + + return certificatePath, testCertificateAuthority{ + certificate: template, + privateKey: privateKey, + } +} + +func writeSignedKeyPair( + t *testing.T, + dir string, + name string, + certificateAuthority testCertificateAuthority, + extendedKeyUsage []x509.ExtKeyUsage, + dnsNames []string, +) (string, string) { + t.Helper() + + privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + + template := &x509.Certificate{ + SerialNumber: big.NewInt(2), + Subject: pkix.Name{CommonName: name}, NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour), - KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, - DNSNames: []string{"localhost"}, - IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, - IsCA: true, + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: extendedKeyUsage, + DNSNames: dnsNames, } - der, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv) + certificateDER, err := x509.CreateCertificate( + rand.Reader, + template, + certificateAuthority.certificate, + &privateKey.PublicKey, + certificateAuthority.privateKey, + ) require.NoError(t, err) - keyBytes, err := x509.MarshalPKCS8PrivateKey(priv) + privateKeyDER, err := x509.MarshalPKCS8PrivateKey(privateKey) require.NoError(t, err) - certPath = filepath.Join(dir, "cert.pem") - keyPath = filepath.Join(dir, "key.pem") - certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) - keyPEM := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyBytes}) - require.NoError(t, os.WriteFile(certPath, certPEM, 0o600)) - require.NoError(t, os.WriteFile(keyPath, keyPEM, 0o600)) + certificatePath := filepath.Join(dir, name+"-cert.pem") + privateKeyPath := filepath.Join(dir, name+"-key.pem") + certificatePEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certificateDER}) + privateKeyPEM := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: privateKeyDER}) + require.NoError(t, os.WriteFile(certificatePath, certificatePEM, 0o600)) + require.NoError(t, os.WriteFile(privateKeyPath, privateKeyPEM, 0o600)) + + return certificatePath, privateKeyPath +} + +func startHealthServer(t *testing.T, opts []grpc.ServerOption) *bufconn.Listener { + t.Helper() + + listener := bufconn.Listen(1024 * 1024) + server := grpc.NewServer(opts...) + healthServer := grpcHealth.NewServer() + grpcHealthV1.RegisterHealthServer(server, healthServer) + healthServer.SetServingStatus("", grpcHealthV1.HealthCheckResponse_SERVING) + + go func() { + _ = server.Serve(listener) + }() + t.Cleanup(func() { + server.Stop() + if err := listener.Close(); err != nil { + t.Errorf("failed to close in-memory listener: %v", err) + } + }) + + return listener +} + +func newClientTLSCredentials( + t *testing.T, + serverCAPath string, + clientCertificatePath string, + clientKeyPath string, +) credentials.TransportCredentials { + t.Helper() + + serverCAPEM, err := os.ReadFile(serverCAPath) + require.NoError(t, err) + serverCAs := x509.NewCertPool() + require.True(t, serverCAs.AppendCertsFromPEM(serverCAPEM)) + + tlsConfig := &tls.Config{ + RootCAs: serverCAs, + ServerName: "localhost", + } + if clientCertificatePath != "" { + clientCertificate, err := tls.LoadX509KeyPair(clientCertificatePath, clientKeyPath) + require.NoError(t, err) + tlsConfig.Certificates = []tls.Certificate{clientCertificate} + } + + return credentials.NewTLS(tlsConfig) +} + +func checkHealth( + listener *bufconn.Listener, + transportCredentials credentials.TransportCredentials, +) error { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + connection, err := grpc.NewClient( + "passthrough:///bufnet", + grpc.WithContextDialer(func(context.Context, string) (net.Conn, error) { + return listener.Dial() + }), + grpc.WithTransportCredentials(transportCredentials), + ) + if err != nil { + return err + } + defer func() { _ = connection.Close() }() - return certPath, keyPath + _, err = grpcHealthV1.NewHealthClient(connection).Check( + ctx, + &grpcHealthV1.HealthCheckRequest{}, + ) + return err } diff --git a/internal/common/telemetry/tracer.go b/internal/common/telemetry/tracer.go index 5903410..e2720ff 100644 --- a/internal/common/telemetry/tracer.go +++ b/internal/common/telemetry/tracer.go @@ -22,7 +22,7 @@ import ( "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" "go.opentelemetry.io/otel/sdk/resource" sdktrace "go.opentelemetry.io/otel/sdk/trace" - semconv "go.opentelemetry.io/otel/semconv/v1.40.0" + semconv "go.opentelemetry.io/otel/semconv/v1.41.0" ) // InitTracer initializes OpenTelemetry tracer provider and propagators. diff --git a/internal/enclave/common/crypto/bls/const.go b/internal/enclave/common/crypto/bls/const.go index 1eb38a9..14a16a4 100644 --- a/internal/enclave/common/crypto/bls/const.go +++ b/internal/enclave/common/crypto/bls/const.go @@ -29,6 +29,6 @@ const ( // Refer to BLS Signatures IETF standard for more details: https://www.ietf.org/archive/id/draft-irtf-cfrg-bls-signature-05.html#section-4.2 // Reference implementation: https://github.com/ava-labs/avalanchego/blob/master/utils/crypto/bls/ciphersuite.go#L13-L16 var ( - // DSTSignature is the domain separation tag for minimal-signature-size BLS signatures. + // DSTSignature is the domain separation tag for minimal-public-key-size BLS signatures. DSTSignature = []byte("BLS_SIG_BLS12381G2_XMD:SHA-256_SSWU_RO_NUL_") ) diff --git a/internal/enclave/common/crypto/ed25519/ed25519.go b/internal/enclave/common/crypto/ed25519/ed25519.go index 4db4bc3..bc1f8cc 100644 --- a/internal/enclave/common/crypto/ed25519/ed25519.go +++ b/internal/enclave/common/crypto/ed25519/ed25519.go @@ -15,9 +15,13 @@ package ed25519 import ( + "bytes" "crypto/ed25519" "crypto/rand" + "errors" "fmt" + + "github.com/hdevalence/ed25519consensus" ) // Key represents an Ed25519 key. @@ -72,12 +76,9 @@ func Deserialize(data []byte) (*Key, error) { return nil, fmt.Errorf("invalid key size: expected %d, got %d", ed25519.PrivateKeySize, len(data)) } - privateKey := ed25519.PrivateKey(data) - - // Verify the key is valid by checking if we can derive the public key - publicKey := privateKey.Public().(ed25519.PublicKey) - if len(publicKey) != ed25519.PublicKeySize { - return nil, fmt.Errorf("failed to derive valid public key") + privateKey := ed25519.NewKeyFromSeed(data[:ed25519.SeedSize]) + if !bytes.Equal(privateKey[ed25519.SeedSize:], data[ed25519.SeedSize:]) { + return nil, errors.New("private key public component does not match seed") } return &Key{ @@ -94,5 +95,6 @@ func VerifySignedMessage(signature []byte, message []byte, publicKey []byte) (bo return false, fmt.Errorf("invalid signature size: expected %d, got %d", ed25519.SignatureSize, len(signature)) } - return ed25519.Verify(ed25519.PublicKey(publicKey), message, signature), nil + // Use ZIP-215 verification so remote signer behavior matches Malachite/Arc consensus semantics. + return ed25519consensus.Verify(ed25519.PublicKey(publicKey), message, signature), nil } diff --git a/internal/enclave/common/crypto/ed25519/ed25519_test.go b/internal/enclave/common/crypto/ed25519/ed25519_test.go index c5de201..4aab843 100644 --- a/internal/enclave/common/crypto/ed25519/ed25519_test.go +++ b/internal/enclave/common/crypto/ed25519/ed25519_test.go @@ -16,6 +16,7 @@ package ed25519 import ( "bytes" "crypto/ed25519" + "encoding/hex" "testing" "github.com/stretchr/testify/suite" @@ -182,6 +183,20 @@ func (s *ed25519TestSuite) TestVerifySignedMessage() { s.True(valid) }) + s.Run("ZIP-215 signature rejected by standard verifier", func() { + // Vector 2 from ed25519consensus v0.2.0's ZIP-215 test suite. + zip215PublicKey, err := hex.DecodeString("0100000000000000000000000000000000000000000000000000000000000000") + s.Require().NoError(err) + zip215Signature, err := hex.DecodeString("c7176a703d4dd84fba3c0b760d10670f2a2053fa2c39ccc64ec7fd7792ac037a0000000000000000000000000000000000000000000000000000000000000000") + s.Require().NoError(err) + zip215Message := []byte("Zcash") + + s.False(ed25519.Verify(zip215PublicKey, zip215Message, zip215Signature)) + valid, err := VerifySignedMessage(zip215Signature, zip215Message, zip215PublicKey) + s.NoError(err) + s.True(valid) + }) + s.Run("signature verification with wrong message fails", func() { wrongMessage := []byte("wrong message") valid, err := VerifySignedMessage(signature, wrongMessage, publicKey) @@ -297,6 +312,40 @@ func (s *ed25519TestSuite) TestDeserialize() { s.True(bytes.Equal(originalSig, deserializedSig), "Signatures should match") }) + s.Run("rejects modified public key component", func() { + malformedKey := append([]byte(nil), serializedKey...) + malformedKey[ed25519.SeedSize] ^= 0xff + + deserializedKey, err := Deserialize(malformedKey) + + s.Error(err) + s.Nil(deserializedKey) + }) + + s.Run("rejects modified seed with original public key component", func() { + malformedKey := append([]byte(nil), serializedKey...) + malformedKey[0] ^= 0xff + + deserializedKey, err := Deserialize(malformedKey) + + s.Error(err) + s.Nil(deserializedKey) + }) + + s.Run("does not retain the input buffer", func() { + input := append([]byte(nil), serializedKey...) + deserializedKey, err := Deserialize(input) + s.Require().NoError(err) + + for i := range input { + input[i] ^= 0xff + } + + got, err := deserializedKey.Serialize() + s.Require().NoError(err) + s.Equal(serializedKey, got) + }) + s.Run("empty data deserialization", func() { deserializedKey, err := Deserialize([]byte{}) s.Error(err) diff --git a/internal/enclave/config.go b/internal/enclave/config.go index 3895cf6..7650085 100644 --- a/internal/enclave/config.go +++ b/internal/enclave/config.go @@ -15,8 +15,11 @@ package enclave import ( + "fmt" + "github.com/circlefin/arc-remote-signer/internal/common/config" grpcServer "github.com/circlefin/arc-remote-signer/internal/common/grpc/server" + "github.com/circlefin/arc-remote-signer/internal/enclave/provider/awsproxy" ) // Config should implement config.ApplicationConfig. @@ -24,14 +27,15 @@ var _ config.ApplicationConfig = &Config{} // Config represents the complete enclave configuration. type Config struct { - // Base config - *config.BaseConfig `mapstructure:",squash"` - // Public provides configuration for the public gRPC server. Public *PublicConfig `mapstructure:"public"` // NitroEnclave contains config for the Nitro enclave. - NitroEnclave *NitroEnclaveConfig + NitroEnclave *NitroEnclaveConfig `mapstructure:"nitroEnclave"` + + // Awsproxy contains config for the enclave-side AWS proxy that bridges + // loopback TCP to the parent (vsock in production, TCP in dev/CI). + Awsproxy *awsproxy.Config `mapstructure:"awsproxy"` } // PublicConfig wraps server configuration to match YAML structure. @@ -43,7 +47,6 @@ type PublicConfig struct { // NewConfig creates a new config with sensible defaults. func NewConfig() *Config { return &Config{ - BaseConfig: config.NewBaseConfig(), Public: &PublicConfig{ Server: &grpcServer.Config{ Host: "127.0.0.1", @@ -51,8 +54,11 @@ func NewConfig() *Config { }, }, NitroEnclave: &NitroEnclaveConfig{ - Enabled: true, + Enabled: true, + AwsproxyEndpoint: fmt.Sprintf("http://127.0.0.1:%d", awsproxy.DefaultBasePort), + KmsConnectTimeoutMs: 3000, }, + Awsproxy: awsproxy.NewConfig(), } } @@ -61,17 +67,26 @@ func (c *Config) GetName() string { return "nitro-enclave-signer-internal" } -// IsProduction returns true if running in production. -func (c *Config) IsProduction() bool { - return c.Env == config.Prod -} - -// IsDevelopment returns true if running in development. -func (c *Config) IsDevelopment() bool { - return c.Env == config.Dev -} - // NitroEnclaveConfig contains config for the Nitro enclave. type NitroEnclaveConfig struct { - Enabled bool + // Enabled controls whether Nitro Enclave features (NSM attestation, vsock) are active. + Enabled bool `mapstructure:"enabled"` + // AwsproxyEndpoint is the URL the in-enclave KMS client uses to + // reach AWS KMS. In both production and dev/CI it points at the + // enclave-side awsproxy loopback listener (default port 10316, see + // awsproxy.DefaultBasePort); the proxy then bridges onward β€” vsock to + // the parent (real KMS) in production, TCP to the standalone vsockproxy + // service (which forwards to LocalStack) in dev/CI. See + // deployments/docker-compose.yaml. + AwsproxyEndpoint string `mapstructure:"awsproxyEndpoint"` + // KmsConnectTimeoutMs is the per-request HTTP timeout in milliseconds + // applied to the in-enclave KMS client. Despite the "Connect" in the + // name (kept for parity with the host-side awskms.Config.ConnectTimeout + // field it threads into), it bounds the whole HTTP request β€” TCP + // dial, TLS handshake, request, and response β€” not just the connect + // phase. Combined with the SDK's default 3 retry attempts and the + // custom 300 ms MaxBackoff in awskms.go, worst-case total time per + // Initialize call is roughly 3*timeout + small backoff. Defaults to + // 3000 ms (3 s). + KmsConnectTimeoutMs int `mapstructure:"kmsConnectTimeoutMs"` } diff --git a/internal/enclave/config_test.go b/internal/enclave/config_test.go index b3d8dae..813e2fc 100644 --- a/internal/enclave/config_test.go +++ b/internal/enclave/config_test.go @@ -14,6 +14,10 @@ package enclave import ( + "fmt" + "os" + "path/filepath" + "reflect" "testing" "github.com/circlefin/arc-remote-signer/internal/common/config" @@ -25,14 +29,10 @@ func TestNewConfig(t *testing.T) { cfg := NewConfig() require.NotNil(t, cfg) - require.NotNil(t, cfg.BaseConfig) require.NotNil(t, cfg.Public) require.NotNil(t, cfg.Public.Server) require.NotNil(t, cfg.NitroEnclave) - // Check base config defaults - assert.Equal(t, config.Dev, cfg.Env) - // Check public server defaults assert.Equal(t, "127.0.0.1", cfg.Public.Server.Host) assert.Equal(t, 5000, cfg.Public.Server.Port) @@ -41,85 +41,16 @@ func TestNewConfig(t *testing.T) { assert.True(t, cfg.NitroEnclave.Enabled) } +func TestConfigHasNoDeploymentEnvironment(t *testing.T) { + _, found := reflect.TypeOf(Config{}).FieldByName("Env") + require.False(t, found, "enclave config must not accept APP_ENV") +} + func TestConfig_GetName(t *testing.T) { cfg := NewConfig() assert.Equal(t, "nitro-enclave-signer-internal", cfg.GetName()) } -func TestConfig_IsProduction(t *testing.T) { - tests := []struct { - name string - env config.Environment - expected bool - }{ - { - name: "Dev environment is not production", - env: config.Dev, - expected: false, - }, - { - name: "QA environment is not production", - env: config.QA, - expected: false, - }, - { - name: "Stg environment is not production", - env: config.Stg, - expected: false, - }, - { - name: "Prod environment is production", - env: config.Prod, - expected: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - cfg := NewConfig() - cfg.Env = tt.env - assert.Equal(t, tt.expected, cfg.IsProduction()) - }) - } -} - -func TestConfig_IsDevelopment(t *testing.T) { - tests := []struct { - name string - env config.Environment - expected bool - }{ - { - name: "Dev environment is development", - env: config.Dev, - expected: true, - }, - { - name: "QA environment is not development", - env: config.QA, - expected: false, - }, - { - name: "Stg environment is not development", - env: config.Stg, - expected: false, - }, - { - name: "Prod environment is not development", - env: config.Prod, - expected: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - cfg := NewConfig() - cfg.Env = tt.env - assert.Equal(t, tt.expected, cfg.IsDevelopment()) - }) - } -} - func TestConfig_ImplementsApplicationConfig(t *testing.T) { cfg := NewConfig() var _ config.ApplicationConfig = cfg @@ -149,20 +80,121 @@ func TestNitroEnclaveConfig_Disabled(t *testing.T) { assert.False(t, cfg.NitroEnclave.Enabled) } +// TestLoadConfig_YamlOverridesMutatedDefaults guards against a +// mapstructure tag drift bug: if a tag does not match the key that +// yaml.Marshal of the defaults produces (lowercased Go field name with +// no separator), the YAML subtree is silently dropped and NewConfig +// defaults take over. We detect that by mutating the defaults before +// LoadConfig and verifying the YAML value wins. +func TestLoadConfig_YamlOverridesMutatedDefaults(t *testing.T) { + yamlPath := filepath.Join(t.TempDir(), "enclave.yaml") + const yamlBody = `public: + server: + host: 1.2.3.4 + port: 11111 +nitroEnclave: + enabled: false + awsproxyEndpoint: http://example:8080 + kmsConnectTimeoutMs: 4242 +` + require.NoError(t, os.WriteFile(yamlPath, []byte(yamlBody), 0o600)) + + cfg := NewConfig() + cfg.Public.Server.Host = "DEFAULT-HOST" + cfg.Public.Server.Port = 99999 + cfg.NitroEnclave.Enabled = true + cfg.NitroEnclave.AwsproxyEndpoint = "DEFAULT-ENDPOINT" + cfg.NitroEnclave.KmsConnectTimeoutMs = 99999 + + config.LoadConfig(cfg, yamlPath) + + assert.Equal(t, "1.2.3.4", cfg.Public.Server.Host) + assert.Equal(t, 11111, cfg.Public.Server.Port) + assert.False(t, cfg.NitroEnclave.Enabled, + "YAML nitroEnclave.enabled=false must override mutated default Enabled=true") + assert.Equal(t, "http://example:8080", cfg.NitroEnclave.AwsproxyEndpoint, + "YAML nitroEnclave.awsproxyEndpoint must override mutated default") + assert.Equal(t, 4242, cfg.NitroEnclave.KmsConnectTimeoutMs, + "YAML nitroEnclave.kmsConnectTimeoutMs must override mutated default") +} + +// TestLoadConfig_EnvOverridesYaml pins the env-var override path used by +// docker-compose to point the in-enclave KMS client at LocalStack in dev +// mode. The docker-compose file sets APP_NITROENCLAVE_AWSPROXYENDPOINT, +// expecting it to flow through viper's env replacer into +// NitroEnclaveConfig.AwsproxyEndpoint. If the mapstructure tag spelling +// or viper key normalisation drifts, this test fails fast so dev runs +// don't silently fall back to the production default. +func TestLoadConfig_EnvOverridesYaml(t *testing.T) { + yamlPath := filepath.Join(t.TempDir(), "enclave.yaml") + const yamlBody = `nitroEnclave: + enabled: true + awsproxyEndpoint: http://yaml-default:10316 +` + require.NoError(t, os.WriteFile(yamlPath, []byte(yamlBody), 0o600)) + + t.Setenv("APP_NITROENCLAVE_AWSPROXYENDPOINT", "http://localstack:4566") + + cfg := NewConfig() + config.LoadConfig(cfg, yamlPath) + + assert.Equal(t, "http://localstack:4566", cfg.NitroEnclave.AwsproxyEndpoint, + "APP_NITROENCLAVE_AWSPROXYENDPOINT must override YAML and default value") +} + +func TestValidateAwsproxyConfig(t *testing.T) { + t.Run("matching default ports pass", func(t *testing.T) { + require.NoError(t, validateAwsproxyConfig(NewConfig())) + }) + + t.Run("port mismatch is rejected", func(t *testing.T) { + cfg := NewConfig() + cfg.Awsproxy.BasePort = cfg.Awsproxy.BasePort + 1 // diverge from the endpoint port + err := validateAwsproxyConfig(cfg) + require.Error(t, err) + assert.Contains(t, err.Error(), "config mismatch") + }) + + t.Run("localhost host is accepted", func(t *testing.T) { + cfg := NewConfig() + cfg.NitroEnclave.AwsproxyEndpoint = fmt.Sprintf("http://localhost:%d", cfg.Awsproxy.BasePort) + require.NoError(t, validateAwsproxyConfig(cfg)) + }) + + t.Run("non-loopback host is rejected", func(t *testing.T) { + cfg := NewConfig() + cfg.NitroEnclave.AwsproxyEndpoint = fmt.Sprintf("http://evil.example:%d", cfg.Awsproxy.BasePort) + err := validateAwsproxyConfig(cfg) + require.Error(t, err) + assert.Contains(t, err.Error(), "loopback") + }) + + t.Run("endpoint without a port is rejected", func(t *testing.T) { + cfg := NewConfig() + cfg.NitroEnclave.AwsproxyEndpoint = "http://127.0.0.1" + err := validateAwsproxyConfig(cfg) + require.Error(t, err) + assert.Contains(t, err.Error(), "has no port") + }) + + t.Run("malformed endpoint is rejected", func(t *testing.T) { + cfg := NewConfig() + cfg.NitroEnclave.AwsproxyEndpoint = "://bad" + err := validateAwsproxyConfig(cfg) + require.Error(t, err) + }) +} + func TestConfig_CustomValues(t *testing.T) { cfg := NewConfig() // Modify config - cfg.Env = config.Prod cfg.Public.Server.Host = "0.0.0.0" cfg.Public.Server.Port = 8080 cfg.NitroEnclave.Enabled = false // Verify modifications - assert.Equal(t, config.Prod, cfg.Env) assert.Equal(t, "0.0.0.0", cfg.Public.Server.Host) assert.Equal(t, 8080, cfg.Public.Server.Port) assert.False(t, cfg.NitroEnclave.Enabled) - assert.True(t, cfg.IsProduction()) - assert.False(t, cfg.IsDevelopment()) } diff --git a/internal/enclave/enclave.go b/internal/enclave/enclave.go index df87b9a..8b577a3 100644 --- a/internal/enclave/enclave.go +++ b/internal/enclave/enclave.go @@ -16,12 +16,17 @@ package enclave import ( "context" + "fmt" + "net/url" + "strconv" "sync" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/circlefin/arc-remote-signer/internal/common/lifecycle" "github.com/circlefin/arc-remote-signer/internal/common/logging" + "github.com/circlefin/arc-remote-signer/internal/enclave/provider/awskms" + "github.com/circlefin/arc-remote-signer/internal/enclave/provider/awsproxy" "github.com/circlefin/arc-remote-signer/internal/enclave/provider/enclave" - "github.com/circlefin/arc-remote-signer/internal/enclave/provider/keystore" "github.com/circlefin/arc-remote-signer/internal/enclave/public" enclaveSvc "github.com/circlefin/arc-remote-signer/internal/enclave/service/enclave" ) @@ -38,14 +43,41 @@ func getLogger() *logging.Logger { return _logger } -// Run runs nitro enclave gRPC server over vsock or tcp. -func Run(cfg *Config) error { +type awsProxyFactory func(*Config) (awsproxy.Provider, error) + +func newAWSKMSFactory(cfg *Config, nitroEnabled bool, enclavePvd enclave.Provider) awskms.Factory { + return func(callCtx context.Context, awsCfg aws.Config, arns []string) (awskms.Provider, error) { + kmsConfig := &awskms.Config{ + Arns: arns, + ConnectTimeout: cfg.NitroEnclave.KmsConnectTimeoutMs, + AwsproxyEndpoint: cfg.NitroEnclave.AwsproxyEndpoint, + } + if !nitroEnabled { + return awskms.NewForDevelopment(callCtx, kmsConfig, awsCfg) + } + if enclavePvd == nil { + return nil, fmt.Errorf("cannot use Nitro mode without an enclave provider") + } + attestationDocument, err := enclavePvd.AttestKMSRecipient() + if err != nil { + return nil, fmt.Errorf("the enclave could not attest the KMS recipient: %w", err) + } + return awskms.NewWithAttestation(callCtx, kmsConfig, awsCfg, attestationDocument) + } +} + +func run(cfg *Config, nitroEnabled bool, newAWSProxy awsProxyFactory) error { var ctx = context.Background() var err error + if err := validateAwsproxyConfig(cfg); err != nil { + getLogger().ErrorErr(ctx, "invalid awsproxy configuration", err, nil) + panic(err) + } + getLogger().Info(ctx, "initializing the providers...", nil) var enclavePvd enclave.Provider - if cfg.NitroEnclave.Enabled { + if nitroEnabled { enclavePvd, err = enclave.New() if err != nil { panic(err) @@ -54,23 +86,76 @@ func Run(cfg *Config) error { // Initialize enclave services getLogger().Info(ctx, "initializing the services...", nil) - enclaveService := enclaveSvc.New(cfg.NitroEnclave.Enabled, keystore.New(), enclavePvd) + awskmsFactory := newAWSKMSFactory(cfg, nitroEnabled, enclavePvd) + enclaveService := enclaveSvc.New( + nitroEnabled, + enclavePvd, + awskmsFactory, + cfg.NitroEnclave.AwsproxyEndpoint, + ) // Create server with engine initialization handled in public package server, err := public.New(cfg.Public.Server, public.CreateServerParams{ ServiceName: cfg.GetName(), - Env: cfg.Env, EnclaveService: enclaveService, - NitroEnclaveEnabled: cfg.NitroEnclave.Enabled, + NitroEnclaveEnabled: nitroEnabled, }) if err != nil { getLogger().ErrorErr(ctx, "failed to create server", err, nil) panic(err) } - // Create and manage server with lifecycle + awsProxy, err := newAWSProxy(cfg) + if err != nil { + getLogger().ErrorErr(ctx, "failed to construct awsproxy", err, nil) + panic(err) + } + + // lifecycle.Manager shuts down in Manage order, so Manage the server + // before the proxy: on SIGTERM the server stops first (rejects new + // RPCs) and the proxy drains last (lets in-flight KMS calls finish). lc := lifecycle.NewManager() lc.Manage(server) + lc.Manage(awsProxy) lc.Run() return nil } + +// validateAwsproxyConfig fails fast when the in-enclave KMS client's dial +// target is misconfigured. It enforces two invariants on +// cfg.NitroEnclave.AwsproxyEndpoint: the host must be loopback (the SDK +// dial is redirected straight to it, so a non-loopback host would route KMS +// traffic off-box), and the port must equal cfg.Awsproxy.BasePort (the port +// the proxy actually binds). The client endpoint and the proxy's base_port +// are configured independently and only align by sharing the same default, +// so a partial override would otherwise surface as an opaque +// connection-refused at Initialize rather than a clear config error. +// (Collapsing these to a single source of truth is tracked as a follow-up.) +func validateAwsproxyConfig(cfg *Config) error { + u, err := url.Parse(cfg.NitroEnclave.AwsproxyEndpoint) + if err != nil { + return fmt.Errorf("parse nitroEnclave.awsproxyEndpoint %q: %w", cfg.NitroEnclave.AwsproxyEndpoint, err) + } + // The endpoint host is the literal TCP dial target (awskms.newAwsproxyHTTPClient + // redirects every SDK dial to it), so it must stay on loopback: the awsproxy + // listener binds 127.0.0.1 and the in-enclave KMS invariant is that traffic + // never leaves the box before the proxy bridges it. Reject any other host so a + // misconfigured endpoint fails loudly at startup rather than silently routing + // KMS traffic (plaintext, in dev/CI) off-box. + if host := u.Hostname(); host != "127.0.0.1" && host != "localhost" { + return fmt.Errorf( + "nitroEnclave.awsproxyEndpoint %q host %q must be a loopback address (127.0.0.1 or localhost)", + cfg.NitroEnclave.AwsproxyEndpoint, host) + } + port := u.Port() + if port == "" { + return fmt.Errorf("nitroEnclave.awsproxyEndpoint %q has no port", cfg.NitroEnclave.AwsproxyEndpoint) + } + if want := strconv.FormatUint(uint64(cfg.Awsproxy.BasePort), 10); port != want { + return fmt.Errorf( + "config mismatch: nitroEnclave.awsproxyEndpoint port %s must equal awsproxy.base_port %s "+ + "(the in-enclave KMS client dials the endpoint; the awsproxy binds base_port)", + port, want) + } + return nil +} diff --git a/internal/enclave/enclave_test.go b/internal/enclave/enclave_test.go new file mode 100644 index 0000000..b7eab7e --- /dev/null +++ b/internal/enclave/enclave_test.go @@ -0,0 +1,105 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !prod + +package enclave + +import ( + "context" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/circlefin/arc-remote-signer/internal/enclave/provider/awsproxy" + "github.com/stretchr/testify/require" +) + +func shutdownAWSProxyOnCleanup(t *testing.T, pvd awsproxy.Provider) { + t.Helper() + t.Cleanup(func() { + if err := pvd.Shutdown(context.Background()); err != nil { + t.Errorf("shutdown AWS proxy: %v", err) + } + }) +} + +// TestSelectTransport checks development transport selection. +// Nitro configuration uses VSOCK. Other configuration uses TCP. +func TestSelectTransport(t *testing.T) { + tests := map[string]struct { + enabled bool + want transportKind + }{ + "nitro enabled selects vsock": {enabled: true, want: transportVsock}, + "nitro disabled selects tcp": {enabled: false, want: transportTCP}, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + cfg := NewConfig() + cfg.NitroEnclave.Enabled = tt.enabled + require.Equal(t, tt.want, selectTransport(cfg)) + }) + } +} + +func TestBuildAWSProxy_SelectsTCP(t *testing.T) { + cfg := NewConfig() + cfg.NitroEnclave.Enabled = false + + pvd, err := buildAWSProxy(cfg) + + require.NoError(t, err) + require.NotNil(t, pvd) + shutdownAWSProxyOnCleanup(t, pvd) +} + +func TestNewAWSKMSFactory_RequestsFreshRecipientAttestation(t *testing.T) { + cfg := NewConfig() + enclavePvd := &recipientAttestationProvider{ + documents: [][]byte{ + []byte("attestation-document-1"), + []byte("attestation-document-2"), + }, + } + factory := newAWSKMSFactory(cfg, true, enclavePvd) + awsCfg := aws.Config{Region: "us-east-1"} + arns := []string{"arn:aws:kms:us-east-1:123456789012:key/test"} + + first, err := factory(context.Background(), awsCfg, arns) + require.NoError(t, err) + second, err := factory(context.Background(), awsCfg, arns) + + require.NoError(t, err) + require.NotNil(t, first) + require.NotNil(t, second) + require.Equal(t, 2, enclavePvd.attestationCalls) +} + +type recipientAttestationProvider struct { + documents [][]byte + attestationCalls int +} + +func (*recipientAttestationProvider) DecryptKMSEnvelopedKey([]byte) ([]byte, error) { + return nil, nil +} + +func (p *recipientAttestationProvider) AttestKMSRecipient() ([]byte, error) { + document := p.documents[p.attestationCalls] + p.attestationCalls++ + return document, nil +} + +func (*recipientAttestationProvider) Attest([]byte) ([]byte, error) { + return nil, nil +} diff --git a/internal/enclave/enclave_vsock.go b/internal/enclave/enclave_vsock.go new file mode 100644 index 0000000..7c32f1f --- /dev/null +++ b/internal/enclave/enclave_vsock.go @@ -0,0 +1,26 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux + +package enclave + +import ( + "github.com/circlefin/arc-remote-signer/internal/enclave/provider/awsproxy" +) + +// buildAwsProxyVsock returns the production vsock-transport awsproxy. +// Linux-only because awsproxy.WithVsockTransport is Linux-only. +func buildAwsProxyVsock(cfg *Config) (awsproxy.Provider, error) { + return awsproxy.New(cfg.Awsproxy, awsproxy.WithVsockTransport()) +} diff --git a/internal/enclave/enclave_vsock_stub.go b/internal/enclave/enclave_vsock_stub.go new file mode 100644 index 0000000..6edf268 --- /dev/null +++ b/internal/enclave/enclave_vsock_stub.go @@ -0,0 +1,33 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !linux + +package enclave + +import ( + "errors" + + "github.com/circlefin/arc-remote-signer/internal/enclave/provider/awsproxy" +) + +// buildAwsProxyVsock is defined in a build-tagged stub because its Linux +// counterpart calls awsproxy.WithVsockTransport, which is Linux-only and +// cannot be referenced from a non-Linux build β€” this file lets the package +// compile on non-Linux hosts (cross-compile sanity builds). It is never +// reached at runtime there because cfg.NitroEnclave.Enabled is false in +// dev/CI, so selectTransport picks TCP. It returns an explicit error rather +// than a no-op provider so any accidental reach fails loudly. +func buildAwsProxyVsock(_ *Config) (awsproxy.Provider, error) { + return nil, errors.New("enclave: vsock transport requires Linux build") +} diff --git a/internal/app/service/signer/cache.go b/internal/enclave/enclave_vsock_stub_test.go similarity index 59% rename from internal/app/service/signer/cache.go rename to internal/enclave/enclave_vsock_stub_test.go index 99a233d..36ac5f7 100644 --- a/internal/app/service/signer/cache.go +++ b/internal/enclave/enclave_vsock_stub_test.go @@ -11,36 +11,22 @@ // See the License for the specific language governing permissions and // limitations under the License. -package signer +//go:build !linux && !prod + +package enclave import ( - "sync" + "testing" - "github.com/circlefin/arc-remote-signer/proto/pb" + "github.com/stretchr/testify/require" ) -type cache struct { - mu sync.RWMutex - key *key -} - -type key struct { - encryptedKeyMaterial *pb.EncryptedKeyMaterial - publicKey []byte -} +func TestBuildAWSProxy_RejectsVSOCK(t *testing.T) { + cfg := NewConfig() + cfg.NitroEnclave.Enabled = true -func newCache() *cache { - return &cache{} -} - -func (c *cache) set(k *key) { - c.mu.Lock() - defer c.mu.Unlock() - c.key = k -} + pvd, err := buildAWSProxy(cfg) -func (c *cache) get() *key { - c.mu.RLock() - defer c.mu.RUnlock() - return c.key + require.EqualError(t, err, "enclave: vsock transport requires Linux build") + require.Nil(t, pvd) } diff --git a/internal/enclave/enclave_vsock_test.go b/internal/enclave/enclave_vsock_test.go new file mode 100644 index 0000000..e2ca558 --- /dev/null +++ b/internal/enclave/enclave_vsock_test.go @@ -0,0 +1,36 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux && !prod + +package enclave + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// TestBuildAWSProxy_SelectsVSOCK checks VSOCK provider construction in development. +// TestSelectTransport checks only the transport value. The VSOCK provider compiles +// only on Linux. Construction does not start the listener or require a VSOCK device. +func TestBuildAWSProxy_SelectsVSOCK(t *testing.T) { + cfg := NewConfig() + cfg.NitroEnclave.Enabled = true + + pvd, err := buildAWSProxy(cfg) + + require.NoError(t, err) + require.NotNil(t, pvd) + shutdownAWSProxyOnCleanup(t, pvd) +} diff --git a/internal/enclave/provider/awskms/amazonroots.go b/internal/enclave/provider/awskms/amazonroots.go new file mode 100644 index 0000000..d7cff9a --- /dev/null +++ b/internal/enclave/provider/awskms/amazonroots.go @@ -0,0 +1,137 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package awskms + +import ( + "bytes" + "crypto/sha256" + "crypto/x509" + // The blank import enables the go:embed directive. + _ "embed" + "encoding/pem" + "errors" + "fmt" +) + +const ( + amazonRootCACount = 5 + invalidAmazonRootCAPEMMessage = "amazon root CA bundle contains invalid PEM data" +) + +var approvedAmazonRootCAFingerprints = map[string]struct{}{ + "8ecde6884f3d87b1125ba31ac3fcb13d7016de7f57cc904fe1cb97c6ae98196e": {}, + "1ba5b2aa8c65401a82960118f80bec4f62304d83cec4713a19c39c011ea46db4": {}, + "18ce6cfe7bf14e60b2e347b8dfe868cb31d02ebb3ada271569f50343b46db3a4": {}, + "e35d28419ed02025cfa69038cd623962458da5c695fbdea3c22b0bfb25897092": {}, + "568d6905a2c88708a4b3025190edcfedb1974a606a13c6e5290fcb2ae63edab5": {}, +} + +// amazonRootCAPEM contains the self-signed roots that Amazon Trust Services +// recommends for custom trust stores. The source is +// https://www.amazontrust.com/repository/. +// +//go:embed certs/amazon-trust-roots.pem +var amazonRootCAPEM []byte + +func newAmazonRootCAPool() (*x509.CertPool, error) { + return parseAmazonRootCAPool(amazonRootCAPEM) +} + +func parseAmazonRootCAPool(bundle []byte) (*x509.CertPool, error) { + pool := x509.NewCertPool() + remaining := bundle + certificateCount := 0 + seenFingerprints := make(map[string]struct{}, amazonRootCACount) + + for len(remaining) > 0 { + block, rest, err := decodeStrictPEMBlock(remaining) + if err != nil { + return nil, err + } + if block.Type != "CERTIFICATE" || len(block.Headers) != 0 { + return nil, fmt.Errorf("amazon root CA bundle contains invalid PEM block %q", block.Type) + } + certificate, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return nil, fmt.Errorf("parse Amazon root CA certificate: %w", err) + } + if err := certificate.CheckSignatureFrom(certificate); err != nil { + return nil, fmt.Errorf("amazon root CA certificate is not self-signed: %w", err) + } + fingerprint := fmt.Sprintf("%x", sha256.Sum256(block.Bytes)) + if _, approved := approvedAmazonRootCAFingerprints[fingerprint]; !approved { + return nil, fmt.Errorf( + "amazon root CA certificate has unapproved SHA-256 fingerprint %s", + fingerprint, + ) + } + if _, duplicate := seenFingerprints[fingerprint]; duplicate { + return nil, fmt.Errorf( + "amazon root CA bundle contains duplicate SHA-256 fingerprint %s", + fingerprint, + ) + } + seenFingerprints[fingerprint] = struct{}{} + pool.AddCert(certificate) + certificateCount++ + remaining = rest + } + + if certificateCount != amazonRootCACount { + return nil, fmt.Errorf( + "amazon root CA bundle contains %d certificates, expected %d", + certificateCount, + amazonRootCACount, + ) + } + return pool, nil +} + +func decodeStrictPEMBlock(data []byte) (*pem.Block, []byte, error) { + if !bytes.HasPrefix(data, []byte("-----BEGIN ")) { + return nil, nil, errors.New(invalidAmazonRootCAPEMMessage) + } + block, rest := pem.Decode(data) + if block == nil { + return nil, nil, errors.New(invalidAmazonRootCAPEMMessage) + } + beginMarker := []byte("-----BEGIN " + block.Type + "-----") + if !bytes.HasPrefix(data, beginMarker) || + !hasSupportedLineEndingPrefix(data[len(beginMarker):]) { + return nil, nil, errors.New(invalidAmazonRootCAPEMMessage) + } + consumed := data[:len(data)-len(rest)] + if !hasSupportedLineEndingSuffix(consumed) { + return nil, nil, errors.New(invalidAmazonRootCAPEMMessage) + } + if len(rest) > 0 && !bytes.HasPrefix(rest, []byte("-----BEGIN ")) { + return nil, nil, errors.New(invalidAmazonRootCAPEMMessage) + } + return block, rest, nil +} + +func hasSupportedLineEndingPrefix(data []byte) bool { + return bytes.HasPrefix(data, []byte("\n")) || + bytes.HasPrefix(data, []byte("\r\n")) +} + +func hasSupportedLineEndingSuffix(data []byte) bool { + if bytes.HasSuffix(data, []byte("\r\n")) { + return true + } + if !bytes.HasSuffix(data, []byte("\n")) { + return false + } + return len(data) < 2 || data[len(data)-2] != '\r' +} diff --git a/internal/enclave/provider/awskms/amazonroots_test.go b/internal/enclave/provider/awskms/amazonroots_test.go new file mode 100644 index 0000000..f49c2b0 --- /dev/null +++ b/internal/enclave/provider/awskms/amazonroots_test.go @@ -0,0 +1,342 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package awskms + +import ( + "bytes" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/hex" + "encoding/pem" + "math/big" + "os" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestParseAmazonRootCAPoolRejectsInvalidBundles(t *testing.T) { + firstBlock, remaining := pem.Decode(amazonRootCAPEM) + require.NotNil(t, firstBlock) + firstCertificate := pem.EncodeToMemory(firstBlock) + + foreignCertificate := newForeignKMSCertificate(t) + foreignRoot := newForeignRootCertificate(t) + tests := map[string]struct { + bundle []byte + message string + }{ + "leading data": { + bundle: append([]byte("unexpected data\n"), amazonRootCAPEM...), + message: "invalid PEM data", + }, + "data between certificates": { + bundle: bytes.Join( + [][]byte{firstCertificate, []byte("unexpected data\n"), remaining}, + nil, + ), + message: "invalid PEM data", + }, + "malformed block before certificates": { + bundle: append( + []byte("-----BEGIN BROKEN-----\nunexpected data\n-----END BROKEN-----\n"), + amazonRootCAPEM..., + ), + message: "invalid PEM data", + }, + "non-certificate block": { + bundle: pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: []byte("key")}), + message: `invalid PEM block "PRIVATE KEY"`, + }, + "PEM headers": { + bundle: pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE", + Headers: map[string]string{"Header": "value"}, + Bytes: firstBlock.Bytes, + }), + message: `invalid PEM block "CERTIFICATE"`, + }, + "invalid certificate": { + bundle: pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: []byte("certificate")}), + message: "parse Amazon root CA certificate", + }, + "certificate is not self-signed": { + bundle: pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE", + Bytes: foreignCertificate.Raw, + }), + message: "certificate is not self-signed", + }, + "wrong certificate count": { + bundle: firstCertificate, + message: "contains 1 certificates, expected 5", + }, + "unapproved self-signed root": { + bundle: append( + pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: foreignRoot.Raw}), + remaining..., + ), + message: "unapproved SHA-256 fingerprint", + }, + "leading space": { + bundle: append([]byte(" "), amazonRootCAPEM...), + message: "invalid PEM data", + }, + "leading unicode whitespace": { + bundle: append([]byte("\u00a0"), amazonRootCAPEM...), + message: "invalid PEM data", + }, + "space between certificates": { + bundle: bytes.Join([][]byte{firstCertificate, []byte(" "), remaining}, nil), + message: "invalid PEM data", + }, + "bare carriage return between certificates": { + bundle: bytes.Join([][]byte{firstCertificate, []byte("\r"), remaining}, nil), + message: "invalid PEM data", + }, + "blank line between certificates": { + bundle: bytes.Join([][]byte{firstCertificate, []byte("\n"), remaining}, nil), + message: "invalid PEM data", + }, + "missing final line ending": { + bundle: bytes.TrimSuffix(amazonRootCAPEM, []byte("\n")), + message: "invalid PEM data", + }, + "trailing space": { + bundle: append(bytes.Clone(amazonRootCAPEM), ' '), + message: "invalid PEM data", + }, + "trailing tab": { + bundle: append(bytes.Clone(amazonRootCAPEM), '\t'), + message: "invalid PEM data", + }, + "trailing form feed": { + bundle: append(bytes.Clone(amazonRootCAPEM), '\f'), + message: "invalid PEM data", + }, + "trailing unicode whitespace": { + bundle: append(bytes.Clone(amazonRootCAPEM), []byte("\u00a0")...), + message: "invalid PEM data", + }, + "trailing blank line": { + bundle: append(bytes.Clone(amazonRootCAPEM), '\n'), + message: "invalid PEM data", + }, + "trailing garbage": { + bundle: append(bytes.Clone(amazonRootCAPEM), []byte("garbage")...), + message: "invalid PEM data", + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + _, err := parseAmazonRootCAPool(tc.bundle) + require.ErrorContains(t, err, tc.message) + }) + } +} + +func TestParseAmazonRootCAPoolAcceptsSupportedLineEndings(t *testing.T) { + tests := map[string][]byte{ + "lf endings": amazonRootCAPEM, + "crlf endings": bytes.ReplaceAll( + amazonRootCAPEM, + []byte("\n"), + []byte("\r\n"), + ), + "mixed LF and CRLF separators": bytes.Replace( + amazonRootCAPEM, + []byte("\n-----BEGIN CERTIFICATE-----"), + []byte("\r\n-----BEGIN CERTIFICATE-----"), + 1, + ), + } + + for name, bundle := range tests { + t.Run(name, func(t *testing.T) { + pool, err := parseAmazonRootCAPool(bundle) + + require.NoError(t, err) + require.NotNil(t, pool) + }) + } +} + +func TestAmazonRootCAPoolVerifiesKMSCertificateChain(t *testing.T) { + chainPEM, err := os.ReadFile("testdata/kms-us-east-1-chain.pem") + require.NoError(t, err) + + var certificates []*x509.Certificate + remaining := chainPEM + for len(remaining) > 0 { + block, rest := pem.Decode(remaining) + require.NotNil(t, block) + require.Equal(t, "CERTIFICATE", block.Type) + certificate, err := x509.ParseCertificate(block.Bytes) + require.NoError(t, err) + certificates = append(certificates, certificate) + remaining = bytes.TrimSpace(rest) + } + require.Len(t, certificates, 2) + + roots, err := newAmazonRootCAPool() + require.NoError(t, err) + intermediates := x509.NewCertPool() + intermediates.AddCert(certificates[1]) + + _, err = certificates[0].Verify(x509.VerifyOptions{ + DNSName: "kms.us-east-1.amazonaws.com", + Roots: roots, + Intermediates: intermediates, + CurrentTime: time.Date(2026, time.September, 4, 0, 0, 0, 0, time.UTC), + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + }) + require.NoError(t, err) +} + +func TestAmazonRootCAPoolContainsApprovedRoots(t *testing.T) { + expectedFingerprints := []string{ + "8ecde6884f3d87b1125ba31ac3fcb13d7016de7f57cc904fe1cb97c6ae98196e", + "1ba5b2aa8c65401a82960118f80bec4f62304d83cec4713a19c39c011ea46db4", + "18ce6cfe7bf14e60b2e347b8dfe868cb31d02ebb3ada271569f50343b46db3a4", + "e35d28419ed02025cfa69038cd623962458da5c695fbdea3c22b0bfb25897092", + "568d6905a2c88708a4b3025190edcfedb1974a606a13c6e5290fcb2ae63edab5", + } + + var actualFingerprints []string + var approvedRoots []*x509.Certificate + remaining := amazonRootCAPEM + for len(remaining) > 0 { + block, rest := pem.Decode(remaining) + require.NotNil(t, block) + require.Equal(t, "CERTIFICATE", block.Type) + fingerprint := sha256.Sum256(block.Bytes) + actualFingerprints = append(actualFingerprints, hex.EncodeToString(fingerprint[:])) + certificate, err := x509.ParseCertificate(block.Bytes) + require.NoError(t, err) + approvedRoots = append(approvedRoots, certificate) + remaining = rest + } + require.ElementsMatch(t, expectedFingerprints, actualFingerprints) + + roots, err := newAmazonRootCAPool() + require.NoError(t, err) + for _, certificate := range approvedRoots { + _, err := certificate.Verify(x509.VerifyOptions{ + Roots: roots, + CurrentTime: certificate.NotBefore.Add(time.Hour), + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageAny}, + }) + require.NoError(t, err) + } +} + +func TestAmazonRootCAPoolRejectsCertificateFromAnotherRoot(t *testing.T) { + roots, err := newAmazonRootCAPool() + require.NoError(t, err) + + leaf := newForeignKMSCertificate(t) + _, err = leaf.Verify(x509.VerifyOptions{ + DNSName: "kms.us-east-1.amazonaws.com", + Roots: roots, + }) + require.Error(t, err) + var unknownAuthority x509.UnknownAuthorityError + require.ErrorAs(t, err, &unknownAuthority) +} + +func newForeignKMSCertificate(t *testing.T) *x509.Certificate { + t.Helper() + + tlsCertificate := newForeignKMSTLSCertificate(t) + leaf, err := x509.ParseCertificate(tlsCertificate.Certificate[0]) + require.NoError(t, err) + return leaf +} + +func newForeignKMSTLSCertificate(t *testing.T) tls.Certificate { + t.Helper() + + rootKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + now := time.Now() + root := createForeignRootCertificate(t, rootKey, now) + + leafKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + leafTemplate := &x509.Certificate{ + SerialNumber: big.NewInt(2), + Subject: pkix.Name{CommonName: "kms.us-east-1.amazonaws.com"}, + DNSNames: []string{"kms.us-east-1.amazonaws.com"}, + NotBefore: now.Add(-time.Hour), + NotAfter: now.Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + leafDER, err := x509.CreateCertificate( + rand.Reader, + leafTemplate, + root, + &leafKey.PublicKey, + rootKey, + ) + require.NoError(t, err) + return tls.Certificate{ + Certificate: [][]byte{leafDER}, + PrivateKey: leafKey, + } +} + +func newForeignRootCertificate(t *testing.T) *x509.Certificate { + t.Helper() + + rootKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + return createForeignRootCertificate(t, rootKey, time.Now()) +} + +func createForeignRootCertificate( + t *testing.T, + rootKey *ecdsa.PrivateKey, + now time.Time, +) *x509.Certificate { + t.Helper() + + rootTemplate := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "Foreign Root CA"}, + NotBefore: now.Add(-time.Hour), + NotAfter: now.Add(time.Hour), + KeyUsage: x509.KeyUsageCertSign, + BasicConstraintsValid: true, + IsCA: true, + } + rootDER, err := x509.CreateCertificate( + rand.Reader, + rootTemplate, + rootTemplate, + &rootKey.PublicKey, + rootKey, + ) + require.NoError(t, err) + root, err := x509.ParseCertificate(rootDER) + require.NoError(t, err) + return root +} diff --git a/internal/enclave/provider/awskms/awsconfig.go b/internal/enclave/provider/awskms/awsconfig.go new file mode 100644 index 0000000..2cd0569 --- /dev/null +++ b/internal/enclave/provider/awskms/awsconfig.go @@ -0,0 +1,172 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package awskms is the enclave-side AWS KMS provider. It owns the KMS +// Provider interface and implementation (New, Decrypt, GenerateDataKey), the +// aws.Config builder the enclave Initialize handler uses to construct an +// ephemeral client, the Factory seam, and the error-to-gRPC-status mapping. +package awskms + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "net" + "net/http" + "net/url" + + "github.com/aws/aws-sdk-go-v2/aws" + awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/circlefin/arc-remote-signer/internal/common/byteproxy" + "github.com/circlefin/arc-remote-signer/proto/pb" +) + +// BuildConfig constructs an aws.Config for use inside the enclave +// during Initialize. Credentials come from the host-supplied request. +// +// awsproxyEndpoint is where the SDK's TCP dial gets redirected to (the +// in-enclave awsproxy listener). +// +// localstackEnabled makes the SDK use awsproxyEndpoint as its base endpoint +// over plain HTTP. Standard AWS KMS keeps the natural AWS endpoint and uses +// end-to-end TLS. +// +// BuildConfig validates creds.Region before it adds the region to the config. +// The enclave-side initClients also validates each ARN region. It gives each +// KMS client a routed HTTP transport. +func BuildConfig(creds *pb.AwsCredentials, awsproxyEndpoint string, localstackEnabled bool) (aws.Config, error) { + if creds == nil { + return aws.Config{}, errors.New("nil credentials") + } + if err := validateCredentialsRegion(creds.Region); err != nil { + return aws.Config{}, err + } + if awsproxyEndpoint == "" { + return aws.Config{}, errors.New("empty awsproxy endpoint") + } + httpClient, err := newAwsproxyHTTPClient(awsproxyEndpoint, nil, "") + if err != nil { + return aws.Config{}, fmt.Errorf("build awsproxy http client: %w", err) + } + cfg := aws.Config{ + Region: creds.Region, + Credentials: credentials.NewStaticCredentialsProvider( + creds.AccessKeyId, + creds.SecretAccessKey, + creds.SessionToken, + ), + HTTPClient: httpClient, + } + if localstackEnabled { + cfg.BaseEndpoint = aws.String(awsproxyEndpoint) + } + return cfg, nil +} + +func validateCredentialsRegion(region string) error { + if err := validateKMSRegion(region); err != nil { + return fmt.Errorf("%w: credentials region: %w", ErrInvalidRegion, err) + } + return nil +} + +// newAwsproxyHTTPClient returns a client whose Transport redirects every TCP +// dial to awsproxy while leaving the SDK's endpoint view intact. A non-nil +// route is written before the SDK stream. BuildConfig leaves it nil for direct +// LocalStack use; production initClients replaces it with per-ARN routes. +// +// It builds on awshttp.BuildableClient rather than a bare *http.Client so +// the SDK's per-request timeout (awskms.WithTimeout) and DefaultsMode +// dial/TLS timeouts still attach β€” those only apply when HTTPClient is a +// *awshttp.BuildableClient; a plain *http.Client silently drops them. +func newAwsproxyHTTPClient( + awsproxyEndpoint string, + route *byteproxy.AWSRoute, + tlsServerName string, +) (*awshttp.BuildableClient, error) { + proxyAddr, err := awsproxyAddress(awsproxyEndpoint) + if err != nil { + return nil, err + } + rootCAs, err := amazonRootCAsForServer(tlsServerName) + if err != nil { + return nil, err + } + return awshttp.NewBuildableClient().WithTransportOptions(func(transport *http.Transport) { + configureKMSTLS(transport, tlsServerName, rootCAs) + transport.DialContext = awsproxyDialContext(proxyAddr, route) + }), nil +} + +func awsproxyAddress(endpoint string) (string, error) { + u, err := url.Parse(endpoint) + if err != nil { + return "", fmt.Errorf("parse awsproxy endpoint %q: %w", endpoint, err) + } + if u.Host == "" { + return "", fmt.Errorf("awsproxy endpoint %q has no host", endpoint) + } + // A non-empty Host does not guarantee a port; net.Dialer needs host:port, + // so reject a portless endpoint here rather than failing later with an + // opaque "missing port in address" at dial time. + if u.Port() == "" { + return "", fmt.Errorf("awsproxy endpoint %q has no port", endpoint) + } + return u.Host, nil +} + +func amazonRootCAsForServer(serverName string) (*x509.CertPool, error) { + if serverName == "" { + return nil, nil + } + rootCAs, err := newAmazonRootCAPool() + if err != nil { + return nil, fmt.Errorf("build Amazon root CA pool: %w", err) + } + return rootCAs, nil +} + +func configureKMSTLS(transport *http.Transport, serverName string, rootCAs *x509.CertPool) { + if serverName == "" { + return + } + if transport.TLSClientConfig == nil { + transport.TLSClientConfig = &tls.Config{MinVersion: tls.VersionTLS12} + } else { + transport.TLSClientConfig = transport.TLSClientConfig.Clone() + } + transport.TLSClientConfig.RootCAs = rootCAs + transport.TLSClientConfig.ServerName = serverName +} + +func awsproxyDialContext( + proxyAddr string, + route *byteproxy.AWSRoute, +) func(context.Context, string, string) (net.Conn, error) { + return func(ctx context.Context, network, _ string) (net.Conn, error) { + conn, err := (&net.Dialer{}).DialContext(ctx, network, proxyAddr) + if err != nil { + return nil, err + } + if route != nil { + if err := byteproxy.WriteAWSRoute(conn, *route); err != nil { + _ = conn.Close() + return nil, fmt.Errorf("write awsproxy route: %w", err) + } + } + return conn, nil + } +} diff --git a/internal/enclave/provider/awskms/awsconfig_test.go b/internal/enclave/provider/awskms/awsconfig_test.go new file mode 100644 index 0000000..b0d4a57 --- /dev/null +++ b/internal/enclave/provider/awskms/awsconfig_test.go @@ -0,0 +1,232 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package awskms + +import ( + "context" + "crypto/tls" + "crypto/x509" + "io" + "net" + "net/http" + "net/http/httptest" + "testing" + + "github.com/circlefin/arc-remote-signer/internal/common/byteproxy" + "github.com/circlefin/arc-remote-signer/proto/pb" + "github.com/stretchr/testify/require" +) + +func TestBuildConfig_HappyPath(t *testing.T) { + creds := &pb.AwsCredentials{ + AccessKeyId: "AKIA-test", + SecretAccessKey: "secret", + SessionToken: "session", + Region: "us-east-1", + } + cfg, err := BuildConfig(creds, "http://127.0.0.1:9000", false) + require.NoError(t, err) + require.Equal(t, "us-east-1", cfg.Region) + // BaseEndpoint stays nil so the SDK keeps the natural KMS endpoint; + // TLS SNI and SigV4 Host then use kms..amazonaws.com. + require.Nil(t, cfg.BaseEndpoint) + require.NotNil(t, cfg.Credentials) + require.NotNil(t, cfg.HTTPClient) + + got, err := cfg.Credentials.Retrieve(context.Background()) + require.NoError(t, err) + require.Equal(t, "AKIA-test", got.AccessKeyID) + require.Equal(t, "secret", got.SecretAccessKey) + require.Equal(t, "session", got.SessionToken) +} + +func TestBuildConfig_RejectsInvalidCredentialsRegion(t *testing.T) { + tests := []string{ + "us@east-1", + "s3", + } + + for _, region := range tests { + t.Run(region, func(t *testing.T) { + creds := &pb.AwsCredentials{ + AccessKeyId: "AKIA-test", + SecretAccessKey: "secret", + SessionToken: "session", + Region: region, + } + + cfg, err := BuildConfig(creds, "http://127.0.0.1:9000", false) + + require.ErrorIs(t, err, ErrInvalidRegion) + require.Empty(t, cfg.Region) + }) + } +} + +// TestBuildConfig_Localstack pins the LocalStack base endpoint. +func TestBuildConfig_Localstack(t *testing.T) { + creds := &pb.AwsCredentials{ + AccessKeyId: "AKIA-test", + SecretAccessKey: "secret", + SessionToken: "session", + Region: "us-east-1", + } + cfg, err := BuildConfig(creds, "http://127.0.0.1:9000", true) + require.NoError(t, err) + require.NotNil(t, cfg.BaseEndpoint) + require.Equal(t, "http://127.0.0.1:9000", *cfg.BaseEndpoint) +} + +// TestBuildConfig_HTTPClientRedirectsDial pins the DialContext rewrite: +// regardless of the address the SDK passes (kms..amazonaws.com:443 +// in practice), the transport must dial the awsproxy host instead. +func TestAwsproxyHTTPClientWritesRouteBeforePayload(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { _ = ln.Close() }) + + wantRoute := byteproxy.AWSRoute{Service: "kms", Region: "us-west-2"} + client, err := newAwsproxyHTTPClient("http://"+ln.Addr().String(), &wantRoute, "") + require.NoError(t, err) + + // BuildableClient so the SDK's per-request timeout still attaches; its + // transport carries the dial redirect. + transport := client.GetTransport() + + conn, err := transport.DialContext(context.Background(), "tcp", "kms.us-east-1.amazonaws.com:443") + require.NoError(t, err) + t.Cleanup(func() { _ = conn.Close() }) + + accepted, err := ln.Accept() + require.NoError(t, err) + t.Cleanup(func() { _ = accepted.Close() }) + gotRoute, err := byteproxy.ReadAWSRoute(accepted) + require.NoError(t, err) + require.Equal(t, wantRoute, gotRoute) + + payload := []byte("tls-client-hello") + _, err = conn.Write(payload) + require.NoError(t, err) + gotPayload := make([]byte, len(payload)) + _, err = io.ReadFull(accepted, gotPayload) + require.NoError(t, err) + require.Equal(t, payload, gotPayload) +} + +func TestAwsproxyHTTPClientPropagatesDialFailure(t *testing.T) { + client, err := newAwsproxyHTTPClient("http://127.0.0.1:0", nil, "") + require.NoError(t, err) + + conn, err := client.GetTransport().DialContext( + context.Background(), + "tcp", + "kms.us-east-1.amazonaws.com:443", + ) + require.Error(t, err) + require.Nil(t, conn) +} + +func TestAwsproxyHTTPClientUsesRestrictedRootsForKMS(t *testing.T) { + const serverName = "kms.us-east-1.amazonaws.com" + + client, err := newAwsproxyHTTPClient("http://127.0.0.1:9000", nil, serverName) + require.NoError(t, err) + + tlsConfig := client.GetTransport().TLSClientConfig + require.NotNil(t, tlsConfig) + require.Equal(t, serverName, tlsConfig.ServerName) + require.NotNil(t, tlsConfig.RootCAs) +} + +func TestAwsproxyHTTPClientRejectsTLSCertificateFromAnotherRoot(t *testing.T) { + server := httptest.NewUnstartedServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { + t.Error("request reached the server with an unapproved certificate") + })) + server.TLS = &tls.Config{ + MinVersion: tls.VersionTLS12, + Certificates: []tls.Certificate{newForeignKMSTLSCertificate(t)}, + } + server.StartTLS() + t.Cleanup(server.Close) + + client, err := newAwsproxyHTTPClient(server.URL, nil, "kms.us-east-1.amazonaws.com") + require.NoError(t, err) + client = client.WithTransportOptions(func(transport *http.Transport) { + transport.Proxy = nil + }) + request, err := http.NewRequestWithContext( + context.Background(), + http.MethodGet, + "https://kms.us-east-1.amazonaws.com", + nil, + ) + require.NoError(t, err) + + response, err := client.Do(request) + if response != nil { + t.Cleanup(func() { _ = response.Body.Close() }) + } + require.Error(t, err) + var unknownAuthority x509.UnknownAuthorityError + require.ErrorAs(t, err, &unknownAuthority) +} + +func TestBuildConfig_MalformedEndpoint(t *testing.T) { + creds := &pb.AwsCredentials{ + AccessKeyId: "AKIA-test", + SecretAccessKey: "secret", + SessionToken: "session", + Region: "us-east-1", + } + // no scheme, no host + cfg, err := BuildConfig(creds, "://bad", false) + require.Error(t, err) + require.Empty(t, cfg.Region) +} + +// TestBuildConfig_EndpointMissingPort pins the port guard: a host without an +// explicit port would otherwise dial-fail later with an opaque "missing port +// in address", so New must reject it up front. +func TestBuildConfig_EndpointMissingPort(t *testing.T) { + creds := &pb.AwsCredentials{ + AccessKeyId: "AKIA-test", + SecretAccessKey: "secret", + SessionToken: "session", + Region: "us-east-1", + } + cfg, err := BuildConfig(creds, "http://127.0.0.1", false) + require.Error(t, err) + require.Contains(t, err.Error(), "no port") + require.Empty(t, cfg.Region) +} + +func TestBuildConfig_NilCredentials(t *testing.T) { + cfg, err := BuildConfig(nil, "http://127.0.0.1:9000", false) + require.Error(t, err) + require.Contains(t, err.Error(), "nil credentials") + require.Empty(t, cfg.Region) +} + +func TestBuildConfig_EmptyEndpoint(t *testing.T) { + creds := &pb.AwsCredentials{ + AccessKeyId: "AKIA-test", + SecretAccessKey: "secret", + SessionToken: "session", + Region: "us-east-1", + } + cfg, err := BuildConfig(creds, "", false) + require.Error(t, err) + require.Contains(t, err.Error(), "empty awsproxy endpoint") + require.Empty(t, cfg.Region) +} diff --git a/internal/app/provider/awskms/awskms.go b/internal/enclave/provider/awskms/awskms.go similarity index 52% rename from internal/app/provider/awskms/awskms.go rename to internal/enclave/provider/awskms/awskms.go index 58e3c63..ba75954 100644 --- a/internal/app/provider/awskms/awskms.go +++ b/internal/enclave/provider/awskms/awskms.go @@ -13,7 +13,6 @@ //go:generate mockgen -source=awskms.go -destination=awskms_mock.go -package=awskms -// Package awskms implement crypto provider interface. package awskms import ( @@ -21,6 +20,7 @@ import ( "errors" "fmt" "net/http" + "net/url" "sync" "time" @@ -30,6 +30,7 @@ import ( awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" "github.com/aws/aws-sdk-go-v2/service/kms" "github.com/aws/aws-sdk-go-v2/service/kms/types" + "github.com/circlefin/arc-remote-signer/internal/common/byteproxy" "github.com/circlefin/arc-remote-signer/internal/common/logging" ) @@ -43,6 +44,19 @@ func getLogger() *logging.Logger { return _logger } +// ErrInvalidARN marks a configured KMS ARN that is syntactically parseable but +// not a usable KMS key ARN (wrong service or missing region). The enclave +// service maps this class of error to codes.InvalidArgument. +var ErrInvalidARN = errors.New("invalid kms key arn") + +// ErrInvalidRegion marks a host-supplied AWS region that fails validation. +// The enclave service maps this error to codes.InvalidArgument. +var ErrInvalidRegion = errors.New("invalid AWS region") + +// ErrAttestationDocumentRequired marks an attempt to construct a Nitro KMS +// provider without an enclave attestation document. +var ErrAttestationDocumentRequired = errors.New("attestation document is required in Nitro mode") + // Provider is the interface for the aws kms provider. type Provider interface { Decrypt(ctx context.Context, ciphertext []byte) (plaintext []byte, ciphertextForRecipient []byte, err error) @@ -62,22 +76,47 @@ type provider struct { mu sync.Mutex } -// New function Init the provider with specific config and return the instance. -func New(ctx context.Context, cfg *Config, awsCfg aws.Config, attestationDocument []byte) (Provider, error) { - clients, err := initClients(awsCfg, cfg.Arns, time.Duration(cfg.ConnectTimeout)*time.Millisecond) +// NewWithAttestation constructs a provider that asks KMS to encrypt data keys +// for the attested enclave. +func NewWithAttestation( + ctx context.Context, + cfg *Config, + awsCfg aws.Config, + attestationDocument []byte, +) (Provider, error) { + if len(attestationDocument) == 0 { + return nil, ErrAttestationDocumentRequired + } + + return newProvider(ctx, cfg, awsCfg, &types.RecipientInfo{ + AttestationDocument: attestationDocument, + KeyEncryptionAlgorithm: types.KeyEncryptionMechanismRsaesOaepSha256, + }) +} + +// NewForDevelopment constructs a provider for development environments where +// KMS returns plaintext data keys without enclave attestation. +func NewForDevelopment(ctx context.Context, cfg *Config, awsCfg aws.Config) (Provider, error) { + return newProvider(ctx, cfg, awsCfg, nil) +} + +func newProvider( + ctx context.Context, + cfg *Config, + awsCfg aws.Config, + recipient *types.RecipientInfo, +) (Provider, error) { + clients, err := initClients( + awsCfg, + cfg.Arns, + time.Duration(cfg.ConnectTimeout)*time.Millisecond, + cfg.AwsproxyEndpoint, + ) if err != nil { getLogger().WarnErr(ctx, "initClients failed", err, nil) return nil, err } - var recipient *types.RecipientInfo - if len(attestationDocument) > 0 { - recipient = &types.RecipientInfo{ - AttestationDocument: attestationDocument, - KeyEncryptionAlgorithm: types.KeyEncryptionMechanismRsaesOaepSha256, - } - } - provider := &provider{ clients: clients, recipient: recipient, @@ -103,13 +142,20 @@ func (p *provider) call(fn func(*client) error) error { copy(clients, p.clients) p.mu.Unlock() var err error + var authErr error for _, client := range clients { err = fn(client) if err == nil { return nil } + if authErr == nil { + authErr = credentialAuthError(err) + } p.moveClientToBack(client) } + if authErr != nil { + err = authErr + } return fmt.Errorf("all multi-region keys are invalid, error: %w", err) } @@ -163,10 +209,15 @@ func (p *provider) GenerateDataKey(ctx context.Context) (plainDataKey, cipherDat } // initClients will create a clients slice for each valid ARN and return if there is an invalid arn. -func initClients(cfg aws.Config, arns []string, timeout time.Duration) (clients []*client, err error) { - // precheck arns - if len(arns) == 0 { - return nil, errors.New("there is no arn") +func initClients( + cfg aws.Config, + arns []string, + timeout time.Duration, + awsproxyEndpoint string, +) (clients []*client, err error) { + regions, err := validateKMSKeyARNs(arns) + if err != nil { + return nil, err } var opts []func(*kms.Options) @@ -187,13 +238,32 @@ func initClients(cfg aws.Config, arns []string, timeout time.Duration) (clients } for i, arn := range arns { - region, err := extractRegionFromKmsKeyArn(arn) - if err != nil { - return nil, fmt.Errorf("invalid arn(arn_%v))", i+1) - } - + region := regions[i] c := cfg.Copy() c.Region = region + var tlsServerName string + if c.BaseEndpoint == nil { + endpoint, err := canonicalKMSEndpoint(region) + if err != nil { + return nil, fmt.Errorf("build KMS endpoint for arn_%d: %w", i+1, err) + } + c.BaseEndpoint = aws.String(endpoint.String()) + tlsServerName = endpoint.Hostname() + } + if awsproxyEndpoint != "" { + route := byteproxy.AWSRoute{ + Service: "kms", + Region: region, + } + httpClient, err := newAwsproxyHTTPClient(awsproxyEndpoint, &route, tlsServerName) + if err != nil { + return nil, fmt.Errorf("build awsproxy HTTP client for arn_%d: %w", i+1, err) + } + httpClient = httpClient.WithTransportOptions(func(tr *http.Transport) { + tr.DisableKeepAlives = true + }) + c.HTTPClient = httpClient + } clients = append(clients, &client{ Client: kms.NewFromConfig(c, opts...), arn: arn, @@ -202,15 +272,115 @@ func initClients(cfg aws.Config, arns []string, timeout time.Duration) (clients return clients, nil } +// ValidateRegions checks every host-supplied region used during enclave KMS +// initialization without constructing AWS endpoints, routes, or SDK clients. +func ValidateRegions(credentialsRegion string, arns []string) error { + if err := validateCredentialsRegion(credentialsRegion); err != nil { + return err + } + _, err := validateKMSKeyARNs(arns) + return err +} + +func validateKMSKeyARNs(arns []string) ([]string, error) { + if len(arns) == 0 { + return nil, errors.New("there is no arn") + } + + regions := make([]string, len(arns)) + for i, arn := range arns { + region, err := extractRegionFromKmsKeyArn(arn) + if err != nil { + return nil, fmt.Errorf("invalid arn_%d: %w: %w", i+1, ErrInvalidARN, err) + } + regions[i] = region + } + return regions, nil +} + func extractRegionFromKmsKeyArn(arnStr string) (string, error) { arnObj, err := arn.Parse(arnStr) if err != nil { getLogger().WarnErr(context.Background(), "arn.Parse failed", err, nil) return "", err } + if arnObj.Partition != "aws" { + return "", fmt.Errorf("ARN partition %q is not supported", arnObj.Partition) + } + if arnObj.Service != "kms" { + return "", fmt.Errorf("ARN service %q is not KMS", arnObj.Service) + } + if arnObj.Region == "" { + return "", errors.New("KMS ARN has no region") + } + if err := validateKMSRegion(arnObj.Region); err != nil { + return "", err + } return arnObj.Region, nil } +func validateKMSRegion(region string) error { + // This list is the AWS commercial partition subset of KMS endpoints at + // https://docs.aws.amazon.com/general/latest/gr/kms.html. Keep this switch and + // TestValidateKMSRegion_AcceptsCommercialKMSRegions synchronized when AWS adds + // a KMS region. GovCloud, China, and isolated partitions are not supported. + switch region { + case "af-south-1", + "ap-east-1", + "ap-east-2", + "ap-northeast-1", + "ap-northeast-2", + "ap-northeast-3", + "ap-south-1", + "ap-south-2", + "ap-southeast-1", + "ap-southeast-2", + "ap-southeast-3", + "ap-southeast-4", + "ap-southeast-5", + "ap-southeast-6", + "ap-southeast-7", + "ca-central-1", + "ca-west-1", + "eu-central-1", + "eu-central-2", + "eu-north-1", + "eu-south-1", + "eu-south-2", + "eu-west-1", + "eu-west-2", + "eu-west-3", + "il-central-1", + "me-central-1", + "me-south-1", + "mx-central-1", + "sa-east-1", + "us-east-1", + "us-east-2", + "us-west-1", + "us-west-2": + return nil + default: + return fmt.Errorf("KMS region %q is not supported", region) + } +} + +func canonicalKMSEndpoint(region string) (*url.URL, error) { + if err := validateKMSRegion(region); err != nil { + return nil, err + } + hostname := "kms." + region + ".amazonaws.com" + endpoint, err := url.Parse("https://" + hostname) + if err != nil { + return nil, fmt.Errorf("parse canonical KMS endpoint: %w", err) + } + if endpoint.Scheme != "https" || endpoint.User != nil || endpoint.Fragment != "" || + endpoint.Hostname() != hostname { + return nil, errors.New("canonical KMS endpoint has invalid URL components") + } + return endpoint, nil +} + // WithTimeout returns a function that sets a custom timeout for the KMS client. func WithTimeout(timeout time.Duration) func(*kms.Options) { return func(opts *kms.Options) { diff --git a/internal/app/provider/awskms/integration_test.go b/internal/enclave/provider/awskms/awskms_integration_test.go similarity index 77% rename from internal/app/provider/awskms/integration_test.go rename to internal/enclave/provider/awskms/awskms_integration_test.go index a4416be..035b027 100644 --- a/internal/app/provider/awskms/integration_test.go +++ b/internal/enclave/provider/awskms/awskms_integration_test.go @@ -17,7 +17,6 @@ package awskms import ( "context" - "fmt" "testing" "time" @@ -32,8 +31,20 @@ import ( const ( testAwsInvalidArn = "arn:aws:kms:us-east-1:000000000000:alias/invalid-arn" testAwsInvalidFormatArn = "arnawskms:us-east-1:000000000000:alias/invalid-arn" + testAWSRegion = "us-east-1" + testLocalStackEndpoint = "http://localhost:4566" ) +func newIntegrationConfig() *Config { + return &Config{ + Arns: []string{ + "arn:aws:kms:us-east-1:000000000000:alias/dev-multi-region-crypto", + "arn:aws:kms:us-west-2:000000000000:alias/dev-multi-region-crypto", + }, + ConnectTimeout: 1500, + } +} + type TestSuite struct { ctrl *gomock.Controller suite.Suite @@ -46,18 +57,18 @@ type TestSuite struct { func (suite *TestSuite) SetupTest() { suite.ctrl = gomock.NewController(suite.T()) // Get default config. - cfg := NewProviderConfig() + cfg := newIntegrationConfig() var err error // Static credentials are required for localstack; without them the SDK falls back to EC2 IMDS which is unavailable in CI. - awsCfg, err := awsSdkConfig.LoadDefaultConfig(context.Background(), awsSdkConfig.WithRegion(cfg.Localstack.Region), awsSdkConfig.WithBaseEndpoint(cfg.Localstack.Endpoint), awsSdkConfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider("test", "test", ""))) + awsCfg, err := awsSdkConfig.LoadDefaultConfig(context.Background(), awsSdkConfig.WithRegion(testAWSRegion), awsSdkConfig.WithBaseEndpoint(testLocalStackEndpoint), awsSdkConfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider("test", "test", ""))) suite.Require().NoError(err) // Init provider. - pvd, err := New(context.Background(), cfg, awsCfg, nil) + pvd, err := NewForDevelopment(context.Background(), cfg, awsCfg) suite.Require().NoError(err) suite.provider = pvd - clients, err := initClients(awsCfg, cfg.Arns, time.Duration(cfg.ConnectTimeout)*time.Millisecond) + clients, err := initClients(awsCfg, cfg.Arns, time.Duration(cfg.ConnectTimeout)*time.Millisecond, "") suite.Require().NoError(err) suite.providerImpl = provider{ @@ -74,13 +85,13 @@ func (suite *TestSuite) TearDownTest() { } func (suite *TestSuite) TestNew() { - awskmsCfg := NewProviderConfig() + awskmsCfg := newIntegrationConfig() // Static credentials are required for localstack; without them the SDK falls back to EC2 IMDS which is unavailable in CI. - awsCfg, err := awsSdkConfig.LoadDefaultConfig(context.Background(), awsSdkConfig.WithRegion(awskmsCfg.Localstack.Region), awsSdkConfig.WithBaseEndpoint(awskmsCfg.Localstack.Endpoint), awsSdkConfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider("test", "test", ""))) + awsCfg, err := awsSdkConfig.LoadDefaultConfig(context.Background(), awsSdkConfig.WithRegion(testAWSRegion), awsSdkConfig.WithBaseEndpoint(testLocalStackEndpoint), awsSdkConfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider("test", "test", ""))) suite.Require().NoError(err) suite.Run("success", func() { - _, err = New(context.Background(), awskmsCfg, awsCfg, nil) + _, err = NewForDevelopment(context.Background(), awskmsCfg, awsCfg) suite.Require().NoError(err) }) @@ -91,7 +102,7 @@ func (suite *TestSuite) TestNew() { }() awskmsCfg.Arns = []string{} - _, err = New(context.Background(), awskmsCfg, awsCfg, nil) + _, err = NewForDevelopment(context.Background(), awskmsCfg, awsCfg) suite.Require().Error(err) }) } @@ -146,11 +157,10 @@ type TestMultiRegionFailureOverSuite struct { func (suite *TestMultiRegionFailureOverSuite) SetupSuite() { // Get default config. - cfg := NewProviderConfig() + cfg := newIntegrationConfig() // Static credentials are required for localstack; without them the SDK falls back to EC2 IMDS which is unavailable in CI. - awsCfg, err := awsSdkConfig.LoadDefaultConfig(context.Background(), awsSdkConfig.WithRegion(cfg.Localstack.Region), awsSdkConfig.WithBaseEndpoint(cfg.Localstack.Endpoint), awsSdkConfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider("test", "test", ""))) + awsCfg, err := awsSdkConfig.LoadDefaultConfig(context.Background(), awsSdkConfig.WithRegion(testAWSRegion), awsSdkConfig.WithBaseEndpoint(testLocalStackEndpoint), awsSdkConfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider("test", "test", ""))) suite.Require().NoError(err) - cfg.ConnectTimeout = 1500 // Get testing KEYs. arns := cfg.Arns @@ -158,13 +168,13 @@ func (suite *TestMultiRegionFailureOverSuite) SetupSuite() { // Init provider1 with arn_1. cfg.Arns = arns[:1] - provider, err := New(context.Background(), cfg, awsCfg, nil) + provider, err := NewForDevelopment(context.Background(), cfg, awsCfg) suite.Require().NoError(err) suite.provider1 = provider // Init provider2 with arn_2. cfg.Arns = arns[1:] - provider, err = New(context.Background(), cfg, awsCfg, nil) + provider, err = NewForDevelopment(context.Background(), cfg, awsCfg) suite.Require().NoError(err) suite.provider2 = provider } @@ -182,9 +192,9 @@ func (suite *TestMultiRegionFailureOverSuite) TestEncryptWithArn1DecryptWithArn2 func (suite *TestMultiRegionFailureOverSuite) TestInvalidArnMultiRegionFailureOver() { // This verifies that when there is an issue with the key ARN, it can be switched to ARN2. // Get default config. - cfg := NewProviderConfig() + cfg := newIntegrationConfig() // Static credentials are required for localstack; without them the SDK falls back to EC2 IMDS which is unavailable in CI. - awsCfg, err := awsSdkConfig.LoadDefaultConfig(context.Background(), awsSdkConfig.WithRegion(cfg.Localstack.Region), awsSdkConfig.WithBaseEndpoint(cfg.Localstack.Endpoint), awsSdkConfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider("test", "test", ""))) + awsCfg, err := awsSdkConfig.LoadDefaultConfig(context.Background(), awsSdkConfig.WithRegion(testAWSRegion), awsSdkConfig.WithBaseEndpoint(testLocalStackEndpoint), awsSdkConfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider("test", "test", ""))) suite.Require().NoError(err) // Get testing KEYs. @@ -193,7 +203,7 @@ func (suite *TestMultiRegionFailureOverSuite) TestInvalidArnMultiRegionFailureOv suite.Run("withOneInvalidARNs", func() { cfg.Arns = append([]string{testAwsInvalidArn}, arns[:1]...) - provider, err := New(context.Background(), cfg, awsCfg, nil) + provider, err := NewForDevelopment(context.Background(), cfg, awsCfg) suite.Require().NoError(err) plainDataKey, cipherDataKey, _, err := provider.GenerateDataKey(context.Background()) @@ -206,7 +216,7 @@ func (suite *TestMultiRegionFailureOverSuite) TestInvalidArnMultiRegionFailureOv suite.Run("withTwoInvalidARNs", func() { cfg.Arns = append([]string{testAwsInvalidArn, testAwsInvalidArn}, arns[:1]...) - provider, err := New(context.Background(), cfg, awsCfg, nil) + provider, err := NewForDevelopment(context.Background(), cfg, awsCfg) suite.Require().NoError(err) plainDataKey, cipherDataKey, _, err := provider.GenerateDataKey(context.Background()) @@ -220,15 +230,14 @@ func (suite *TestMultiRegionFailureOverSuite) TestInvalidArnMultiRegionFailureOv func (suite *TestMultiRegionFailureOverSuite) TestWithAllInvalidArn() { // Get default config. - cfg := NewProviderConfig() - cfg.ConnectTimeout = 1500 + cfg := newIntegrationConfig() // Static credentials are required for localstack; without them the SDK falls back to EC2 IMDS which is unavailable in CI. - awsCfg, err := awsSdkConfig.LoadDefaultConfig(context.Background(), awsSdkConfig.WithRegion(cfg.Localstack.Region), awsSdkConfig.WithBaseEndpoint(cfg.Localstack.Endpoint), awsSdkConfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider("test", "test", ""))) + awsCfg, err := awsSdkConfig.LoadDefaultConfig(context.Background(), awsSdkConfig.WithRegion(testAWSRegion), awsSdkConfig.WithBaseEndpoint(testLocalStackEndpoint), awsSdkConfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider("test", "test", ""))) suite.Require().NoError(err) // Init provider with two invalid arn. cfg.Arns = []string{testAwsInvalidArn, testAwsInvalidArn} - provider, err := New(context.Background(), cfg, awsCfg, nil) + provider, err := NewForDevelopment(context.Background(), cfg, awsCfg) suite.Require().NoError(err) suite.Run("Encrypt", func() { @@ -249,15 +258,14 @@ func (suite *TestMultiRegionFailureOverSuite) TestWithAllInvalidArn() { func (suite *TestMultiRegionFailureOverSuite) TestNewProviderWithInvalidFormatArn() { // This verifies that when creating a new provider, an error will be returned if an ARN that does not comply with the format is provided. - cfg := NewProviderConfig() - cfg.ConnectTimeout = 1500 + cfg := newIntegrationConfig() // Static credentials are required to prevent the SDK from attempting EC2 IMDS credential lookup in CI. - awsCfg, err := awsSdkConfig.LoadDefaultConfig(context.Background(), awsSdkConfig.WithRegion(cfg.Localstack.Region), awsSdkConfig.WithBaseEndpoint(cfg.Localstack.Endpoint), awsSdkConfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider("test", "test", ""))) + awsCfg, err := awsSdkConfig.LoadDefaultConfig(context.Background(), awsSdkConfig.WithRegion(testAWSRegion), awsSdkConfig.WithBaseEndpoint(testLocalStackEndpoint), awsSdkConfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider("test", "test", ""))) suite.Require().NoError(err) cfg.Arns = []string{testAwsInvalidFormatArn} - _, err = New(context.Background(), cfg, awsCfg, nil) - suite.Require().EqualError(err, fmt.Sprintf("invalid arn(arn_%v))", 1)) + _, err = NewForDevelopment(context.Background(), cfg, awsCfg) + suite.Require().ErrorIs(err, ErrInvalidARN) } func TestTestMultiRegionFailureOverSuite(t *testing.T) { diff --git a/internal/app/provider/awskms/awskms_mock.go b/internal/enclave/provider/awskms/awskms_mock.go similarity index 100% rename from internal/app/provider/awskms/awskms_mock.go rename to internal/enclave/provider/awskms/awskms_mock.go diff --git a/internal/enclave/provider/awskms/awskms_test.go b/internal/enclave/provider/awskms/awskms_test.go new file mode 100644 index 0000000..6ef6ffa --- /dev/null +++ b/internal/enclave/provider/awskms/awskms_test.go @@ -0,0 +1,716 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package awskms + +import ( + "context" + "errors" + "io" + "net" + "net/http" + "net/url" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" + "github.com/aws/aws-sdk-go-v2/service/kms" + "github.com/aws/aws-sdk-go-v2/service/kms/types" + "github.com/aws/smithy-go" + "github.com/circlefin/arc-remote-signer/internal/common/byteproxy" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type kmsHTTPClientFunc func(*http.Request) (*http.Response, error) + +func (f kmsHTTPClientFunc) Do(req *http.Request) (*http.Response, error) { + return f(req) +} + +func newGenerateDataKeyTestClient( + region string, + do kmsHTTPClientFunc, +) *client { + arn := "arn:aws:kms:" + region + ":123456789012:key/test" + sdkClient := kms.New(kms.Options{ + Credentials: aws.AnonymousCredentials{}, + HTTPClient: do, + Region: region, + Retryer: aws.NopRetryer{}, + }) + return &client{Client: sdkClient, arn: arn} +} + +func generateDataKeyResponse(req *http.Request) *http.Response { + return &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{ + "Content-Type": []string{"application/x-amz-json-1.1"}, + }, + Body: io.NopCloser(strings.NewReader( + `{"Plaintext":"cGxhaW50ZXh0","CiphertextBlob":"Y2lwaGVydGV4dA=="}`, + )), + Request: req, + } +} + +func newTestClient(t *testing.T, arn string) *client { + t.Helper() + return &client{arn: arn} +} + +func newConstructorTestConfig() *Config { + return &Config{ + Arns: []string{"arn:aws:kms:us-east-1:123456789012:key/test"}, + } +} + +func TestNewWithAttestation_RequiresAttestationDocument(t *testing.T) { + tests := []struct { + name string + attestationDocument []byte + }{ + { + name: "nil document", + attestationDocument: nil, + }, + { + name: "empty document", + attestationDocument: []byte{}, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + gotProvider, err := NewWithAttestation( + context.Background(), + newConstructorTestConfig(), + aws.Config{}, + tt.attestationDocument, + ) + + require.ErrorIs(t, err, ErrAttestationDocumentRequired) + require.Nil(t, gotProvider) + }) + } +} + +func TestNewWithAttestation_AttachesRecipientInfo(t *testing.T) { + wantDocument := []byte("attestation-document") + + gotProvider, err := NewWithAttestation( + context.Background(), + newConstructorTestConfig(), + aws.Config{}, + wantDocument, + ) + + require.NoError(t, err) + got, ok := gotProvider.(*provider) + require.True(t, ok) + require.NotNil(t, got.recipient) + require.Equal(t, wantDocument, got.recipient.AttestationDocument) + require.Equal(t, types.KeyEncryptionMechanismRsaesOaepSha256, got.recipient.KeyEncryptionAlgorithm) +} + +func TestNewForDevelopment_OmitsRecipientInfo(t *testing.T) { + gotProvider, err := NewForDevelopment( + context.Background(), + newConstructorTestConfig(), + aws.Config{}, + ) + + require.NoError(t, err) + got, ok := gotProvider.(*provider) + require.True(t, ok) + require.Nil(t, got.recipient) +} + +func TestCall_FirstClientSuccess(t *testing.T) { + first := newTestClient(t, "arn:aws:kms:us-east-1:123456789012:key/first") + p := &provider{ + clients: []*client{first}, + } + + err := p.call(func(gotClient *client) error { + require.Same(t, first, gotClient) + return nil + }) + + require.NoError(t, err) + require.Len(t, p.clients, 1) + require.Same(t, first, p.clients[0]) +} + +func TestCall_PartialFailureFallback(t *testing.T) { + first := newTestClient(t, "arn:aws:kms:us-east-1:123456789012:key/first") + second := newTestClient(t, "arn:aws:kms:us-west-2:123456789012:key/second") + p := &provider{ + clients: []*client{first, second}, + } + + firstErr := errors.New("first client failed") + var callOrder []*client + err := p.call(func(gotClient *client) error { + callOrder = append(callOrder, gotClient) + if gotClient == first { + return firstErr + } + require.Same(t, second, gotClient) + return nil + }) + + require.NoError(t, err) + require.Equal(t, []*client{first, second}, callOrder) + require.Len(t, p.clients, 2) + require.Same(t, second, p.clients[0]) + require.Same(t, first, p.clients[1]) +} + +func TestCall_AllClientsFail(t *testing.T) { + first := newTestClient(t, "arn:aws:kms:us-east-1:123456789012:key/first") + second := newTestClient(t, "arn:aws:kms:us-west-2:123456789012:key/second") + p := &provider{ + clients: []*client{first, second}, + } + + firstErr := errors.New("first client failed") + lastErr := errors.New("last client failed") + err := p.call(func(gotClient *client) error { + if gotClient == first { + return firstErr + } + require.Same(t, second, gotClient) + return lastErr + }) + + require.Error(t, err) + require.ErrorContains(t, err, "all multi-region keys are invalid") + require.ErrorIs(t, err, lastErr) +} + +func expiredTokenError() error { + return &smithy.GenericAPIError{Code: "ExpiredTokenException", Message: "security token expired"} +} + +func transportFailureError() error { + return &url.Error{ + Op: "Post", + URL: "https://kms.us-west-2.amazonaws.com", + Err: &net.OpError{Op: "dial", Err: errors.New("connection refused")}, + } +} + +func requireExpiredTokenUnauthenticated(t *testing.T, err error) { + t.Helper() + var apiErr smithy.APIError + require.ErrorAs(t, err, &apiErr) + require.Equal(t, "ExpiredTokenException", apiErr.ErrorCode()) + mapped := StatusFromError(err) + st, ok := status.FromError(mapped) + require.True(t, ok) + require.Equal(t, codes.Unauthenticated, st.Code()) +} + +func TestCall_PreservesExpiredTokenOverLaterTransportFailure(t *testing.T) { + first := newTestClient(t, "arn:aws:kms:us-east-1:123456789012:key/first") + second := newTestClient(t, "arn:aws:kms:us-west-2:123456789012:key/second") + p := &provider{ + clients: []*client{first, second}, + } + + firstErr := expiredTokenError() + var callOrder []*client + err := p.call(func(gotClient *client) error { + callOrder = append(callOrder, gotClient) + if gotClient == first { + return firstErr + } + require.Same(t, second, gotClient) + return transportFailureError() + }) + + require.Equal(t, []*client{first, second}, callOrder) + require.ErrorContains(t, err, "all multi-region keys are invalid") + requireExpiredTokenUnauthenticated(t, err) +} + +func TestGenerateDataKey_FailsOverAfterFirstRegionTimeoutOrThrottle(t *testing.T) { + tests := map[string]error{ + "timeout": context.DeadlineExceeded, + "throttling": &smithy.GenericAPIError{Code: "ThrottlingException", Message: "rate exceeded"}, + } + + for name, firstRegionErr := range tests { + t.Run(name, func(t *testing.T) { + var callOrder []string + first := newGenerateDataKeyTestClient( + "us-east-1", + func(*http.Request) (*http.Response, error) { + callOrder = append(callOrder, "us-east-1") + return nil, firstRegionErr + }, + ) + second := newGenerateDataKeyTestClient( + "us-west-2", + func(req *http.Request) (*http.Response, error) { + callOrder = append(callOrder, "us-west-2") + return generateDataKeyResponse(req), nil + }, + ) + p := &provider{clients: []*client{first, second}} + + plaintext, ciphertext, _, err := p.GenerateDataKey(context.Background()) + + require.NoError(t, err) + require.Equal(t, []byte("plaintext"), plaintext) + require.Equal(t, []byte("ciphertext"), ciphertext) + require.Equal(t, []string{"us-east-1", "us-west-2"}, callOrder) + }) + } +} + +func TestGenerateDataKey_AllRegionsTimeout(t *testing.T) { + var callOrder []string + timeoutClient := func(region string) *client { + return newGenerateDataKeyTestClient( + region, + func(*http.Request) (*http.Response, error) { + callOrder = append(callOrder, region) + return nil, context.DeadlineExceeded + }, + ) + } + p := &provider{clients: []*client{ + timeoutClient("us-east-1"), + timeoutClient("us-west-2"), + }} + + _, _, _, err := p.GenerateDataKey(context.Background()) + + require.ErrorIs(t, err, context.DeadlineExceeded) + require.Equal(t, []string{"us-east-1", "us-west-2"}, callOrder) +} + +func TestGenerateDataKey_PreservesExpiredTokenOverLaterTransportFailure(t *testing.T) { + var callOrder []string + first := newGenerateDataKeyTestClient( + "us-east-1", + func(*http.Request) (*http.Response, error) { + callOrder = append(callOrder, "us-east-1") + return nil, expiredTokenError() + }, + ) + second := newGenerateDataKeyTestClient( + "us-west-2", + func(*http.Request) (*http.Response, error) { + callOrder = append(callOrder, "us-west-2") + return nil, transportFailureError() + }, + ) + p := &provider{clients: []*client{first, second}} + + _, _, _, err := p.GenerateDataKey(context.Background()) + + require.Equal(t, []string{"us-east-1", "us-west-2"}, callOrder) + requireExpiredTokenUnauthenticated(t, err) +} + +func TestGenerateDataKey_PrefersExpiredTokenOverEarlierTimeout(t *testing.T) { + var callOrder []string + first := newGenerateDataKeyTestClient( + "us-east-1", + func(*http.Request) (*http.Response, error) { + callOrder = append(callOrder, "us-east-1") + return nil, context.DeadlineExceeded + }, + ) + second := newGenerateDataKeyTestClient( + "us-west-2", + func(*http.Request) (*http.Response, error) { + callOrder = append(callOrder, "us-west-2") + return nil, expiredTokenError() + }, + ) + p := &provider{clients: []*client{first, second}} + + _, _, _, err := p.GenerateDataKey(context.Background()) + + require.Equal(t, []string{"us-east-1", "us-west-2"}, callOrder) + requireExpiredTokenUnauthenticated(t, err) +} + +func TestDecrypt_EmptyCiphertext(t *testing.T) { + tests := []struct { + name string + ciphertext []byte + }{ + { + name: "nil ciphertext", + ciphertext: nil, + }, + { + name: "empty ciphertext", + ciphertext: []byte{}, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + p := &provider{ + clients: []*client{ + newTestClient(t, "arn:aws:kms:us-east-1:123456789012:key/test"), + }, + } + + var ( + plaintext []byte + ciphertextForRecipient []byte + err error + ) + require.NotPanics(t, func() { + plaintext, ciphertextForRecipient, err = p.Decrypt(context.Background(), tt.ciphertext) + }) + + require.EqualError(t, err, "invalid ciphertext") + require.Nil(t, plaintext) + require.Nil(t, ciphertextForRecipient) + }) + } +} + +func TestExtractRegionFromKmsKeyArn_ValidArn(t *testing.T) { + region, err := extractRegionFromKmsKeyArn("arn:aws:kms:us-east-1:123456789012:key/1234abcd") + + require.NoError(t, err) + require.Equal(t, "us-east-1", region) +} + +func TestExtractRegionFromKmsKeyArn_MalformedArn(t *testing.T) { + region, err := extractRegionFromKmsKeyArn("not-an-arn") + + require.Error(t, err) + require.Empty(t, region) +} + +func TestExtractRegionFromKmsKeyArn_RejectsNonKMSArn(t *testing.T) { + region, err := extractRegionFromKmsKeyArn("arn:aws:s3:::my-bucket") + + require.Error(t, err) + require.Empty(t, region) +} + +func TestExtractRegionFromKmsKeyArn_RejectsNonAWSPartition(t *testing.T) { + region, err := extractRegionFromKmsKeyArn( + "arn:aws-cn:kms:cn-north-1:123456789012:key/1234abcd", + ) + + require.Error(t, err) + require.Empty(t, region) +} + +func TestExtractRegionFromKmsKeyArn_RejectsAuthorityInjection(t *testing.T) { + region, err := extractRegionFromKmsKeyArn( + "arn:aws:kms:@attackerdomain.com#:123456789012:key/1234abcd", + ) + + require.Error(t, err) + require.Empty(t, region) +} + +func TestExtractRegionFromKmsKeyArn_RejectsUnsupportedRegion(t *testing.T) { + region, err := extractRegionFromKmsKeyArn( + "arn:aws:kms:s3:123456789012:key/1234abcd", + ) + + require.Error(t, err) + require.Empty(t, region) +} + +func TestValidateKMSRegion_RejectsInvalidSyntax(t *testing.T) { + tests := map[string]string{ + "empty": "", + "at sign": "us@east-1", + "fragment": "us#east-1", + "slash": "us/east-1", + "query": "us?east-1", + "percent": "us%east-1", + "colon": "us:east-1", + "period": "us.east-1", + "space": "us east-1", + "control": "us\neast-1", + "uppercase": "US-east-1", + "non-ASCII": "us-Γ©ast-1", + "leading hyphen": "-us-east-1", + "trailing hyphen": "us-east-1-", + "longer than label": strings.Repeat("a", 64), + } + + for name, region := range tests { + t.Run(name, func(t *testing.T) { + err := validateKMSRegion(region) + require.Error(t, err) + }) + } +} + +func TestValidateKMSRegion_AcceptsCommercialKMSRegions(t *testing.T) { + tests := []string{ + "af-south-1", + "ap-east-1", + "ap-east-2", + "ap-northeast-1", + "ap-northeast-2", + "ap-northeast-3", + "ap-south-1", + "ap-south-2", + "ap-southeast-1", + "ap-southeast-2", + "ap-southeast-3", + "ap-southeast-4", + "ap-southeast-5", + "ap-southeast-6", + "ap-southeast-7", + "ca-central-1", + "ca-west-1", + "eu-central-1", + "eu-central-2", + "eu-north-1", + "eu-south-1", + "eu-south-2", + "eu-west-1", + "eu-west-2", + "eu-west-3", + "il-central-1", + "me-central-1", + "me-south-1", + "mx-central-1", + "sa-east-1", + "us-east-1", + "us-east-2", + "us-west-1", + "us-west-2", + } + + for _, region := range tests { + t.Run(region, func(t *testing.T) { + require.NoError(t, validateKMSRegion(region)) + }) + } +} + +func TestValidateKMSRegion_RejectsUnsupportedRegions(t *testing.T) { + tests := []string{ + "s3", + "a", + strings.Repeat("a", 63), + "us-mars-1", + "us-gov-west-1", + "cn-north-1", + } + + for _, region := range tests { + t.Run(region, func(t *testing.T) { + require.Error(t, validateKMSRegion(region)) + }) + } +} + +func TestValidateRegions(t *testing.T) { + tests := []struct { + name string + credentialsRegion string + arns []string + wantErr error + }{ + { + name: "valid credentials and ARN regions", + credentialsRegion: "us-east-1", + arns: []string{"arn:aws:kms:us-west-2:123456789012:key/test"}, + }, + { + name: "invalid credentials region", + credentialsRegion: "s3", + arns: []string{"arn:aws:kms:us-west-2:123456789012:key/test"}, + wantErr: ErrInvalidRegion, + }, + { + name: "invalid ARN region", + credentialsRegion: "us-east-1", + arns: []string{"arn:aws:kms:s3:123456789012:key/test"}, + wantErr: ErrInvalidARN, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := ValidateRegions(tt.credentialsRegion, tt.arns) + if tt.wantErr == nil { + require.NoError(t, err) + return + } + require.ErrorIs(t, err, tt.wantErr) + }) + } +} + +func TestCanonicalKMSEndpoint_ReturnsExpectedURL(t *testing.T) { + endpoint, err := canonicalKMSEndpoint("us-west-2") + + require.NoError(t, err) + require.Equal(t, "https", endpoint.Scheme) + require.Nil(t, endpoint.User) + require.Empty(t, endpoint.Fragment) + require.Equal(t, "kms.us-west-2.amazonaws.com", endpoint.Hostname()) + require.Equal(t, "https://kms.us-west-2.amazonaws.com", endpoint.String()) +} + +func TestInitClients_EmptyArns(t *testing.T) { + clients, err := initClients(aws.Config{}, nil, 0, "") + + require.Error(t, err) + require.Nil(t, clients) +} + +func TestInitClients_InvalidArn(t *testing.T) { + clients, err := initClients(aws.Config{}, []string{"bad-arn"}, 0, "") + + require.Error(t, err) + require.ErrorContains(t, err, "invalid arn") + require.Nil(t, clients) +} + +func TestInitClients_RejectsUnsupportedRegion(t *testing.T) { + clients, err := initClients( + aws.Config{}, + []string{"arn:aws:kms:s3:123456789012:key/1234abcd"}, + 0, + "", + ) + + require.Error(t, err) + require.ErrorContains(t, err, "invalid arn_1") + require.Nil(t, clients) +} + +func TestInitClients_ValidatesAllARNsBeforeConstructingClients(t *testing.T) { + clients, err := initClients( + aws.Config{}, + []string{ + "arn:aws:kms:us-east-1:123456789012:key/first", + "arn:aws:kms:s3:123456789012:key/second", + }, + 0, + "://invalid-proxy-endpoint", + ) + + require.ErrorIs(t, err, ErrInvalidARN) + require.ErrorContains(t, err, "invalid arn_2") + require.Nil(t, clients) +} + +func TestInitClients_StandardAWSPinsCanonicalEndpoint(t *testing.T) { + clients, err := initClients( + aws.Config{}, + []string{"arn:aws:kms:us-west-2:123456789012:key/1234abcd"}, + time.Second, + "http://127.0.0.1:9000", + ) + + require.NoError(t, err) + require.Len(t, clients, 1) + options := clients[0].Options() + require.NotNil(t, options.BaseEndpoint) + require.Equal(t, "https://kms.us-west-2.amazonaws.com", *options.BaseEndpoint) + httpClient, ok := options.HTTPClient.(*awshttp.BuildableClient) + require.True(t, ok) + transport := httpClient.GetTransport() + require.NotNil(t, transport.TLSClientConfig) + require.Equal(t, "kms.us-west-2.amazonaws.com", transport.TLSClientConfig.ServerName) + require.NotNil(t, transport.TLSClientConfig.RootCAs) +} + +func TestInitClients_DisablesKeepAlivesForAttemptScopedClients(t *testing.T) { + clients, err := initClients( + aws.Config{}, + []string{"arn:aws:kms:us-west-2:123456789012:key/1234abcd"}, + time.Second, + "http://127.0.0.1:9000", + ) + + require.NoError(t, err) + require.Len(t, clients, 1) + options := clients[0].Options() + httpClient, ok := options.HTTPClient.(*awshttp.BuildableClient) + require.True(t, ok) + require.True(t, httpClient.GetTransport().DisableKeepAlives) +} + +func TestInitClients_LocalStackPreservesEndpointWithoutTLSPinning(t *testing.T) { + localstackEndpoint := "http://127.0.0.1:4566" + clients, err := initClients( + aws.Config{BaseEndpoint: aws.String(localstackEndpoint)}, + []string{"arn:aws:kms:us-west-2:123456789012:key/1234abcd"}, + 0, + "http://127.0.0.1:9000", + ) + + require.NoError(t, err) + require.Len(t, clients, 1) + options := clients[0].Options() + require.NotNil(t, options.BaseEndpoint) + require.Equal(t, localstackEndpoint, *options.BaseEndpoint) + httpClient, ok := options.HTTPClient.(*awshttp.BuildableClient) + require.True(t, ok) + transport := httpClient.GetTransport() + if transport.TLSClientConfig != nil { + require.Empty(t, transport.TLSClientConfig.ServerName) + } +} + +func TestInitClientsRoutesEachARNRegion(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { _ = ln.Close() }) + + arns := []string{ + "arn:aws:kms:us-east-1:123456789012:key/first", + "arn:aws:kms:us-west-2:123456789012:key/second", + } + clients, err := initClients(aws.Config{}, arns, 0, "http://"+ln.Addr().String()) + require.NoError(t, err) + require.Len(t, clients, 2) + + for i, wantRegion := range []string{"us-east-1", "us-west-2"} { + httpClient, ok := clients[i].Options().HTTPClient.(*awshttp.BuildableClient) + require.True(t, ok) + conn, dialErr := httpClient.GetTransport().DialContext( + context.Background(), "tcp", "kms."+wantRegion+".amazonaws.com:443", + ) + require.NoError(t, dialErr) + t.Cleanup(func() { _ = conn.Close() }) + + accepted, acceptErr := ln.Accept() + require.NoError(t, acceptErr) + route, routeErr := byteproxy.ReadAWSRoute(accepted) + require.NoError(t, routeErr) + require.NoError(t, accepted.Close()) + require.Equal(t, byteproxy.AWSRoute{Service: "kms", Region: wantRegion}, route) + } +} diff --git a/internal/enclave/provider/awskms/certs/README.md b/internal/enclave/provider/awskms/certs/README.md new file mode 100644 index 0000000..9782e05 --- /dev/null +++ b/internal/enclave/provider/awskms/certs/README.md @@ -0,0 +1,76 @@ +# Amazon Trust Roots + +The enclave uses `amazon-trust-roots.pem` as the exclusive root pool for TLS +connections to AWS KMS in production. The bundle contains these self-signed +certificates: + +- Amazon Root CA 1 +- Amazon Root CA 2 +- Amazon Root CA 3 +- Amazon Root CA 4 +- Starfield Services Root Certificate Authority - G2 + +The certificates come from the +[Amazon Trust Services repository](https://www.amazontrust.com/repository/). +The direct source files are: + +- `https://www.amazontrust.com/repository/AmazonRootCA1.pem` +- `https://www.amazontrust.com/repository/AmazonRootCA2.pem` +- `https://www.amazontrust.com/repository/AmazonRootCA3.pem` +- `https://www.amazontrust.com/repository/AmazonRootCA4.pem` +- `https://www.amazontrust.com/repository/SFSRootCAG2.pem` + +Amazon publishes these certificate files under the +[Creative Commons Attribution-NoDerivatives 4.0 International License](https://creativecommons.org/licenses/by-nd/4.0/). +The bundle concatenates the source files without modifying the certificates. + +## Certificate verification + +Before you update the bundle, verify the SHA-256 hash of each DER certificate: + +```sh +openssl x509 -in AmazonRootCA1.pem -outform DER | sha256sum +``` + +`approvedAmazonRootCAFingerprints` contains the approved hashes from the Amazon +repository. The runtime parser rejects a certificate if it is added, removed, +duplicated, or replaced. + +## KMS chain fixture + +The test fixture at `../testdata/kms-us-east-1-chain.pem` contains the leaf +certificate and the intermediate certificate. AWS KMS served the certificate +chain on September 4, 2026. The chain test uses a time when the leaf certificate +is valid. The selected time keeps the test result the same after the leaf +certificate expires. The fixture is not part of the trusted root pool. + +## Root rotation + +1. Review the Amazon Trust Services repository for announced root changes. +2. Download each new certificate from the official HTTPS location. +3. Verify the DER hash against the value that Amazon publishes. +4. Before you remove a retiring root, add each new root. +5. Update `amazonRootCACount` for the temporary number of trusted roots. +6. Add each new fingerprint to `approvedAmazonRootCAFingerprints` and + `expectedFingerprints`. +7. Run the provider tests for AWS KMS and all repository tests. +8. Build the new enclave image. +9. Record the PCR measurements for the new enclave image. +10. Add the new PCR measurements to each applicable KMS policy. +11. Keep the previous PCR measurements in each policy. +12. Deploy the new enclave image. +13. Verify that the new enclave can use each configured KMS key. +14. Deployment gate: before AWS's announced activation or start-serving date + for the new root, confirm that every replica runs this EIF, its PCR + measurement is authorized by the applicable KMS policies, and KMS access + has been verified. If this gate cannot be completed by that date, treat + production readiness as blocked and escalate before AWS begins serving the + new chain. +15. After all replicas use the new image, remove the previous PCR measurements. +16. Wait until AWS completes the root migration. +17. Remove the retiring root from the bundle. +18. Update `amazonRootCACount`, `approvedAmazonRootCAFingerprints`, and + `expectedFingerprints`. +19. Repeat steps 7 through 15 for the reduced root bundle. + +Do not add leaf certificates or intermediate certificates to this bundle. diff --git a/internal/enclave/provider/awskms/certs/amazon-trust-roots.pem b/internal/enclave/provider/awskms/certs/amazon-trust-roots.pem new file mode 100644 index 0000000..83a6e40 --- /dev/null +++ b/internal/enclave/provider/awskms/certs/amazon-trust-roots.pem @@ -0,0 +1,100 @@ +-----BEGIN CERTIFICATE----- +MIIDQTCCAimgAwIBAgITBmyfz5m/jAo54vB4ikPmljZbyjANBgkqhkiG9w0BAQsF +ADA5MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRkwFwYDVQQDExBBbWF6 +b24gUm9vdCBDQSAxMB4XDTE1MDUyNjAwMDAwMFoXDTM4MDExNzAwMDAwMFowOTEL +MAkGA1UEBhMCVVMxDzANBgNVBAoTBkFtYXpvbjEZMBcGA1UEAxMQQW1hem9uIFJv +b3QgQ0EgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALJ4gHHKeNXj +ca9HgFB0fW7Y14h29Jlo91ghYPl0hAEvrAIthtOgQ3pOsqTQNroBvo3bSMgHFzZM +9O6II8c+6zf1tRn4SWiw3te5djgdYZ6k/oI2peVKVuRF4fn9tBb6dNqcmzU5L/qw +IFAGbHrQgLKm+a/sRxmPUDgH3KKHOVj4utWp+UhnMJbulHheb4mjUcAwhmahRWa6 +VOujw5H5SNz/0egwLX0tdHA114gk957EWW67c4cX8jJGKLhD+rcdqsq08p8kDi1L +93FcXmn/6pUCyziKrlA4b9v7LWIbxcceVOF34GfID5yHI9Y/QCB/IIDEgEw+OyQm +jgSubJrIqg0CAwEAAaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMC +AYYwHQYDVR0OBBYEFIQYzIU07LwMlJQuCFmcx7IQTgoIMA0GCSqGSIb3DQEBCwUA +A4IBAQCY8jdaQZChGsV2USggNiMOruYou6r4lK5IpDB/G/wkjUu0yKGX9rbxenDI +U5PMCCjjmCXPI6T53iHTfIUJrU6adTrCC2qJeHZERxhlbI1Bjjt/msv0tadQ1wUs +N+gDS63pYaACbvXy8MWy7Vu33PqUXHeeE6V/Uq2V8viTO96LXFvKWlJbYK8U90vv +o/ufQJVtMVT8QtPHRh8jrdkPSHCa2XV4cdFyQzR1bldZwgJcJmApzyMZFo6IQ6XU +5MsI+yMRQ+hDKXJioaldXgjUkK642M4UwtBV8ob2xJNDd2ZhwLnoQdeXeGADbkpy +rqXRfboQnoZsG4q5WTP468SQvvG5 +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIFQTCCAymgAwIBAgITBmyf0pY1hp8KD+WGePhbJruKNzANBgkqhkiG9w0BAQwF +ADA5MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRkwFwYDVQQDExBBbWF6 +b24gUm9vdCBDQSAyMB4XDTE1MDUyNjAwMDAwMFoXDTQwMDUyNjAwMDAwMFowOTEL +MAkGA1UEBhMCVVMxDzANBgNVBAoTBkFtYXpvbjEZMBcGA1UEAxMQQW1hem9uIFJv +b3QgQ0EgMjCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK2Wny2cSkxK +gXlRmeyKy2tgURO8TW0G/LAIjd0ZEGrHJgw12MBvIITplLGbhQPDW9tK6Mj4kHbZ +W0/jTOgGNk3Mmqw9DJArktQGGWCsN0R5hYGCrVo34A3MnaZMUnbqQ523BNFQ9lXg +1dKmSYXpN+nKfq5clU1Imj+uIFptiJXZNLhSGkOQsL9sBbm2eLfq0OQ6PBJTYv9K +8nu+NQWpEjTj82R0Yiw9AElaKP4yRLuH3WUnAnE72kr3H9rN9yFVkE8P7K6C4Z9r +2UXTu/Bfh+08LDmG2j/e7HJV63mjrdvdfLC6HM783k81ds8P+HgfajZRRidhW+me +z/CiVX18JYpvL7TFz4QuK/0NURBs+18bvBt+xa47mAExkv8LV/SasrlX6avvDXbR +8O70zoan4G7ptGmh32n2M8ZpLpcTnqWHsFcQgTfJU7O7f/aS0ZzQGPSSbtqDT6Zj +mUyl+17vIWR6IF9sZIUVyzfpYgwLKhbcAS4y2j5L9Z469hdAlO+ekQiG+r5jqFoz +7Mt0Q5X5bGlSNscpb/xVA1wf+5+9R+vnSUeVC06JIglJ4PVhHvG/LopyboBZ/1c6 ++XUyo05f7O0oYtlNc/LMgRdg7c3r3NunysV+Ar3yVAhU/bQtCSwXVEqY0VThUWcI +0u1ufm8/0i2BWSlmy5A5lREedCf+3euvAgMBAAGjQjBAMA8GA1UdEwEB/wQFMAMB +Af8wDgYDVR0PAQH/BAQDAgGGMB0GA1UdDgQWBBSwDPBMMPQFWAJI/TPlUq9LhONm +UjANBgkqhkiG9w0BAQwFAAOCAgEAqqiAjw54o+Ci1M3m9Zh6O+oAA7CXDpO8Wqj2 +LIxyh6mx/H9z/WNxeKWHWc8w4Q0QshNabYL1auaAn6AFC2jkR2vHat+2/XcycuUY ++gn0oJMsXdKMdYV2ZZAMA3m3MSNjrXiDCYZohMr/+c8mmpJ5581LxedhpxfL86kS +k5Nrp+gvU5LEYFiwzAJRGFuFjWJZY7attN6a+yb3ACfAXVU3dJnJUH/jWS5E4ywl +7uxMMne0nxrpS10gxdr9HIcWxkPo1LsmmkVwXqkLN1PiRnsn/eBG8om3zEK2yygm +btmlyTrIQRNg91CMFa6ybRoVGld45pIq2WWQgj9sAq+uEjonljYE1x2igGOpm/Hl +urR8FLBOybEfdF849lHqm/osohHUqS0nGkWxr7JOcQ3AWEbWaQbLU8uz/mtBzUF+ +fUwPfHJ5elnNXkoOrJupmHN5fLT0zLm4BwyydFy4x2+IoZCn9Kr5v2c69BoVYh63 +n749sSmvZ6ES8lgQGVMDMBu4Gon2nL2XA46jCfMdiyHxtN/kHNGfZQIG6lzWE7OE +76KlXIx3KadowGuuQNKotOrN8I1LOJwZmhsoVLiJkO/KdYE+HvJkJMcYr07/R54H +9jVlpNMKVv/1F2Rs76giJUmTtt8AF9pYfl3uxRuw0dFfIRDH+fO6AgonB8Xx1sfT +4PsJYGw= +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIBtjCCAVugAwIBAgITBmyf1XSXNmY/Owua2eiedgPySjAKBggqhkjOPQQDAjA5 +MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRkwFwYDVQQDExBBbWF6b24g +Um9vdCBDQSAzMB4XDTE1MDUyNjAwMDAwMFoXDTQwMDUyNjAwMDAwMFowOTELMAkG +A1UEBhMCVVMxDzANBgNVBAoTBkFtYXpvbjEZMBcGA1UEAxMQQW1hem9uIFJvb3Qg +Q0EgMzBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABCmXp8ZBf8ANm+gBG1bG8lKl +ui2yEujSLtf6ycXYqm0fc4E7O5hrOXwzpcVOho6AF2hiRVd9RFgdszflZwjrZt6j +QjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgGGMB0GA1UdDgQWBBSr +ttvXBp43rDCGB5Fwx5zEGbF4wDAKBggqhkjOPQQDAgNJADBGAiEA4IWSoxe3jfkr +BqWTrBqYaGFy+uGh0PsceGCmQ5nFuMQCIQCcAu/xlJyzlvnrxir4tiz+OpAUFteM +YyRIHN8wfdVoOw== +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIB8jCCAXigAwIBAgITBmyf18G7EEwpQ+Vxe3ssyBrBDjAKBggqhkjOPQQDAzA5 +MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRkwFwYDVQQDExBBbWF6b24g +Um9vdCBDQSA0MB4XDTE1MDUyNjAwMDAwMFoXDTQwMDUyNjAwMDAwMFowOTELMAkG +A1UEBhMCVVMxDzANBgNVBAoTBkFtYXpvbjEZMBcGA1UEAxMQQW1hem9uIFJvb3Qg +Q0EgNDB2MBAGByqGSM49AgEGBSuBBAAiA2IABNKrijdPo1MN/sGKe0uoe0ZLY7Bi +9i0b2whxIdIA6GO9mif78DluXeo9pcmBqqNbIJhFXRbb/egQbeOc4OO9X4Ri83Bk +M6DLJC9wuoihKqB1+IGuYgbEgds5bimwHvouXKNCMEAwDwYDVR0TAQH/BAUwAwEB +/zAOBgNVHQ8BAf8EBAMCAYYwHQYDVR0OBBYEFNPsxzplbszh2naaVvuc84ZtV+WB +MAoGCCqGSM49BAMDA2gAMGUCMDqLIfG9fhGt0O9Yli/W651+kI0rz2ZVwyzjKKlw +CkcO8DdZEv8tmZQoTipPNU0zWgIxAOp1AE47xDqUEpHJWEadIRNyp4iciuRMStuW +1KyLa2tJElMzrdfkviT8tQp21KW8EA== +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIID7zCCAtegAwIBAgIBADANBgkqhkiG9w0BAQsFADCBmDELMAkGA1UEBhMCVVMx +EDAOBgNVBAgTB0FyaXpvbmExEzARBgNVBAcTClNjb3R0c2RhbGUxJTAjBgNVBAoT +HFN0YXJmaWVsZCBUZWNobm9sb2dpZXMsIEluYy4xOzA5BgNVBAMTMlN0YXJmaWVs +ZCBTZXJ2aWNlcyBSb290IENlcnRpZmljYXRlIEF1dGhvcml0eSAtIEcyMB4XDTA5 +MDkwMTAwMDAwMFoXDTM3MTIzMTIzNTk1OVowgZgxCzAJBgNVBAYTAlVTMRAwDgYD +VQQIEwdBcml6b25hMRMwEQYDVQQHEwpTY290dHNkYWxlMSUwIwYDVQQKExxTdGFy +ZmllbGQgVGVjaG5vbG9naWVzLCBJbmMuMTswOQYDVQQDEzJTdGFyZmllbGQgU2Vy +dmljZXMgUm9vdCBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkgLSBHMjCCASIwDQYJKoZI +hvcNAQEBBQADggEPADCCAQoCggEBANUMOsQq+U7i9b4Zl1+OiFOxHz/Lz58gE20p +OsgPfTz3a3Y4Y9k2YKibXlwAgLIvWX/2h/klQ4bnaRtSmpDhcePYLQ1Ob/bISdm2 +8xpWriu2dBTrz/sm4xq6HZYuajtYlIlHVv8loJNwU4PahHQUw2eeBGg6345AWh1K +Ts9DkTvnVtYAcMtS7nt9rjrnvDH5RfbCYM8TWQIrgMw0R9+53pBlbQLPLJGmpufe +hRhJfGZOozptqbXuNC66DQO4M99H67FrjSXZm86B0UVGMpZwh94CDklDhbZsc7tk +6mFBrMnUVN+HL8cisibMn1lUaJ/8viovxFUcdUBgF4UCVTmLfwUCAwEAAaNCMEAw +DwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYEFJxfAN+q +AdcwKziIorhtSpzyEZGDMA0GCSqGSIb3DQEBCwUAA4IBAQBLNqaEd2ndOxmfZyMI +bw5hyf2E3F/YNoHN2BtBLZ9g3ccaaNnRbobhiCPPE95Dz+I0swSdHynVv/heyNXB +ve6SbzJ08pGCL72CQnqtKrcgfU28elUSwhXqvfdqlS5sdJ/PHLTyxQGjhdByPq1z +qwubdQxtRbeOlKyWN7Wg0I8VRw7j6IPdj/3vQQF3zCepYoUz8jcI73HPdwbeyBkd +iEDPfUYd/x7H4c7/I9vG+o1VTqkC50cRRj70/b17KSa7qWFiNyi2LSr2EIZkyXCn +0q23KXB56jzaYyWf/Wi3MOxw+3WKt21gZ7IeyLnp2KhvAotnDU0mV3HaIPzBSlCN +sSi6 +-----END CERTIFICATE----- diff --git a/internal/enclave/provider/awskms/config.go b/internal/enclave/provider/awskms/config.go new file mode 100644 index 0000000..d63a508 --- /dev/null +++ b/internal/enclave/provider/awskms/config.go @@ -0,0 +1,21 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package awskms + +// Config ... +type Config struct { + Arns []string `json:"arns"` // example: arn:aws:kms:us-east-2:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab + ConnectTimeout int `json:"connectTimeout"` + AwsproxyEndpoint string `json:"awsproxyEndpoint"` +} diff --git a/internal/enclave/provider/awskms/factory.go b/internal/enclave/provider/awskms/factory.go new file mode 100644 index 0000000..0809ab9 --- /dev/null +++ b/internal/enclave/provider/awskms/factory.go @@ -0,0 +1,26 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package awskms + +import ( + "context" + + "github.com/aws/aws-sdk-go-v2/aws" +) + +// Factory builds an ephemeral Provider for one Initialize call. +// The runtime selects NewWithAttestation or NewForDevelopment and captures +// configuration via a closure. Tests inject fakes. The ctx is the per-call +// request context so construction-time work observes the caller's deadline. +type Factory func(ctx context.Context, awsCfg aws.Config, arns []string) (Provider, error) diff --git a/internal/enclave/provider/awskms/kmserror.go b/internal/enclave/provider/awskms/kmserror.go new file mode 100644 index 0000000..a23bdd3 --- /dev/null +++ b/internal/enclave/provider/awskms/kmserror.go @@ -0,0 +1,75 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package awskms + +import ( + "errors" + "net" + + "github.com/aws/aws-sdk-go-v2/service/kms/types" + "github.com/aws/smithy-go" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// StatusFromError translates AWS KMS SDK errors to gRPC status codes +// per the CCHAIN-2027 design. Returns nil if err is nil. Returns the +// original err unchanged when the error is not recognised, so the +// caller can apply its own fallback (e.g. wrap as Internal). The name +// mirrors grpc/status.FromError, this being a KMS-aware variant. +func StatusFromError(err error) error { + if err == nil { + return nil + } + + var invalidCt *types.InvalidCiphertextException + if errors.As(err, &invalidCt) { + return status.Error(codes.InvalidArgument, err.Error()) + } + + var limitExc *types.LimitExceededException + if errors.As(err, &limitExc) { + return status.Error(codes.Unavailable, err.Error()) + } + + var apiErr smithy.APIError + if errors.As(err, &apiErr) { + switch apiErr.ErrorCode() { + case "AccessDeniedException": + return status.Error(codes.PermissionDenied, err.Error()) + case "ExpiredTokenException", "InvalidClientTokenId", "UnrecognizedClientException": + return status.Error(codes.Unauthenticated, err.Error()) + case "ThrottlingException", "RequestLimitExceeded": + return status.Error(codes.Unavailable, err.Error()) + } + } + + var netErr net.Error + if errors.As(err, &netErr) { + return status.Error(codes.Unavailable, err.Error()) + } + + return err +} + +// credentialAuthError returns err when it is a credential-wide KMS +// authentication failure. These errors are not region-specific, so a later +// regional transport or timeout failure must not hide them. +func credentialAuthError(err error) error { + mapped := StatusFromError(err) + if mapped != err && status.Code(mapped) == codes.Unauthenticated { + return err + } + return nil +} diff --git a/internal/enclave/provider/awskms/kmserror_test.go b/internal/enclave/provider/awskms/kmserror_test.go new file mode 100644 index 0000000..a62cef6 --- /dev/null +++ b/internal/enclave/provider/awskms/kmserror_test.go @@ -0,0 +1,104 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package awskms + +import ( + "errors" + "net" + "net/url" + "testing" + + "github.com/aws/aws-sdk-go-v2/service/kms/types" + "github.com/aws/smithy-go" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func TestStatusFromError(t *testing.T) { + tests := map[string]struct { + err error + wantCode codes.Code + wantMapped bool // true => StatusFromError returns a status.Error; false => returns input unchanged + }{ + "Nil": { + err: nil, wantMapped: false, + }, + "AccessDenied": { + err: &smithy.GenericAPIError{Code: "AccessDeniedException", Message: "no kms:GenerateDataKey"}, + wantCode: codes.PermissionDenied, + wantMapped: true, + }, + "ExpiredToken": { + err: &smithy.GenericAPIError{Code: "ExpiredTokenException", Message: "security token expired"}, + wantCode: codes.Unauthenticated, + wantMapped: true, + }, + "InvalidClientTokenID": { + err: &smithy.GenericAPIError{Code: "InvalidClientTokenId", Message: "access key does not exist"}, + wantCode: codes.Unauthenticated, + wantMapped: true, + }, + "UnrecognizedClient": { + err: &smithy.GenericAPIError{Code: "UnrecognizedClientException", Message: "invalid credentials"}, + wantCode: codes.Unauthenticated, + wantMapped: true, + }, + "Throttling": { + err: &smithy.GenericAPIError{Code: "ThrottlingException", Message: "Rate exceeded"}, + wantCode: codes.Unavailable, + wantMapped: true, + }, + "RequestLimitExceeded": { + err: &smithy.GenericAPIError{Code: "RequestLimitExceeded", Message: "Too many"}, + wantCode: codes.Unavailable, + wantMapped: true, + }, + "InvalidCiphertext": { + err: &types.InvalidCiphertextException{Message: stringPtr("bad blob")}, + wantCode: codes.InvalidArgument, + wantMapped: true, + }, + "LimitExceeded": { + err: &types.LimitExceededException{Message: stringPtr("limit")}, + wantCode: codes.Unavailable, + wantMapped: true, + }, + "WrappedNetError": { + err: &url.Error{Op: "Post", URL: "http://127.0.0.1:9000", Err: &net.OpError{Op: "dial", Err: errors.New("connection refused")}}, + wantCode: codes.Unavailable, + wantMapped: true, + }, + "Unrecognised": { + err: errors.New("something else"), + wantMapped: false, + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + got := StatusFromError(tt.err) + if !tt.wantMapped { + require.Equal(t, tt.err, got, "unrecognised error must pass through unchanged") + return + } + require.NotNil(t, got) + st, ok := status.FromError(got) + require.True(t, ok, "mapped error must be a gRPC status") + require.Equal(t, tt.wantCode, st.Code()) + }) + } +} + +func stringPtr(s string) *string { return &s } diff --git a/internal/enclave/provider/awskms/testdata/kms-us-east-1-chain.pem b/internal/enclave/provider/awskms/testdata/kms-us-east-1-chain.pem new file mode 100644 index 0000000..e971032 --- /dev/null +++ b/internal/enclave/provider/awskms/testdata/kms-us-east-1-chain.pem @@ -0,0 +1,75 @@ +-----BEGIN CERTIFICATE----- +MIIIujCCB6KgAwIBAgIQBKR73vAIxaSB0qepym3r1zANBgkqhkiG9w0BAQsFADA8 +MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRwwGgYDVQQDExNBbWF6b24g +UlNBIDIwNDggTTA0MB4XDTI1MTIxMTAwMDAwMFoXDTI2MTIwNjIzNTk1OVowJjEk +MCIGA1UEAxMba21zLnVzLWVhc3QtMS5hbWF6b25hd3MuY29tMIIBIjANBgkqhkiG +9w0BAQEFAAOCAQ8AMIIBCgKCAQEAm0n42+jwUNVn4eVR3pOyJGMSFLjm+zcOMq7N +1o5xsOm5OTci8plob9APTXIkzuKXTZZ+JrEPcrs3Yv8PXs3wXUgl+UXc1yuc3gS4 +SGNWe69YnmcysIEQKKYBMCo1P3FvtHuSYMzntI4WuaWKOMLLTzaUqAf70W/gCFVR +icp9AD3b+zT3ZN5CElyYPpMKXne5DyaFMxY3BfpSXWfU/1gpzLwDIU4MWQchn/M6 +Wj9nJAy0++XR9//fUxxvK9wZeaTWyU6/zlnWQ43F2oeUTL1B9XGuskXj/VJSoLPH +m8KkoGpVVpzsOxmvJsL7GT4FC3Y0Q05FaSoDKicah6Pz1uEmJQIDAQABo4IFzDCC +BcgwHwYDVR0jBBgwFoAUH1KSYVaCVH+BZtgdPQqqMlyH3QgwHQYDVR0OBBYEFBqT +UfhHhtjNx5G+Se8XN5cHgmt+MIIDBQYDVR0RBIIC/DCCAviCG2ttcy51cy1lYXN0 +LTEuYW1hem9uYXdzLmNvbYIXa21zLWwudXMtZWFzdC0xLmFwaS5hd3OCHWttcy1o +LnVzLWVhc3QtMS5hbWF6b25hd3MuY29tghdrbXMtaS51cy1lYXN0LTEuYXBpLmF3 +c4IXa21zLWYudXMtZWFzdC0xLmFwaS5hd3OCF2ttcy1jLnVzLWVhc3QtMS5hcGku +YXdzgh1rbXMtZC51cy1lYXN0LTEuYW1hem9uYXdzLmNvbYIda21zLWMudXMtZWFz +dC0xLmFtYXpvbmF3cy5jb22CHWttcy1rLnVzLWVhc3QtMS5hbWF6b25hd3MuY29t +ghVrbXMudXMtZWFzdC0xLmFwaS5hd3OCHWttcy1sLnVzLWVhc3QtMS5hbWF6b25h +d3MuY29tgh1rbXMtZy51cy1lYXN0LTEuYW1hem9uYXdzLmNvbYIda21zLWIudXMt +ZWFzdC0xLmFtYXpvbmF3cy5jb22CF2ttcy1lLnVzLWVhc3QtMS5hcGkuYXdzgh1r +bXMtai51cy1lYXN0LTEuYW1hem9uYXdzLmNvbYIXa21zLWIudXMtZWFzdC0xLmFw +aS5hd3OCF2ttcy1oLnVzLWVhc3QtMS5hcGkuYXdzgh1rbXMtZi51cy1lYXN0LTEu +YW1hem9uYXdzLmNvbYIXa21zLWsudXMtZWFzdC0xLmFwaS5hd3OCF2ttcy1nLnVz +LWVhc3QtMS5hcGkuYXdzghdrbXMtai51cy1lYXN0LTEuYXBpLmF3c4Ida21zLWEu +dXMtZWFzdC0xLmFtYXpvbmF3cy5jb22CHWttcy1pLnVzLWVhc3QtMS5hbWF6b25h +d3MuY29tgiIqLmttcy51cy1lYXN0LTEudnBjZS5hbWF6b25hd3MuY29tghdrbXMt +YS51cy1lYXN0LTEuYXBpLmF3c4IXa21zLWQudXMtZWFzdC0xLmFwaS5hd3OCHWtt +cy1lLnVzLWVhc3QtMS5hbWF6b25hd3MuY29tMBMGA1UdIAQMMAowCAYGZ4EMAQIB +MA4GA1UdDwEB/wQEAwIFoDATBgNVHSUEDDAKBggrBgEFBQcDATA7BgNVHR8ENDAy +MDCgLqAshipodHRwOi8vY3JsLnIybTA0LmFtYXpvbnRydXN0LmNvbS9yMm0wNC5j +cmwwdQYIKwYBBQUHAQEEaTBnMC0GCCsGAQUFBzABhiFodHRwOi8vb2NzcC5yMm0w +NC5hbWF6b250cnVzdC5jb20wNgYIKwYBBQUHMAKGKmh0dHA6Ly9jcnQucjJtMDQu +YW1hem9udHJ1c3QuY29tL3IybTA0LmNlcjAMBgNVHRMBAf8EAjAAMIIBfwYKKwYB +BAHWeQIEAgSCAW8EggFrAWkAdQDXbX0Q0af1d8LH6V/XAL/5gskzWmXh0LMBcxfA +yMVpdwAAAZsLJK2fAAAEAwBGMEQCIBco88tsgTVCk/VJ9XqQ/dLfYYq65+MumaUh +hoaxaRVZAiBa24LjwgJFCJFXLX4tgAG2YXUOKcgjghaeGaKLeno2RgB3AMijxH/H +s625NWsBP2p6Em3jOk5DpcZG+ZetOXWZHc+aAAABmwskrfAAAAQDAEgwRgIhAJUf +lyA7ZN+cHa3voeteCCy1SKqrRUKD3oSdsF2LdPhvAiEArZrMsSsJQbNQxTk19Ifk +oz3BEsoeicfxmrMXEvxVpKoAdwDCMX5XRRmjRe5/ON6ykEHrx8IhWiK/f9W1rXaa +2Q5SzQAAAZsLJK3QAAAEAwBIMEYCIQD+5pTdVgSk9cfdUa5vWpHZ70PcshVjpsZs +nnrB9BA98gIhAP57ouJveQZtvNKXjp1mABYuyA2BQr3TZEKOvmV3WwQxMA0GCSqG +SIb3DQEBCwUAA4IBAQCam1n2NUUdi7XRbQNOTKepAhBMzOAGqMNZmD6liUqWjM4p +8+RRRQ2Dal+33phM7HuTK/KUrNYhUnRXIac3H38eFzD69U2zUfAWaJ7CQMclj/3x +bguHn5ZwIyj1VaePa5PDvaUVeJu2ykJ9M8wKEuu92ljDBX8Hnbd+TdXzm3U3aUSS +ds3QqI4T2QrqedgqF8AcY+4a2eS6YXRRZBJz0KIzCuzS8kz443dPyCl/SL4v6nvL +R8Wy4ZVNL/0wsAVu9j4aIDSJVkyR2LfMmoMcgsQ0fF9yXSau8pAIBYKUA4jP4VW7 +Lu6PzgUsbpQlvMDMbqZTQBXW/tWYumIB4/W4Qw+g +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIEXjCCA0agAwIBAgITB3MSTyqVLj7Rili9uF0bwM5fJzANBgkqhkiG9w0BAQsF +ADA5MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRkwFwYDVQQDExBBbWF6 +b24gUm9vdCBDQSAxMB4XDTIyMDgyMzIyMjYzNVoXDTMwMDgyMzIyMjYzNVowPDEL +MAkGA1UEBhMCVVMxDzANBgNVBAoTBkFtYXpvbjEcMBoGA1UEAxMTQW1hem9uIFJT +QSAyMDQ4IE0wNDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAM3pVR6A +lQOp4xe776FdePXyejgA38mYx1ou9/jrpV6Sfn+/oqBKgwhY6ePsQHHQayWBJdBn +v4Wz363qRI4XUh9swBFJ11TnZ3LqOMvHmWq2+loA0QPtOfXdJ2fHBLrBrngtJ/GB +0p5olAVYrSZgvQGP16Rf8ddtNyxEEhYm3HuhmNi+vSeAq1tLYJPAvRCXonTpWdSD +xY6hvdmxlqTYi82AtBXSfpGQ58HHM0hw0C6aQakghrwWi5fGslLOqzpimNMIsT7c +qa0GJx6JfKqJqmQQNplO2h8n9ZsFJgBowof01ppdoLAWg6caMOM0om/VILKaa30F +9W/r8Qjah7ltGVkCAwEAAaOCAVowggFWMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYD +VR0PAQH/BAQDAgGGMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAdBgNV +HQ4EFgQUH1KSYVaCVH+BZtgdPQqqMlyH3QgwHwYDVR0jBBgwFoAUhBjMhTTsvAyU +lC4IWZzHshBOCggwewYIKwYBBQUHAQEEbzBtMC8GCCsGAQUFBzABhiNodHRwOi8v +b2NzcC5yb290Y2ExLmFtYXpvbnRydXN0LmNvbTA6BggrBgEFBQcwAoYuaHR0cDov +L2NydC5yb290Y2ExLmFtYXpvbnRydXN0LmNvbS9yb290Y2ExLmNlcjA/BgNVHR8E +ODA2MDSgMqAwhi5odHRwOi8vY3JsLnJvb3RjYTEuYW1hem9udHJ1c3QuY29tL3Jv +b3RjYTEuY3JsMBMGA1UdIAQMMAowCAYGZ4EMAQIBMA0GCSqGSIb3DQEBCwUAA4IB +AQA+1O5UsAaNuW3lHzJtpNGwBnZd9QEYFtxpiAnIaV4qApnGS9OCw5ZPwie7YSlD +ZF5yyFPsFhUC2Q9uJHY/CRV1b5hIiGH0+6+w5PgKiY1MWuWT8VAaJjFxvuhM7a/e +fN2TIw1Wd6WCl6YRisunjQOrSP+unqC8A540JNyZ1JOE3jVqat3OZBGgMvihdj2w +Y23EpwesrKiQzkHzmvSH67PVW4ycbPy08HVZnBxZ5NrlGG9bwXR3fNTaz+c+Ej6c +5AnwI3qkOFgSkg3Y75cdFz6pO/olK+e3AqygAcv0WjzmkDPuBjssuZjCHMC56oH3 +GJkV29Di2j5prHJbwZjG1inU +-----END CERTIFICATE----- diff --git a/internal/enclave/provider/awsproxy/awsproxy.go b/internal/enclave/provider/awsproxy/awsproxy.go new file mode 100644 index 0000000..9c34619 --- /dev/null +++ b/internal/enclave/provider/awsproxy/awsproxy.go @@ -0,0 +1,83 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux + +package awsproxy + +import ( + "context" + "errors" + "fmt" + "net" + + "github.com/circlefin/arc-remote-signer/internal/common/byteproxy" +) + +// proxyName is the lifecycle.Runnable name reported by both the enabled +// and disabled providers. +const proxyName = "enclave-awsproxy" + +// Provider exposes the proxy as a byteproxy.Proxy so the enclave's +// lifecycle.Manager can supervise it alongside the gRPC server. +type Provider interface { + byteproxy.Proxy +} + +// New creates an enclave-side proxy. The transport defaults to vsock (the +// zero value); pass WithTCPTransport() to select TCP for dev/CI. Passing +// both transport options is allowed and the last one wins. +func New(cfg *Config, opts ...Option) (Provider, error) { + if cfg == nil { + return nil, errors.New("awsproxy: nil config") + } + // transport's zero value is vsock; start with its listen/dial so a call + // with no transport option runs vsock. A transport option overrides both. + o := options{listen: vsockTransportListen, dial: vsockTransportDial} + for _, apply := range opts { + apply(&o) + } + + p, err := byteproxy.NewAWSProxy(proxyName, cfg.BasePort, cfg.MaxConns, + func(_ string, port uint32) (byteproxy.AWSServiceBinding, error) { + endpoint, err := describeEndpoint(&o, port) + if err != nil { + return byteproxy.AWSServiceBinding{}, err + } + return byteproxy.AWSServiceBinding{ + Endpoint: endpoint, + Listen: func() (net.Listener, error) { return o.listen(port) }, + Dial: func(ctx context.Context, _ net.Conn) (net.Conn, error) { return o.dial(ctx, port) }, + }, nil + }) + if err != nil { + return nil, fmt.Errorf("awsproxy: %w", err) + } + return p, nil +} + +// describeEndpoint produces the informational upstream string logged by +// byteproxy on dial failure. Both transports dial port-based; only the +// host part differs. The default branch guards against a future transport +// kind added without a matching case, surfacing it rather than silently +// logging an empty endpoint. +func describeEndpoint(o *options, port uint32) (string, error) { + switch o.transport { + case byteproxy.TransportVsock: + return fmt.Sprintf("vsock CID=%d:%d", parentCID, port), nil + case byteproxy.TransportTCP: + return fmt.Sprintf("tcp %s:%d", o.tcpParentHost, port), nil + default: + return "", fmt.Errorf("awsproxy: unhandled transport kind %d", o.transport) + } +} diff --git a/internal/enclave/provider/awsproxy/awsproxy_stub.go b/internal/enclave/provider/awsproxy/awsproxy_stub.go new file mode 100644 index 0000000..2b43f19 --- /dev/null +++ b/internal/enclave/provider/awsproxy/awsproxy_stub.go @@ -0,0 +1,63 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !linux + +package awsproxy + +import ( + "context" + "errors" + "fmt" + "net" + + "github.com/circlefin/arc-remote-signer/internal/common/byteproxy" +) + +// proxyName is the lifecycle.Runnable name reported by the stub; matches +// the Linux build so logs are identical across platforms. +const proxyName = "enclave-awsproxy" + +// Provider mirrors the Linux Provider so cross-platform callers compile. +type Provider interface { + byteproxy.Proxy +} + +// New creates an enclave-side proxy. On non-Linux only the TCP transport +// is available; selecting WithVsockTransport is a compile-time error +// because the option lives in the Linux-only transport_vsock.go. +func New(cfg *Config, opts ...Option) (Provider, error) { + if cfg == nil { + return nil, errors.New("awsproxy: nil config") + } + o := options{} + for _, apply := range opts { + apply(&o) + } + if o.transport != byteproxy.TransportTCP { + return nil, errors.New("awsproxy: non-Linux requires WithTCPTransport()") + } + p, err := byteproxy.NewAWSProxy(proxyName, cfg.BasePort, cfg.MaxConns, + func(_ string, port uint32) (byteproxy.AWSServiceBinding, error) { + endpoint := fmt.Sprintf("tcp %s:%d", o.tcpParentHost, port) + return byteproxy.AWSServiceBinding{ + Endpoint: endpoint, + Listen: func() (net.Listener, error) { return o.listen(port) }, + Dial: func(ctx context.Context, _ net.Conn) (net.Conn, error) { return o.dial(ctx, port) }, + }, nil + }) + if err != nil { + return nil, fmt.Errorf("awsproxy: %w", err) + } + return p, nil +} diff --git a/internal/enclave/provider/awsproxy/awsproxy_stub_test.go b/internal/enclave/provider/awsproxy/awsproxy_stub_test.go new file mode 100644 index 0000000..1e66578 --- /dev/null +++ b/internal/enclave/provider/awsproxy/awsproxy_stub_test.go @@ -0,0 +1,52 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !linux + +package awsproxy + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" +) + +// stubCfg seeds BasePort: 0 so the OS assigns an ephemeral loopback port +// later, when New()'s TCP transport actually calls net.Listen; this lets +// the TCP happy path bind without a fixed-port conflict. +func stubCfg() *Config { + return &Config{BasePort: 0, MaxConns: 1} +} + +func TestNew_NilConfigErrors(t *testing.T) { + _, err := New(nil, WithTCPTransport()) + require.Error(t, err) + require.Contains(t, err.Error(), "nil config") +} + +// TestNew_NoTransportOptionErrors pins the non-Linux gate: the zero-value +// transport is vsock, which the stub cannot serve, so a call without +// WithTCPTransport() must error rather than silently no-op. +func TestNew_NoTransportOptionErrors(t *testing.T) { + _, err := New(stubCfg()) + require.Error(t, err) + require.Contains(t, err.Error(), "requires WithTCPTransport") +} + +func TestNew_TCPTransportSucceeds(t *testing.T) { + p, err := New(stubCfg(), WithTCPTransport()) + require.NoError(t, err) + require.NotNil(t, p) + t.Cleanup(func() { _ = p.Shutdown(context.Background()) }) +} diff --git a/internal/enclave/provider/awsproxy/awsproxy_test.go b/internal/enclave/provider/awsproxy/awsproxy_test.go new file mode 100644 index 0000000..117d31c --- /dev/null +++ b/internal/enclave/provider/awsproxy/awsproxy_test.go @@ -0,0 +1,190 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux + +package awsproxy + +import ( + "context" + "errors" + "net" + "testing" + + "github.com/circlefin/arc-remote-signer/internal/common/byteproxy" + "github.com/stretchr/testify/suite" +) + +// AwsProxyTestSuite covers the wrapper-specific behaviour layered on top of +// the byteproxy framework: transport selection, port mapping per service, +// and bind failure propagation. Bidirectional copy, half-close, and dial +// failure are covered by the byteproxy package tests. +type AwsProxyTestSuite struct { + suite.Suite +} + +// newLocalListener returns a TCP listener bound to a random loopback port. +func (s *AwsProxyTestSuite) newLocalListener() net.Listener { + s.T().Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + s.Require().NoError(err) + s.T().Cleanup(func() { _ = ln.Close() }) + return ln +} + +// shutdownOnCleanup releases the listeners a successfully constructed proxy +// bound and, unlike a bare deferred Shutdown, keeps a happy-path Shutdown +// regression (resource leak, double-close error) visible: a non-nil error +// fails the test. Reported via t.Errorf rather than require because the +// repo policy forbids FailNow-triggering assertions inside t.Cleanup. +func (s *AwsProxyTestSuite) shutdownOnCleanup(p Provider) { + s.T().Helper() + s.T().Cleanup(func() { + if err := p.Shutdown(context.Background()); err != nil { + s.T().Errorf("Shutdown returned error: %v", err) + } + }) +} + +// TestNew covers constructor validation and transport selection in a +// table-driven form. +func (s *AwsProxyTestSuite) TestNew() { + failingListen := func(_ uint32) (net.Listener, error) { return nil, errors.New("bind failed") } + failingDial := func(_ context.Context, _ uint32) (net.Conn, error) { return nil, errors.New("not used") } + okListen := func(_ uint32) (net.Listener, error) { return s.newLocalListener(), nil } + + tests := map[string]struct { + cfg *Config + opts []Option + wantErr bool + }{ + "NilConfig": { + cfg: nil, + opts: []Option{WithListen(failingListen), WithDial(failingDial)}, + wantErr: true, + }, + "InvalidMaxConns_VsockTransport": { + cfg: &Config{BasePort: 30000, MaxConns: 0}, + opts: []Option{WithVsockTransport(), WithListen(okListen), WithDial(failingDial)}, + wantErr: true, + }, + "BindFailure_VsockTransport": { + cfg: &Config{BasePort: 30000, MaxConns: 4}, + opts: []Option{WithVsockTransport(), WithListen(failingListen), WithDial(failingDial)}, + wantErr: true, + }, + "HappyPath_VsockTransport": { + cfg: &Config{BasePort: 30000, MaxConns: 4}, + opts: []Option{WithVsockTransport(), WithListen(okListen), WithDial(failingDial)}, + wantErr: false, + }, + "HappyPath_TCPTransport": { + cfg: &Config{BasePort: 30000, MaxConns: 4}, + opts: []Option{WithTCPTransport(), WithListen(okListen), WithDial(failingDial)}, + wantErr: false, + }, + // No transport option falls back to the vsock zero value; the + // supplied listen/dial seams let New succeed without a real vsock. + "NoTransportOption_DefaultsToVsock": { + cfg: &Config{BasePort: 30000, MaxConns: 4}, + opts: []Option{WithListen(okListen), WithDial(failingDial)}, + wantErr: false, + }, + "BothTransportOptions_LastWins_Succeeds": { + cfg: &Config{BasePort: 30000, MaxConns: 4}, + opts: []Option{WithVsockTransport(), WithTCPTransport(), WithListen(okListen), WithDial(failingDial)}, + wantErr: false, + }, + } + for name, tt := range tests { + s.Run(name, func() { + p, err := New(tt.cfg, tt.opts...) + if tt.wantErr { + s.Require().Error(err) + return + } + s.Require().NoError(err) + s.Require().NotNil(p) + s.shutdownOnCleanup(p) + }) + } +} + +// TestNew_PortMapping verifies that each service in byteproxy.AWSServiceNames +// maps to BasePort + index, that listen is called once per service, and that +// the service name flows through to the byteproxy ServiceConfig. The +// assertion is non-vacuous because AWSServiceNames is required to be +// non-empty. +func (s *AwsProxyTestSuite) TestNew_PortMapping() { + s.Require().NotEmpty(byteproxy.AWSServiceNames, "AWSServiceNames must declare at least one service") + + var seenPorts []uint32 + listen := func(port uint32) (net.Listener, error) { + seenPorts = append(seenPorts, port) + return s.newLocalListener(), nil + } + dial := func(_ context.Context, _ uint32) (net.Conn, error) { return nil, errors.New("dial unused") } + + const base uint32 = 40000 + p, err := New( + &Config{BasePort: base, MaxConns: 4}, + WithVsockTransport(), WithListen(listen), WithDial(dial), + ) + s.Require().NoError(err) + s.shutdownOnCleanup(p) + + wantPorts := make([]uint32, len(byteproxy.AWSServiceNames)) + for i := range byteproxy.AWSServiceNames { + wantPorts[i] = base + uint32(i) //nolint:gosec // bounded by len(AWSServiceNames) + } + s.Equal(wantPorts, seenPorts) + s.Len(seenPorts, len(byteproxy.AWSServiceNames), + "listen must be called exactly once per service") +} + +// TestWithTransport_LastWins verifies that passing both transport options +// is not an error and that the last option applied selects the transport, +// including its TCP parent host, so the selected kind and its dialer stay +// coherent. +func (s *AwsProxyTestSuite) TestWithTransport_LastWins() { + s.Run("tcp last", func() { + o := options{} + WithVsockTransport()(&o) + WithTCPTransport()(&o) + s.Equal(byteproxy.TransportTCP, o.transport) + s.Equal(tcpParentHost, o.tcpParentHost) + }) + s.Run("vsock last", func() { + o := options{} + WithTCPTransport()(&o) + WithVsockTransport()(&o) + s.Equal(byteproxy.TransportVsock, o.transport) + }) +} + +// TestNew_UnhandledTransportKind covers describeEndpoint's default guard: +// an out-of-range transport (e.g. a future byteproxy.TransportKind added +// without a matching case) must surface an error from New rather than +// silently logging an empty endpoint. Injected via a raw Option so the +// guard stays protected from silent removal. +func (s *AwsProxyTestSuite) TestNew_UnhandledTransportKind() { + badTransport := Option(func(o *options) { o.transport = byteproxy.TransportKind(99) }) + _, err := New(&Config{BasePort: 30000, MaxConns: 4}, badTransport) + s.Require().Error(err) + s.Contains(err.Error(), "unhandled transport kind") +} + +// TestAwsProxyTestSuite is the entry point for the suite. +func TestAwsProxyTestSuite(t *testing.T) { + suite.Run(t, &AwsProxyTestSuite{Suite: suite.Suite{}}) +} diff --git a/internal/enclave/provider/awsproxy/config.go b/internal/enclave/provider/awsproxy/config.go new file mode 100644 index 0000000..db3a3be --- /dev/null +++ b/internal/enclave/provider/awsproxy/config.go @@ -0,0 +1,52 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package awsproxy bridges enclave-internal loopback TCP traffic over +// vsock (production) or TCP (dev/CI) to the parent EC2 instance / host +// gateway. See awsproxy.go for the Linux implementation and +// awsproxy_stub.go for the non-Linux build. +package awsproxy + +// DefaultBasePort is the default loopback TCP port the enclave-side proxy +// listens on. Additional AWS services occupy consecutive ports starting at +// BasePort + len(byteproxy.AWSServiceNames) - 1. +const DefaultBasePort uint32 = 10316 + +// DefaultMaxConns mirrors gateway-tee-signer's socat max-children=50 default +// and bounds the number of concurrent bridged connections per service. +const DefaultMaxConns = 50 + +// Config configures the enclave-side AWS proxy. +// +// The proxy is mandatory: the enclave always bridges its KMS traffic +// through it β€” vsock to the parent in production, TCP to the host +// vsockproxy in dev/CI β€” so there is no on/off toggle. +// +// Construct it via NewConfig(); a zero-value Config is not usable, since +// MaxConns must be > 0 (New rejects MaxConns == 0 via byteproxy.New). +type Config struct { + // BasePort is the loopback TCP port for the first service. Service i + // (per byteproxy.AWSServiceNames) binds to BasePort + i. + BasePort uint32 `mapstructure:"base_port"` + + // MaxConns caps in-flight connections per service. + MaxConns int `mapstructure:"max_conns"` +} + +// NewConfig returns Config defaults suitable for an enclave run. +func NewConfig() *Config { + return &Config{ + BasePort: DefaultBasePort, + MaxConns: DefaultMaxConns, + } +} diff --git a/internal/enclave/provider/awsproxy/config_test.go b/internal/enclave/provider/awsproxy/config_test.go new file mode 100644 index 0000000..ce444e5 --- /dev/null +++ b/internal/enclave/provider/awsproxy/config_test.go @@ -0,0 +1,37 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package awsproxy + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// TestNewConfig_Defaults pins the config defaults. Operators rely on these +// constants when wiring deployment YAML and gateway-tee-signer parity, so +// changing them without intent would silently shift port assignments. +func TestNewConfig_Defaults(t *testing.T) { + cfg := NewConfig() + + require.NotNil(t, cfg) + require.Equal(t, DefaultBasePort, cfg.BasePort) + require.Equal(t, DefaultMaxConns, cfg.MaxConns) + + // Default constants documented to match gateway-tee-signer. + require.Equal(t, uint32(10316), DefaultBasePort, + "DefaultBasePort must match gateway-tee-signer's aws_proxy_port default") + require.Equal(t, 50, DefaultMaxConns, + "DefaultMaxConns must match gateway-tee-signer's socat max-children=50") +} diff --git a/internal/enclave/provider/awsproxy/options.go b/internal/enclave/provider/awsproxy/options.go new file mode 100644 index 0000000..8120bd0 --- /dev/null +++ b/internal/enclave/provider/awsproxy/options.go @@ -0,0 +1,53 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package awsproxy + +import ( + "context" + "net" + + "github.com/circlefin/arc-remote-signer/internal/common/byteproxy" +) + +// listenFunc binds a listener for the given port. +type listenFunc func(port uint32) (net.Listener, error) + +// dialFunc opens an upstream connection for the given service port. The +// ctx is forwarded from byteproxy.Shutdown. +type dialFunc func(ctx context.Context, port uint32) (net.Conn, error) + +// options carries injection points and the selected transport. +type options struct { + listen listenFunc + dial dialFunc + transport byteproxy.TransportKind + tcpParentHost string +} + +// Option customises New for tests and transport selection. +type Option func(*options) + +// WithListen overrides the loopback listener. Production callers should +// rely on the default produced by the selected transport. Must be passed +// AFTER any transport-selection option (WithVsockTransport / WithTCPTransport), +// which install listen/dial unconditionally and would otherwise overwrite +// this override. +func WithListen(fn listenFunc) Option { return func(o *options) { o.listen = fn } } + +// WithDial overrides the upstream dialer. Production callers should rely +// on the default produced by the selected transport. Must be passed AFTER +// any transport-selection option (WithVsockTransport / WithTCPTransport), +// which install listen/dial unconditionally and would otherwise overwrite +// this override. +func WithDial(fn dialFunc) Option { return func(o *options) { o.dial = fn } } diff --git a/internal/enclave/provider/awsproxy/transport_tcp.go b/internal/enclave/provider/awsproxy/transport_tcp.go new file mode 100644 index 0000000..910c1d0 --- /dev/null +++ b/internal/enclave/provider/awsproxy/transport_tcp.go @@ -0,0 +1,63 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package awsproxy + +import ( + "context" + "fmt" + "net" + "time" + + "github.com/circlefin/arc-remote-signer/internal/common/byteproxy" +) + +// tcpDialTimeout bounds an in-flight upstream TCP dial in TCP transport +// mode so a hung dev parent host does not pin Shutdown past its deadline. +const tcpDialTimeout = 10 * time.Second + +// tcpParentHost is the host the enclave reaches the parent vsockproxy on +// in TCP transport mode. Dev/CI is the only consumer; the compose service +// name "vsockproxy" is fixed, so it is hardcoded rather than configured. +const tcpParentHost = "vsockproxy" + +// WithTCPTransport selects TCP transport. Dev/CI is the only consumer; +// the upstream target is the fixed vsockproxy host in compose. The +// transport's listen/dial are installed unconditionally so that when more +// than one transport option is passed the last one wins coherently; the +// WithListen/WithDial test seams still override because they are applied +// after the transport option. +func WithTCPTransport() Option { + return func(o *options) { + o.transport = byteproxy.TransportTCP + o.tcpParentHost = tcpParentHost + o.listen = tcpListenLoopback + o.dial = tcpDialParent(tcpParentHost) + } +} + +// tcpListenLoopback binds 127.0.0.1: on the enclave's loopback +// interface β€” the same listen address used in production. Production uses +// this same listener; only the dial side changes between transports. +func tcpListenLoopback(port uint32) (net.Listener, error) { + return net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port)) +} + +// tcpDialParent returns a dialFunc that TCP-dials parentHost: for +// each accepted session. +func tcpDialParent(parentHost string) dialFunc { + return func(ctx context.Context, port uint32) (net.Conn, error) { + d := net.Dialer{Timeout: tcpDialTimeout} + return d.DialContext(ctx, "tcp", fmt.Sprintf("%s:%d", parentHost, port)) + } +} diff --git a/internal/enclave/provider/awsproxy/transport_tcp_test.go b/internal/enclave/provider/awsproxy/transport_tcp_test.go new file mode 100644 index 0000000..81166aa --- /dev/null +++ b/internal/enclave/provider/awsproxy/transport_tcp_test.go @@ -0,0 +1,47 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package awsproxy + +import ( + "context" + "net" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestTCPListenLoopback_BindsLoopback(t *testing.T) { + ln, err := tcpListenLoopback(0) + require.NoError(t, err) + t.Cleanup(func() { _ = ln.Close() }) + + addr, ok := ln.Addr().(*net.TCPAddr) + require.True(t, ok) + require.Equal(t, "127.0.0.1", addr.IP.String(), + "awsproxy TCP listener must bind 127.0.0.1 inside the enclave") +} + +func TestTCPDialParent_ReachesTarget(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { _ = ln.Close() }) + + addr, ok := ln.Addr().(*net.TCPAddr) + require.True(t, ok) + + dial := tcpDialParent("127.0.0.1") + conn, err := dial(context.Background(), uint32(addr.Port)) //nolint:gosec // bounded by ephemeral port range + require.NoError(t, err) + require.NoError(t, conn.Close()) +} diff --git a/internal/enclave/provider/awsproxy/transport_vsock.go b/internal/enclave/provider/awsproxy/transport_vsock.go new file mode 100644 index 0000000..7b1ca69 --- /dev/null +++ b/internal/enclave/provider/awsproxy/transport_vsock.go @@ -0,0 +1,113 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux + +package awsproxy + +import ( + "context" + "fmt" + "net" + "time" + + "github.com/circlefin/arc-remote-signer/internal/common/byteproxy" + "github.com/mdlayher/vsock" +) + +// parentCID is the well-known vsock context ID for the parent EC2 instance. +const parentCID uint32 = 3 + +// vsockDialTimeout bounds an in-flight vsock dial, matching the 10s bound +// the TCP path gets from net.Dialer.Timeout. mdlayher/vsock exposes no +// dial deadline, so we race vsock.Dial against this timeout (and against +// ctx.Done) rather than configuring one on the dialer. +const vsockDialTimeout = 10 * time.Second + +// WithVsockTransport selects the production vsock transport: loopback +// listen on the configured BasePort, vsock dial to the parent CID for +// each accepted session. References to this option do not compile on +// non-Linux platforms. The transport's listen/dial are installed +// unconditionally so that when more than one transport option is passed +// the last one wins coherently; the WithListen/WithDial test seams still +// override because they are applied after the transport option. +func WithVsockTransport() Option { + return func(o *options) { + o.transport = byteproxy.TransportVsock + o.listen = vsockTransportListen + o.dial = vsockTransportDial + } +} + +// vsockTransportListen binds 127.0.0.1: on the enclave's loopback +// interface (vsock transport still listens on TCP loopback; only the +// dial side is vsock). +func vsockTransportListen(port uint32) (net.Listener, error) { + return net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port)) +} + +// vsockDialResult carries the outcome of vsock.Dial back to the racing +// dial. +type vsockDialResult struct { + conn net.Conn + err error +} + +// vsockTransportDial opens a vsock connection to the parent EC2 instance. +// vsock.Dial has no ctx hook and no dial deadline, so we race it against +// both ctx.Done() and vsockDialTimeout. Whichever loses, a hung dial would +// otherwise pin its byteproxy bridge slot forever and, once every MaxConns +// slot is stuck, the proxy would reject all new connections. +// +// On either abort we drain the buffered result in the background and close +// any conn that arrives late, so a slow-but-eventually-returning dial does +// not leak an fd. A dial that never returns still leaks two goroutines +// (the blocked vsock.Dial and its drainer) plus the fd (mdlayher/vsock +// cannot cancel an in-flight dial), but the bridge slot is released, which +// is the failure that matters. +func vsockTransportDial(ctx context.Context, port uint32) (net.Conn, error) { + // A timeout context bounds the dial and unifies the two abort cases; + // cancel() stops its timer on the success path so a per-dial timer does + // not linger until it fires (unlike a bare time.After). + dialCtx, cancel := context.WithTimeout(ctx, vsockDialTimeout) + defer cancel() + + resultCh := make(chan vsockDialResult, 1) + go func() { + c, err := vsock.Dial(parentCID, port, nil) + resultCh <- vsockDialResult{conn: c, err: err} + }() + select { + case r := <-resultCh: + return r.conn, r.err + case <-dialCtx.Done(): + go drainVsockDial(resultCh) + // dialCtx fires on either the parent ctx (Shutdown) or our timeout. + // If the parent is done, propagate its error verbatim β€” its + // DeadlineExceeded would otherwise be mis-reported as our timeout. + // Only when the parent is still alive did our vsockDialTimeout fire. + if err := ctx.Err(); err != nil { + return nil, err + } + return nil, fmt.Errorf("awsproxy: vsock dial to parent CID %d port %d timed out after %s, err: %w", parentCID, port, vsockDialTimeout, dialCtx.Err()) + } +} + +// drainVsockDial waits for an abandoned vsock.Dial to return and closes any +// connection it produced, so an aborted dial does not leak the fd. +func drainVsockDial(resultCh <-chan vsockDialResult) { + r := <-resultCh + if r.conn != nil { + _ = r.conn.Close() + } +} diff --git a/internal/enclave/provider/enclave/enclave.go b/internal/enclave/provider/enclave/enclave.go index c7b6955..3540ea0 100644 --- a/internal/enclave/provider/enclave/enclave.go +++ b/internal/enclave/provider/enclave/enclave.go @@ -17,49 +17,108 @@ package enclave import ( - enclave "github.com/edgebitio/nitro-enclaves-sdk-go" + "bytes" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "errors" + "fmt" + + "github.com/circlefin/arc-remote-signer/internal/enclave/provider/enclave/kmsrecipient" + "github.com/hf/nsm" + "github.com/hf/nsm/request" + "github.com/hf/nsm/response" ) +const rsaKeyBits = 2048 + var _ Provider = (*provider)(nil) // Provider is the interface for the enclave provider. type Provider interface { DecryptKMSEnvelopedKey(ciphertext []byte) (plainText []byte, err error) - AttestationDocument() []byte + AttestKMSRecipient() ([]byte, error) + Attest(userData []byte) ([]byte, error) +} + +type nsmSession interface { + Send(request.Request) (response.Response, error) + Close() error } +type nsmSessionFactory func() (nsmSession, error) + // provider is the implementation of the Provider interface. type provider struct { - enclaveHandle *enclave.EnclaveHandle - attestationDocument []byte + privateKey *rsa.PrivateKey + publicKey []byte + openSession nsmSessionFactory } // New creates a new provider. func New() (Provider, error) { - handler, err := enclave.GetOrInitializeHandle() + privateKey, err := rsa.GenerateKey(rand.Reader, rsaKeyBits) if err != nil { - return nil, err + return nil, fmt.Errorf("generate attestation key: %w", err) } - attestationDocument, err := handler.Attest(enclave.AttestationOptions{}) + return newProvider(openDefaultSession, privateKey) +} + +func openDefaultSession() (nsmSession, error) { + return nsm.OpenDefaultSession() +} + +func newProvider(openSession nsmSessionFactory, privateKey *rsa.PrivateKey) (*provider, error) { + if privateKey == nil { + return nil, errors.New("attestation private key is nil") + } + publicKey, err := x509.MarshalPKIXPublicKey(&privateKey.PublicKey) if err != nil { - return nil, err + return nil, fmt.Errorf("marshal attestation public key: %w", err) } return &provider{ - enclaveHandle: handler, - attestationDocument: attestationDocument, + privateKey: privateKey, + publicKey: publicKey, + openSession: openSession, }, nil } -// DecryptKMSEnvelopedKey decrypts the given ciphertext using the Nitro enclave. -func (n *provider) DecryptKMSEnvelopedKey(ciphertext []byte) (plainText []byte, err error) { - plainText, err = n.enclaveHandle.DecryptKMSEnvelopedKey(ciphertext) +func requestAttestation(openSession nsmSessionFactory, req *request.Attestation) ([]byte, error) { + session, err := openSession() if err != nil { - return nil, err + return nil, fmt.Errorf("open NSM session: %w", err) } - return plainText, nil + defer func() { _ = session.Close() }() + + result, err := session.Send(req) + if err != nil { + return nil, fmt.Errorf("request NSM attestation: %w", err) + } + if result.Error != "" { + return nil, fmt.Errorf("request NSM attestation: %s", result.Error) + } + if result.Attestation == nil || len(result.Attestation.Document) == 0 { + return nil, errors.New("NSM response missing attestation document") + } + + return bytes.Clone(result.Attestation.Document), nil +} + +// DecryptKMSEnvelopedKey decrypts CiphertextForRecipient with the in-memory RSA +// private key whose public key is included in the NSM-signed attestation. +func (n *provider) DecryptKMSEnvelopedKey(ciphertext []byte) (plainText []byte, err error) { + return kmsrecipient.Decrypt(n.privateKey, ciphertext) +} + +// AttestKMSRecipient requests a new document for the RSA public key that the +// enclave retains. KMS uses this document to encrypt a data key for the enclave. +func (n *provider) AttestKMSRecipient() ([]byte, error) { + return requestAttestation(n.openSession, &request.Attestation{PublicKey: bytes.Clone(n.publicKey)}) } -// AttestationDocument returns the attestation document. -func (n *provider) AttestationDocument() []byte { - return n.attestationDocument +// Attest opens an NSM session. Attest requests a new document. Attest closes the +// session before it returns. The signed user_data field contains the supplied +// bytes. +func (n *provider) Attest(userData []byte) ([]byte, error) { + return requestAttestation(n.openSession, &request.Attestation{UserData: bytes.Clone(userData)}) } diff --git a/internal/enclave/provider/enclave/enclave_mock.go b/internal/enclave/provider/enclave/enclave_mock.go index e4b1829..7b51094 100644 --- a/internal/enclave/provider/enclave/enclave_mock.go +++ b/internal/enclave/provider/enclave/enclave_mock.go @@ -8,6 +8,8 @@ import ( reflect "reflect" gomock "github.com/golang/mock/gomock" + request "github.com/hf/nsm/request" + response "github.com/hf/nsm/response" ) // MockProvider is a mock of Provider interface. @@ -33,18 +35,34 @@ func (m *MockProvider) EXPECT() *MockProviderMockRecorder { return m.recorder } -// AttestationDocument mocks base method. -func (m *MockProvider) AttestationDocument() []byte { +// Attest mocks base method. +func (m *MockProvider) Attest(userData []byte) ([]byte, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "AttestationDocument") + ret := m.ctrl.Call(m, "Attest", userData) ret0, _ := ret[0].([]byte) - return ret0 + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Attest indicates an expected call of Attest. +func (mr *MockProviderMockRecorder) Attest(userData interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Attest", reflect.TypeOf((*MockProvider)(nil).Attest), userData) +} + +// AttestKMSRecipient mocks base method. +func (m *MockProvider) AttestKMSRecipient() ([]byte, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AttestKMSRecipient") + ret0, _ := ret[0].([]byte) + ret1, _ := ret[1].(error) + return ret0, ret1 } -// AttestationDocument indicates an expected call of AttestationDocument. -func (mr *MockProviderMockRecorder) AttestationDocument() *gomock.Call { +// AttestKMSRecipient indicates an expected call of AttestKMSRecipient. +func (mr *MockProviderMockRecorder) AttestKMSRecipient() *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AttestationDocument", reflect.TypeOf((*MockProvider)(nil).AttestationDocument)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AttestKMSRecipient", reflect.TypeOf((*MockProvider)(nil).AttestKMSRecipient)) } // DecryptKMSEnvelopedKey mocks base method. @@ -61,3 +79,55 @@ func (mr *MockProviderMockRecorder) DecryptKMSEnvelopedKey(ciphertext interface{ mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DecryptKMSEnvelopedKey", reflect.TypeOf((*MockProvider)(nil).DecryptKMSEnvelopedKey), ciphertext) } + +// MocknsmSession is a mock of nsmSession interface. +type MocknsmSession struct { + ctrl *gomock.Controller + recorder *MocknsmSessionMockRecorder +} + +// MocknsmSessionMockRecorder is the mock recorder for MocknsmSession. +type MocknsmSessionMockRecorder struct { + mock *MocknsmSession +} + +// NewMocknsmSession creates a new mock instance. +func NewMocknsmSession(ctrl *gomock.Controller) *MocknsmSession { + mock := &MocknsmSession{ctrl: ctrl} + mock.recorder = &MocknsmSessionMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MocknsmSession) EXPECT() *MocknsmSessionMockRecorder { + return m.recorder +} + +// Close mocks base method. +func (m *MocknsmSession) Close() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Close") + ret0, _ := ret[0].(error) + return ret0 +} + +// Close indicates an expected call of Close. +func (mr *MocknsmSessionMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MocknsmSession)(nil).Close)) +} + +// Send mocks base method. +func (m *MocknsmSession) Send(arg0 request.Request) (response.Response, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Send", arg0) + ret0, _ := ret[0].(response.Response) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Send indicates an expected call of Send. +func (mr *MocknsmSessionMockRecorder) Send(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Send", reflect.TypeOf((*MocknsmSession)(nil).Send), arg0) +} diff --git a/internal/enclave/provider/enclave/enclave_test.go b/internal/enclave/provider/enclave/enclave_test.go new file mode 100644 index 0000000..b3d0280 --- /dev/null +++ b/internal/enclave/provider/enclave/enclave_test.go @@ -0,0 +1,228 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enclave + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "errors" + "testing" + + "github.com/hf/nsm/request" + "github.com/hf/nsm/response" + "github.com/stretchr/testify/require" +) + +type testNSMSession struct { + send func(request.Request) (response.Response, error) + closed bool +} + +func (s *testNSMSession) Send(req request.Request) (response.Response, error) { + return s.send(req) +} + +func (s *testNSMSession) Close() error { + s.closed = true + return nil +} + +func TestProvider_AttestBindsUserDataAndUsesNewSession(t *testing.T) { + userData := []byte("validator-public-key") + documents := [][]byte{[]byte("attestation-document-1"), []byte("attestation-document-2")} + var gotRequests []*request.Attestation + var sessions []*testNSMSession + pvd := &provider{ + openSession: func() (nsmSession, error) { + index := len(sessions) + session := &testNSMSession{ + send: func(req request.Request) (response.Response, error) { + gotRequests = append(gotRequests, req.(*request.Attestation)) + return response.Response{ + Attestation: &response.Attestation{Document: documents[index]}, + }, nil + }, + } + sessions = append(sessions, session) + return session, nil + }, + } + + firstDocument, err := pvd.Attest(userData) + require.NoError(t, err) + secondDocument, err := pvd.Attest(userData) + + require.NoError(t, err) + require.Equal(t, documents[0], firstDocument) + require.Equal(t, documents[1], secondDocument) + require.Len(t, gotRequests, 2) + require.Len(t, sessions, 2) + for index := range gotRequests { + require.Equal(t, userData, gotRequests[index].UserData) + require.Empty(t, gotRequests[index].Nonce) + require.Empty(t, gotRequests[index].PublicKey) + require.True(t, sessions[index].closed) + } + + documents[0][0] = 'X' + require.Equal(t, []byte("attestation-document-1"), firstDocument) +} + +func TestProvider_AttestRejectsInvalidResponse(t *testing.T) { + tests := []struct { + name string + openErr error + response response.Response + sendErr error + }{ + { + name: "session open failure", + openErr: errors.New("open failed"), + }, + { + name: "send failure", + sendErr: errors.New("send failed"), + }, + { + name: "NSM error", + response: response.Response{Error: response.ECInternalError}, + }, + { + name: "missing attestation", + response: response.Response{}, + }, + { + name: "empty document", + response: response.Response{ + Attestation: &response.Attestation{}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + session := &testNSMSession{ + send: func(request.Request) (response.Response, error) { + return tt.response, tt.sendErr + }, + } + pvd := &provider{ + openSession: func() (nsmSession, error) { + return session, tt.openErr + }, + } + + document, err := pvd.Attest([]byte("validator-public-key")) + + require.Error(t, err) + require.Nil(t, document) + require.Equal(t, tt.openErr == nil, session.closed) + }) + } +} + +func TestProvider_AttestKMSRecipientUsesRetainedRSAKeyAndFreshSession(t *testing.T) { + privateKey, err := rsa.GenerateKey(rand.Reader, rsaKeyBits) + require.NoError(t, err) + documents := [][]byte{[]byte("attestation-document-1"), []byte("attestation-document-2")} + var gotRequests []*request.Attestation + var sessions []*testNSMSession + gotProvider, err := newProvider(func() (nsmSession, error) { + index := len(sessions) + session := &testNSMSession{ + send: func(req request.Request) (response.Response, error) { + gotRequests = append(gotRequests, req.(*request.Attestation)) + return response.Response{ + Attestation: &response.Attestation{Document: documents[index]}, + }, nil + }, + } + sessions = append(sessions, session) + return session, nil + }, privateKey) + require.NoError(t, err) + + firstDocument, err := gotProvider.AttestKMSRecipient() + require.NoError(t, err) + secondDocument, err := gotProvider.AttestKMSRecipient() + require.NoError(t, err) + + require.Equal(t, documents[0], firstDocument) + require.Equal(t, documents[1], secondDocument) + require.Len(t, gotRequests, 2) + require.Len(t, sessions, 2) + for index := range gotRequests { + parsedPublicKey, err := x509.ParsePKIXPublicKey(gotRequests[index].PublicKey) + require.NoError(t, err) + require.Equal(t, &privateKey.PublicKey, parsedPublicKey) + require.Empty(t, gotRequests[index].Nonce) + require.Empty(t, gotRequests[index].UserData) + require.True(t, sessions[index].closed) + } +} + +func TestProvider_AttestKMSRecipientRejectsInvalidResponse(t *testing.T) { + privateKey, err := rsa.GenerateKey(rand.Reader, rsaKeyBits) + require.NoError(t, err) + tests := []struct { + name string + response response.Response + err error + }{ + { + name: "send failure", + err: errors.New("send failed"), + }, + { + name: "NSM error", + response: response.Response{Error: response.ECInternalError}, + }, + { + name: "missing attestation", + response: response.Response{}, + }, + { + name: "empty document", + response: response.Response{ + Attestation: &response.Attestation{}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + session := &testNSMSession{ + send: func(request.Request) (response.Response, error) { + return tt.response, tt.err + }, + } + + gotProvider, gotErr := newProvider(func() (nsmSession, error) { return session, nil }, privateKey) + require.NoError(t, gotErr) + + document, gotErr := gotProvider.AttestKMSRecipient() + require.Error(t, gotErr) + require.Nil(t, document) + require.True(t, session.closed) + }) + } +} + +func TestNewProvider_RejectsNilPrivateKey(t *testing.T) { + gotProvider, err := newProvider(nil, nil) + + require.Error(t, err) + require.Nil(t, gotProvider) +} diff --git a/internal/enclave/provider/enclave/kmsrecipient/LICENSE-MIT b/internal/enclave/provider/enclave/kmsrecipient/LICENSE-MIT new file mode 100644 index 0000000..2923ab7 --- /dev/null +++ b/internal/enclave/provider/enclave/kmsrecipient/LICENSE-MIT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2015 Andrew Smith + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/internal/enclave/provider/enclave/kmsrecipient/ber.go b/internal/enclave/provider/enclave/kmsrecipient/ber.go new file mode 100644 index 0000000..d17de0a --- /dev/null +++ b/internal/enclave/provider/enclave/kmsrecipient/ber.go @@ -0,0 +1,223 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kmsrecipient + +import ( + "bytes" + "errors" +) + +// The BER-to-DER conversion is derived from mozilla-services/pkcs7 (MIT). +// This version adds strict bounds, trailing-data, nesting, and size checks. +const ( + maxBERDepth = 32 + maxBERSize = 1 << 20 + maxBERTagSize = 9 +) + +func berToDER(input []byte) ([]byte, error) { + if len(input) == 0 || len(input) > maxBERSize { + return nil, errors.New("invalid BER input size") + } + + encoded, next, eoc, err := readBERObject(input, 0, 0) + if err != nil { + return nil, err + } + if eoc || next != len(input) { + return nil, errors.New("invalid BER top-level object") + } + return encoded, nil +} + +func readBERObject(input []byte, offset, depth int) ([]byte, int, bool, error) { + if depth > maxBERDepth || offset < 0 || offset >= len(input) { + return nil, 0, false, errors.New("invalid BER object boundary") + } + if len(input)-offset >= 2 && input[offset] == 0 && input[offset+1] == 0 { + return nil, offset + 2, true, nil + } + + tag, constructed, offset, err := readBERTag(input, offset) + if err != nil { + return nil, 0, false, err + } + + length, indefinite, offset, err := readBERLength(input, offset) + if err != nil { + return nil, 0, false, err + } + if indefinite && !constructed { + return nil, 0, false, errors.New("indefinite primitive BER object") + } + + if !constructed { + if length > len(input)-offset { + return nil, 0, false, errors.New("truncated BER primitive") + } + content := input[offset : offset+length] + return encodeDERObject(tag, content), offset + length, false, nil + } + + return readBERConstructed(input, tag, offset, depth, length, indefinite) +} + +// readBERTag parses a BER identifier, including any high-tag-number +// continuation octets, and reports whether the object is constructed. +func readBERTag(input []byte, offset int) (tag []byte, constructed bool, next int, err error) { + tagStart := offset + firstTagByte := input[offset] + offset++ + if firstTagByte&0x1f == 0x1f { + var err error + offset, err = readBERHighTagEnd(input, tagStart, offset) + if err != nil { + return nil, false, 0, err + } + } + return input[tagStart:offset], firstTagByte&0x20 != 0, offset, nil +} + +func readBERHighTagEnd(input []byte, tagStart, offset int) (int, error) { + var seen bool + for { + if offset >= len(input) { + return 0, errors.New("truncated BER high tag") + } + b := input[offset] + offset++ + if !seen && b == 0x80 { + return 0, errors.New("invalid BER high tag") + } + seen = true + if b&0x80 == 0 { + return offset, nil + } + if offset-tagStart >= maxBERTagSize { + return 0, errors.New("BER tag is too long") + } + } +} + +// readBERConstructed reassembles the children of a constructed BER object +// into DER, handling both definite- and indefinite-length forms. +func readBERConstructed(input, tag []byte, offset, depth, length int, indefinite bool) ([]byte, int, bool, error) { + contentEnd := len(input) + if !indefinite { + if length > len(input)-offset { + return nil, 0, false, errors.New("truncated BER constructed object") + } + contentEnd = offset + length + } + + var content bytes.Buffer + for { + child, childEnd, done, err := readBERConstructedChild(input, offset, depth, contentEnd, indefinite) + if err != nil { + return nil, 0, false, err + } + if done { + offset = childEnd + break + } + if content.Len()+len(child) > maxBERSize { + return nil, 0, false, errors.New("BER output is too large") + } + _, _ = content.Write(child) + offset = childEnd + } + + return encodeDERObject(tag, content.Bytes()), offset, false, nil +} + +func readBERConstructedChild( + input []byte, + offset, depth, contentEnd int, + indefinite bool, +) (child []byte, next int, done bool, err error) { + if indefinite { + if offset >= len(input) { + return nil, 0, false, errors.New("unterminated BER object") + } + } else if offset == contentEnd { + return nil, offset, true, nil + } else if offset > contentEnd { + return nil, 0, false, errors.New("BER child exceeds parent") + } + + child, next, eoc, err := readBERObject(input, offset, depth+1) + if err != nil { + return nil, 0, false, err + } + if eoc { + if !indefinite { + return nil, 0, false, errors.New("unexpected BER end-of-contents") + } + return nil, next, true, nil + } + if next <= offset { + return nil, 0, false, errors.New("BER parser made no progress") + } + return child, next, false, nil +} + +func readBERLength(input []byte, offset int) (length int, indefinite bool, next int, err error) { + if offset >= len(input) { + return 0, false, 0, errors.New("truncated BER length") + } + first := input[offset] + offset++ + if first == 0x80 { + return 0, true, offset, nil + } + if first&0x80 == 0 { + return int(first), false, offset, nil + } + + count := int(first & 0x7f) + if count == 0 || count > 4 || count > len(input)-offset || input[offset] == 0 { + return 0, false, 0, errors.New("invalid BER long length") + } + for i := 0; i < count; i++ { + length = length<<8 | int(input[offset+i]) + } + if length < 128 { + return 0, false, 0, errors.New("non-minimal BER long length") + } + return length, false, offset + count, nil +} + +func encodeDERObject(tag, content []byte) []byte { + encoded := make([]byte, 0, len(tag)+5+len(content)) + encoded = append(encoded, tag...) + encoded = appendDERLength(encoded, len(content)) + encoded = append(encoded, content...) + return encoded +} + +func appendDERLength(dst []byte, length int) []byte { + if length < 128 { + return append(dst, byte(length)) + } + + var encoded [8]byte + i := len(encoded) + for length > 0 { + i-- + encoded[i] = byte(length) + length >>= 8 + } + dst = append(dst, 0x80|byte(len(encoded)-i)) + return append(dst, encoded[i:]...) +} diff --git a/internal/enclave/provider/enclave/kmsrecipient/decrypt.go b/internal/enclave/provider/enclave/kmsrecipient/decrypt.go new file mode 100644 index 0000000..066f889 --- /dev/null +++ b/internal/enclave/provider/enclave/kmsrecipient/decrypt.go @@ -0,0 +1,291 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package kmsrecipient decrypts the CMS envelope returned by attested AWS KMS +// calls in CiphertextForRecipient. +package kmsrecipient + +import ( + "bytes" + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "crypto/rsa" + "crypto/sha256" + "crypto/subtle" + "encoding/asn1" + "errors" +) + +var ( + // ErrDecrypt deliberately hides which CMS, RSA, AES, or padding check failed. + ErrDecrypt = errors.New("decrypt KMS recipient ciphertext") + + oidData = asn1.ObjectIdentifier{1, 2, 840, 113549, 1, 7, 1} + oidEnvelopedData = asn1.ObjectIdentifier{1, 2, 840, 113549, 1, 7, 3} + oidRSAESOAEP = asn1.ObjectIdentifier{1, 2, 840, 113549, 1, 1, 7} + oidMGF1 = asn1.ObjectIdentifier{1, 2, 840, 113549, 1, 1, 8} + oidPSpecified = asn1.ObjectIdentifier{1, 2, 840, 113549, 1, 1, 9} + oidSHA256 = asn1.ObjectIdentifier{2, 16, 840, 1, 101, 3, 4, 2, 1} + oidAES256CBC = asn1.ObjectIdentifier{2, 16, 840, 1, 101, 3, 4, 1, 42} +) + +const ( + envelopedDataVersion = 2 + recipientInfoVersion = 2 + aes256KeySize = 32 +) + +type contentInfo struct { + ContentType asn1.ObjectIdentifier + Content envelopedData `asn1:"explicit,tag:0"` + Extra asn1.RawValue `asn1:"optional"` +} + +type envelopedData struct { + Version int + RecipientInfos []keyTransRecipientInfo `asn1:"set"` + EncryptedContentInfo encryptedContentInfo + Extra asn1.RawValue `asn1:"optional"` +} + +type keyTransRecipientInfo struct { + Version int + RecipientIdentifier []byte `asn1:"tag:0"` + KeyEncryptionAlgorithm algorithmIdentifier + EncryptedKey []byte + Extra asn1.RawValue `asn1:"optional"` +} + +type encryptedContentInfo struct { + ContentType asn1.ObjectIdentifier + ContentEncryptionAlgorithm algorithmIdentifier + EncryptedContent asn1.RawValue `asn1:"tag:0,optional"` + Extra asn1.RawValue `asn1:"optional"` +} + +type rsaOAEPParameters struct { + HashAlgorithm algorithmIdentifier `asn1:"explicit,optional,tag:0"` + MaskGenAlgorithm algorithmIdentifier `asn1:"explicit,optional,tag:1"` + PSourceAlgorithm algorithmIdentifier `asn1:"explicit,optional,tag:2"` + Extra asn1.RawValue `asn1:"optional"` +} + +type algorithmIdentifier struct { + Algorithm asn1.ObjectIdentifier + // Parameters consumes the optional second element; Extra captures a third. + // Algorithm-specific validators must still reject unsupported parameters. + Parameters asn1.RawValue `asn1:"optional"` + Extra asn1.RawValue `asn1:"optional"` +} + +type encryptedKey struct { + rsaCiphertext []byte + ciphertext []byte + iv []byte +} + +// Decrypt unwraps an attested AWS KMS CiphertextForRecipient value. Every +// failure is collapsed to ErrDecrypt so callers cannot distinguish stages. +func Decrypt(privateKey *rsa.PrivateKey, content []byte) ([]byte, error) { + if privateKey == nil { + return nil, ErrDecrypt + } + + envelope, err := parse(content) + if err != nil { + return nil, ErrDecrypt + } + plaintext, err := envelope.decrypt(privateKey) + if err != nil { + return nil, ErrDecrypt + } + return plaintext, nil +} + +func parse(content []byte) (*encryptedKey, error) { + der, err := berToDER(content) + if err != nil { + return nil, err + } + + var ci contentInfo + rest, err := asn1.Unmarshal(der, &ci) + if err != nil || len(rest) != 0 { + return nil, errors.New("invalid CMS content info") + } + if hasExtra(ci.Extra) || hasExtra(ci.Content.Extra) || + !ci.ContentType.Equal(oidEnvelopedData) || ci.Content.Version != envelopedDataVersion { + return nil, errors.New("invalid CMS envelope") + } + if len(ci.Content.RecipientInfos) != 1 { + return nil, errors.New("invalid CMS recipient count") + } + + recipient := ci.Content.RecipientInfos[0] + if hasExtra(recipient.Extra) || recipient.Version != recipientInfoVersion || + len(recipient.RecipientIdentifier) == 0 || len(recipient.EncryptedKey) == 0 { + return nil, errors.New("invalid CMS recipient") + } + if err := validateOAEP(recipient.KeyEncryptionAlgorithm); err != nil { + return nil, err + } + + encryptedContent := ci.Content.EncryptedContentInfo + if hasExtra(encryptedContent.Extra) || hasExtra(encryptedContent.ContentEncryptionAlgorithm.Extra) || + !encryptedContent.ContentType.Equal(oidData) || + !encryptedContent.ContentEncryptionAlgorithm.Algorithm.Equal(oidAES256CBC) { + return nil, errors.New("invalid CMS content encryption algorithm") + } + iv, err := decodeOctetString(encryptedContent.ContentEncryptionAlgorithm.Parameters) + if err != nil || len(iv) != aes.BlockSize { + return nil, errors.New("invalid CMS initialization vector") + } + ciphertext, err := decodeEncryptedContent(encryptedContent.EncryptedContent) + if err != nil || len(ciphertext) == 0 || len(ciphertext)%aes.BlockSize != 0 { + return nil, errors.New("invalid CMS ciphertext") + } + + return &encryptedKey{ + rsaCiphertext: recipient.EncryptedKey, + ciphertext: ciphertext, + iv: iv, + }, nil +} + +func validateOAEP(algorithm algorithmIdentifier) error { + if hasExtra(algorithm.Extra) || !algorithm.Algorithm.Equal(oidRSAESOAEP) || len(algorithm.Parameters.FullBytes) == 0 { + return errors.New("invalid CMS key encryption algorithm") + } + + var params rsaOAEPParameters + rest, err := asn1.Unmarshal(algorithm.Parameters.FullBytes, ¶ms) + if err != nil || len(rest) != 0 || hasExtra(params.Extra) { + return errors.New("invalid RSA-OAEP parameters") + } + if !isSHA256(params.HashAlgorithm) || hasExtra(params.MaskGenAlgorithm.Extra) || + !params.MaskGenAlgorithm.Algorithm.Equal(oidMGF1) { + return errors.New("unsupported RSA-OAEP parameters") + } + if !containsSHA256Algorithm(params.MaskGenAlgorithm.Parameters.FullBytes) { + return errors.New("unsupported RSA-OAEP mask generation function") + } + if len(params.PSourceAlgorithm.Algorithm) != 0 && !isEmptyPSpecified(params.PSourceAlgorithm) { + return errors.New("unsupported RSA-OAEP label") + } + return nil +} + +func isSHA256(algorithm algorithmIdentifier) bool { + return !hasExtra(algorithm.Extra) && algorithm.Algorithm.Equal(oidSHA256) && isNullOrAbsent(algorithm.Parameters) +} + +func containsSHA256Algorithm(data []byte) bool { + var algorithm algorithmIdentifier + rest, err := asn1.Unmarshal(data, &algorithm) + return err == nil && len(rest) == 0 && isSHA256(algorithm) +} + +func isEmptyPSpecified(algorithm algorithmIdentifier) bool { + if hasExtra(algorithm.Extra) || !algorithm.Algorithm.Equal(oidPSpecified) { + return false + } + value, err := decodeOctetString(algorithm.Parameters) + return err == nil && len(value) == 0 +} + +func hasExtra(value asn1.RawValue) bool { + return len(value.FullBytes) != 0 +} + +func isNullOrAbsent(value asn1.RawValue) bool { + return len(value.FullBytes) == 0 || + (value.Class == asn1.ClassUniversal && value.Tag == asn1.TagNull && !value.IsCompound && len(value.Bytes) == 0) +} + +func decodeOctetString(value asn1.RawValue) ([]byte, error) { + if value.Class != asn1.ClassUniversal || value.Tag != asn1.TagOctetString || value.IsCompound { + return nil, errors.New("expected OCTET STRING") + } + return bytes.Clone(value.Bytes), nil +} + +func decodeEncryptedContent(value asn1.RawValue) ([]byte, error) { + if value.Class != asn1.ClassContextSpecific || value.Tag != 0 { + return nil, errors.New("invalid encrypted content tag") + } + if !value.IsCompound { + return bytes.Clone(value.Bytes), nil + } + + remaining := value.Bytes + var ciphertext []byte + for len(remaining) > 0 { + var part []byte + rest, err := asn1.Unmarshal(remaining, &part) + if err != nil || len(rest) >= len(remaining) { + return nil, errors.New("invalid encrypted content fragment") + } + ciphertext = append(ciphertext, part...) + remaining = rest + } + return ciphertext, nil +} + +// decrypt returns stage-specific errors and does not normalize execution time. +// Keep it package-internal; Decrypt is the boundary that collapses failures. +func (e *encryptedKey) decrypt(privateKey *rsa.PrivateKey) ([]byte, error) { + contentKey, err := rsa.DecryptOAEP(sha256.New(), rand.Reader, privateKey, e.rsaCiphertext, nil) + if err != nil { + return nil, err + } + defer clear(contentKey) + if len(contentKey) != aes256KeySize { + return nil, errors.New("invalid AES-256 content key") + } + + block, err := aes.NewCipher(contentKey) + if err != nil { + return nil, err + } + if len(e.iv) != block.BlockSize() || len(e.ciphertext) == 0 || len(e.ciphertext)%block.BlockSize() != 0 { + return nil, errors.New("invalid AES-CBC parameters") + } + + plaintext := make([]byte, len(e.ciphertext)) + cipher.NewCBCDecrypter(block, e.iv).CryptBlocks(plaintext, e.ciphertext) + unpadded, err := unpadPKCS7(plaintext, block.BlockSize()) + if err != nil { + clear(plaintext) + return nil, err + } + return unpadded, nil +} + +func unpadPKCS7(data []byte, blockSize int) ([]byte, error) { + if blockSize <= 0 || len(data) == 0 || len(data)%blockSize != 0 { + return nil, errors.New("invalid PKCS#7 input") + } + + paddingLength := int(data[len(data)-1]) + valid := subtle.ConstantTimeLessOrEq(1, paddingLength) & subtle.ConstantTimeLessOrEq(paddingLength, blockSize) + for i := 0; i < blockSize; i++ { + inPadding := subtle.ConstantTimeLessOrEq(i+1, paddingLength) + matches := subtle.ConstantTimeByteEq(data[len(data)-1-i], byte(paddingLength)) + valid &= subtle.ConstantTimeSelect(inPadding, matches, 1) + } + if valid != 1 { + return nil, errors.New("invalid PKCS#7 padding") + } + return data[:len(data)-paddingLength], nil +} diff --git a/internal/enclave/provider/enclave/kmsrecipient/decrypt_test.go b/internal/enclave/provider/enclave/kmsrecipient/decrypt_test.go new file mode 100644 index 0000000..8bcf3b5 --- /dev/null +++ b/internal/enclave/provider/enclave/kmsrecipient/decrypt_test.go @@ -0,0 +1,578 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kmsrecipient + +import ( + "bytes" + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "crypto/rsa" + "crypto/sha256" + "crypto/x509" + "encoding/asn1" + "encoding/base64" + "encoding/pem" + "testing" + + "github.com/stretchr/testify/require" +) + +// The fixture was produced by AWS KMS and published with the Apache-2.0 +// Edgebit Nitro Enclaves SDK for Go compatibility test. +const testPrivateKeyPEM = `-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAwg8xlWTIwm44aLEqiA5lweHUSm2eeKwrTg3qEUhOVyGAo3eN +XRoD9wOHzjcvS8r/qfQdSdLA9p6IbSxV9LU2fXgYnT3IDhNuQ1rVkiIYqWqPWUn2 +izUMJmbdVFRsgWi7/keXkslZD0DeKQM1R2QsCRZnPGHU3Jo/+2b6dTg8IRoBH2cq +rAPuynqBXYCC9+wNdYMQLA5vdaVzhFBASIVkMDDWlMaFgdOsISMHy9Klm0cXj3RE +02VsHcOQ1NRLY4Ddgpb5r0LUB0nfB4HMeK9plYqkkVF5BJihoGtGmebGuMqSFNgU +XflrxH152bHAZqqV+aIPIy2y4IdaQgP1VJrVKwIDAQABAoIBAQCQhtJNyh6+t2np +hrD/XYGpkPATcmqIwukJm9FMh8ZYnAn7NKmiwiJb0FRPX8gosYoRYE6D0aOGyPEg +Jdnqgx+O+GeUjBO3b/85yKewyxYE7ujN/gjRCnP/EbMbADlDc+Y27cjUOILMmmoa +r1n5zoABUJ8YWGA43+Rw7vPvYy9dEn1fbmsp850u/Grqdi0MUwIpQe9VKkVsYZ0n +HKAz+uY9Mhb/CsveD75cHrpaa5Ilfjkzo47Gah/+E6LB3/5wRjlzNzLMAQT449PW +yt2E/DYtVAR8uAtbfHB3cFcgNrWVg9IwU1G74SwqqwgQfpfEqKqsqG9BBXz0vwLT +o3vczVWZAoGBANJbz5+1XRlblmDV8MnVGoaHoylIA6+xE5iTiAUtopxfh3lMgTAh +sIepf7na0nkNPXFrR48Tkm29Y4f8EU2LY0a1t9WyAyufz9UTA4ABlHCuKztSqpG7 +SgGEQvr/bAE61uN7JwVXGUICAR27OVfy7+iIOCzFDaOwhyfrE2XuP82VAoGBAOwq +DYedgoxuV63BWYDtvUt4olQbBCczJKyDirTGGdiPyQbsfE5eegcfZYxRkiCJ0Z5z +9OQlafIrok93kwkWgta2dj3onbXKLUviyGMSW1kGXoaTZu47rTZ7nxhqS5QeySGl +sHs/8j3+2UPHnwvLMlrMAOhIFQYrlFeQkxvIw+e/AoGAZh2Xjon2JccmGuAAQZon +hEL326RP1cv6HUkQ8KKUm6BsHWAcHodcMJ8Bl/E31vesahCP7k6r+IXFeU/N/ny5 +tqukECKYE2dC9saCHnOl4YVLC0M39gKbDF1uPnYbsgUkJ82yxY7gfgCHFi26yozu +FU17J5CI7HtXQPOGuSaM5nkCgYEAqI4PIAbMYVxz2cDRF9MWsuIDwdGSckPvXe14 +tzNYyRc+nGF3CxwlLiY7fR3PFMgow1XxqFAHwN9htiQa3nahpYuO8vqubUxCbhIL +gaJdbjm8h4J3CXuwUd2DnJJpJOugFBLE1gK664KUIOs92dYKN4G4+BBSaRf7hU/b +nw34vNMCgYBfG/VbQXT1WCcJgVycnU1hX7zmyzB/hk0xkmLR0nUzTgXMKOKUUXgX +2mD7U5VGZPYj7t8P+bz6/HEZqKmOoxFkXpsMPug34ZUWfjv3uCm7CFHtxA+BDT+5 +cJEGAbCDYhyjvtjBLNy7YDQ1hdmCnqMxg/5AIwUMkvTTRg+qepfboA== +-----END RSA PRIVATE KEY-----` + +const testCiphertextForRecipient = `MIAGCSqGSIb3DQEHA6CAMIACAQIxggFrMIIBZwIBAoAgljGgxlmRCtWqvB/s/Aw+ZNTDlc6Uka86SLVmlNmFGAMwPAYJKoZIhvcNAQEHMC+gDzANBglghkgBZQMEAgEFAKEcMBoGCSqGSIb3DQEBCDANBglghkgBZQMEAgEFAASCAQAXmjTiHpg+OcYaf2ISaDNpQcEOq61Sm3re3v+5z2hZPe8eoUGhmMS6pCuC+BRW7RpkjwDaXQzzR/jExnraEET3lj9oyAMMwKIahhHHIZ33qOTq1c/9NtMVZmm/j4UfyCpP8WMAFb2hvwIJbjnAGO9Xbw+NzWaQdvEyNDGUX+bPIuSDc75jjGH5KtdFLopk5k6nsTdU26qLkVE6Mg9Y//s0OJCvmYFgfw15IXDb50xJupWxCwbqGXWmfTBEo9M9AhelVbOXkitZR7hbnT6BZnsfpS2acZRNL4XxC+gg4Ml9fOiYsGWqSK8Lkwlp22rtL70CIHnggbb+oIE4ObR4TV8qMIAGCSqGSIb3DQEHATAdBglghkgBZQMEASoEEEMr/6uiZK+CzgfJvr61JTGggAQwfp0W0Q/QPYmg6AoC3DkE5+beNswVOX9ct5IIgIsvaAhTF9IiHdbX7yLa8YS2WQ/FAAAAAAAAAAAAAA==` + +func TestDecrypt_ValidKMSFixture(t *testing.T) { + privateKey := decodeTestPrivateKey(t) + ciphertext, err := base64.StdEncoding.DecodeString(testCiphertextForRecipient) + require.NoError(t, err) + + got, err := Decrypt(privateKey, ciphertext) + + require.NoError(t, err) + require.Equal(t, []byte{ + 0x3b, 0xe8, 0x2c, 0x44, 0x0f, 0x06, 0xcb, 0x4d, + 0x44, 0xc4, 0xc2, 0xec, 0x3b, 0xf3, 0x0d, 0x47, + 0x24, 0x07, 0xd3, 0xa9, 0x12, 0x5a, 0xa4, 0xc1, + 0x84, 0x2b, 0x98, 0xf6, 0xbd, 0xd2, 0x6e, 0x41, + }, got) +} + +func TestDecrypt_InvalidInputsReturnStableError(t *testing.T) { + privateKey := decodeTestPrivateKey(t) + validCiphertext, err := base64.StdEncoding.DecodeString(testCiphertextForRecipient) + require.NoError(t, err) + wrongPrivateKey, err := rsa.GenerateKey(rand.Reader, 1024) + require.NoError(t, err) + + parsed, err := parse(validCiphertext) + require.NoError(t, err) + ciphertextOffset := bytes.Index(validCiphertext, parsed.ciphertext) + require.NotEqual(t, -1, ciphertextOffset) + badPadding := bytes.Clone(validCiphertext) + badPadding[ciphertextOffset+len(parsed.ciphertext)-1] ^= 0xff + + tests := []struct { + name string + privateKey *rsa.PrivateKey + ciphertext []byte + }{ + {name: "nil private key", ciphertext: validCiphertext}, + {name: "empty CMS", privateKey: privateKey}, + {name: "truncated CMS", privateKey: privateKey, ciphertext: validCiphertext[:len(validCiphertext)-1]}, + {name: "trailing CMS data", privateKey: privateKey, ciphertext: append(bytes.Clone(validCiphertext), 0x00)}, + {name: "RSA failure", privateKey: wrongPrivateKey, ciphertext: validCiphertext}, + {name: "padding failure", privateKey: privateKey, ciphertext: badPadding}, + {name: "oversized CMS", privateKey: privateKey, ciphertext: make([]byte, maxBERSize+1)}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, gotErr := Decrypt(tt.privateKey, tt.ciphertext) + + require.Nil(t, got) + require.ErrorIs(t, gotErr, ErrDecrypt) + require.Equal(t, ErrDecrypt.Error(), gotErr.Error()) + }) + } +} + +func TestDecrypt_TruncatedCMSNeverPanics(t *testing.T) { + privateKey := decodeTestPrivateKey(t) + validCiphertext, err := base64.StdEncoding.DecodeString(testCiphertextForRecipient) + require.NoError(t, err) + + for length := range validCiphertext { + got, gotErr := Decrypt(privateKey, validCiphertext[:length]) + + require.Nil(t, got) + require.ErrorIs(t, gotErr, ErrDecrypt) + } +} + +func TestDecrypt_RejectsUnexpectedTopLevelField(t *testing.T) { + privateKey := decodeTestPrivateKey(t) + validCiphertext, err := base64.StdEncoding.DecodeString(testCiphertextForRecipient) + require.NoError(t, err) + der, err := berToDER(validCiphertext) + require.NoError(t, err) + var topLevel asn1.RawValue + rest, err := asn1.Unmarshal(der, &topLevel) + require.NoError(t, err) + require.Empty(t, rest) + malformed, err := asn1.Marshal(asn1.RawValue{ + Class: asn1.ClassUniversal, + Tag: asn1.TagSequence, + IsCompound: true, + Bytes: append(bytes.Clone(topLevel.Bytes), 0x05, 0x00), + }) + require.NoError(t, err) + + got, err := Decrypt(privateKey, malformed) + + require.ErrorIs(t, err, ErrDecrypt) + require.Nil(t, got) +} + +func TestDecrypt_RejectsUnexpectedNestedFields(t *testing.T) { + privateKey := decodeTestPrivateKey(t) + validCiphertext, err := base64.StdEncoding.DecodeString(testCiphertextForRecipient) + require.NoError(t, err) + der, err := berToDER(validCiphertext) + require.NoError(t, err) + extra := asn1.RawValue{FullBytes: []byte{0x05, 0x00}} + tests := []struct { + name string + mutate func(*contentInfo) + }{ + {name: "enveloped data", mutate: func(ci *contentInfo) { ci.Content.Extra = extra }}, + {name: "recipient", mutate: func(ci *contentInfo) { ci.Content.RecipientInfos[0].Extra = extra }}, + {name: "key algorithm", mutate: func(ci *contentInfo) { ci.Content.RecipientInfos[0].KeyEncryptionAlgorithm.Extra = extra }}, + {name: "encrypted content info", mutate: func(ci *contentInfo) { ci.Content.EncryptedContentInfo.Extra = extra }}, + {name: "content algorithm", mutate: func(ci *contentInfo) { ci.Content.EncryptedContentInfo.ContentEncryptionAlgorithm.Extra = extra }}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var envelope contentInfo + rest, err := asn1.Unmarshal(der, &envelope) + require.NoError(t, err) + require.Empty(t, rest) + tt.mutate(&envelope) + malformed, err := asn1.Marshal(envelope) + require.NoError(t, err) + + got, err := Decrypt(privateKey, malformed) + + require.ErrorIs(t, err, ErrDecrypt) + require.Nil(t, got) + }) + } +} + +func TestDecrypt_RejectsUnexpectedMaskGenAlgorithmField(t *testing.T) { + privateKey := decodeTestPrivateKey(t) + validCiphertext, err := base64.StdEncoding.DecodeString(testCiphertextForRecipient) + require.NoError(t, err) + der, err := berToDER(validCiphertext) + require.NoError(t, err) + + var envelope contentInfo + rest, err := asn1.Unmarshal(der, &envelope) + require.NoError(t, err) + require.Empty(t, rest) + + var params rsaOAEPParameters + rest, err = asn1.Unmarshal( + envelope.Content.RecipientInfos[0].KeyEncryptionAlgorithm.Parameters.FullBytes, + ¶ms, + ) + require.NoError(t, err) + require.Empty(t, rest) + params.MaskGenAlgorithm.Extra = asn1.RawValue{FullBytes: []byte{0x05, 0x00}} + paramsDER, err := asn1.Marshal(params) + require.NoError(t, err) + envelope.Content.RecipientInfos[0].KeyEncryptionAlgorithm.Parameters = asn1.RawValue{FullBytes: paramsDER} + malformed, err := asn1.Marshal(envelope) + require.NoError(t, err) + + got, err := Decrypt(privateKey, malformed) + + require.ErrorIs(t, err, ErrDecrypt) + require.Nil(t, got) +} + +func TestDecrypt_ConstructedEncryptedContent(t *testing.T) { + privateKey := decodeTestPrivateKey(t) + validCiphertext, err := base64.StdEncoding.DecodeString(testCiphertextForRecipient) + require.NoError(t, err) + + tests := []struct { + name string + mutateContent func(*testing.T, []byte) []byte + wantErr error + }{ + { + name: "reassembles octet string fragments", + mutateContent: func(t *testing.T, ciphertext []byte) []byte { + t.Helper() + midpoint := len(ciphertext) / 2 + return marshalConstructedEncryptedContent(t, ciphertext[:midpoint], ciphertext[midpoint:]) + }, + }, + { + name: "rejects malformed octet string fragment", + mutateContent: func(t *testing.T, ciphertext []byte) []byte { + t.Helper() + validFragment, err := asn1.Marshal(ciphertext[:aes.BlockSize]) + require.NoError(t, err) + return marshalRawConstructedEncryptedContent(validFragment, []byte{0x04, 0x02, 0x01}) + }, + wantErr: ErrDecrypt, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mutated := rewriteEncryptedContent(t, validCiphertext, func(ciphertext []byte) asn1.RawValue { + return asn1.RawValue{FullBytes: tt.mutateContent(t, ciphertext)} + }) + + got, err := Decrypt(privateKey, mutated) + + if tt.wantErr != nil { + require.ErrorIs(t, err, tt.wantErr) + require.Nil(t, got) + return + } + require.NoError(t, err) + require.Equal(t, []byte{ + 0x3b, 0xe8, 0x2c, 0x44, 0x0f, 0x06, 0xcb, 0x4d, + 0x44, 0xc4, 0xc2, 0xec, 0x3b, 0xf3, 0x0d, 0x47, + 0x24, 0x07, 0xd3, 0xa9, 0x12, 0x5a, 0xa4, 0xc1, + 0x84, 0x2b, 0x98, 0xf6, 0xbd, 0xd2, 0x6e, 0x41, + }, got) + }) + } +} + +func TestUnpadPKCS7(t *testing.T) { + tests := []struct { + name string + data []byte + want []byte + wantErr bool + }{ + {name: "full block", data: bytes.Repeat([]byte{0x10}, aesBlockSize), want: []byte{}}, + {name: "one byte", data: append(bytes.Repeat([]byte{0x2a}, aesBlockSize-1), 0x01), want: bytes.Repeat([]byte{0x2a}, aesBlockSize-1)}, + {name: "zero length padding", data: append(bytes.Repeat([]byte{0x2a}, aesBlockSize-1), 0x00), wantErr: true}, + {name: "padding exceeds block", data: append(bytes.Repeat([]byte{0x2a}, aesBlockSize-1), 0x11), wantErr: true}, + {name: "inconsistent padding", data: append(bytes.Repeat([]byte{0x2a}, aesBlockSize-2), 0x01, 0x02), wantErr: true}, + {name: "empty input", wantErr: true}, + {name: "partial block", data: []byte{0x01}, wantErr: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := unpadPKCS7(tt.data, aesBlockSize) + + if tt.wantErr { + require.Error(t, err) + require.Nil(t, got) + return + } + require.NoError(t, err) + require.Equal(t, tt.want, got) + }) + } +} + +func TestEncryptedKeyDecrypt_RejectsNonAES256ContentKey(t *testing.T) { + privateKey := decodeTestPrivateKey(t) + contentKey := bytes.Repeat([]byte{0x42}, 16) + rsaCiphertext, err := rsa.EncryptOAEP(sha256.New(), rand.Reader, &privateKey.PublicKey, contentKey, nil) + require.NoError(t, err) + block, err := aes.NewCipher(contentKey) + require.NoError(t, err) + iv := make([]byte, aes.BlockSize) + ciphertext := make([]byte, aes.BlockSize) + cipher.NewCBCEncrypter(block, iv).CryptBlocks(ciphertext, bytes.Repeat([]byte{aes.BlockSize}, aes.BlockSize)) + + got, err := (&encryptedKey{ + rsaCiphertext: rsaCiphertext, + ciphertext: ciphertext, + iv: iv, + }).decrypt(privateKey) + + require.Error(t, err) + require.Nil(t, got) +} + +func TestValidateOAEP_PSpecifiedLabel(t *testing.T) { + tests := []struct { + name string + oid asn1.ObjectIdentifier + label []byte + wantErr bool + }{ + {name: "explicit empty default", oid: oidPSpecified}, + {name: "non-empty label", oid: oidPSpecified, label: []byte("label"), wantErr: true}, + {name: "unexpected algorithm", oid: oidSHA256, wantErr: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + algorithm := buildTestOAEPAlgorithm(t, tt.oid, tt.label) + + err := validateOAEP(algorithm) + + if tt.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + }) + } +} + +func TestValidateOAEP_RejectsExtraParameterField(t *testing.T) { + algorithm := buildTestOAEPAlgorithm(t, oidPSpecified, nil) + var params rsaOAEPParameters + rest, err := asn1.Unmarshal(algorithm.Parameters.FullBytes, ¶ms) + require.NoError(t, err) + require.Empty(t, rest) + params.Extra = asn1.RawValue{FullBytes: []byte{0x05, 0x00}} + paramsDER, err := asn1.Marshal(params) + require.NoError(t, err) + algorithm.Parameters = asn1.RawValue{FullBytes: paramsDER} + + err = validateOAEP(algorithm) + + require.Error(t, err) +} + +func TestDecodeEncryptedContent_Primitive(t *testing.T) { + got, err := decodeEncryptedContent(asn1.RawValue{ + Class: asn1.ClassContextSpecific, + Tag: 0, + Bytes: []byte{0x01, 0x02, 0x03}, + }) + + require.NoError(t, err) + require.Equal(t, []byte{0x01, 0x02, 0x03}, got) +} + +func TestDecodeEncryptedContent_Constructed(t *testing.T) { + first, err := asn1.Marshal([]byte{0x01, 0x02}) + require.NoError(t, err) + second, err := asn1.Marshal([]byte{0x03, 0x04}) + require.NoError(t, err) + + got, err := decodeEncryptedContent(asn1.RawValue{ + Class: asn1.ClassContextSpecific, + Tag: 0, + IsCompound: true, + Bytes: append(first, second...), + }) + + require.NoError(t, err) + require.Equal(t, []byte{0x01, 0x02, 0x03, 0x04}, got) +} + +func TestDecodeEncryptedContent_RejectsMalformedFragment(t *testing.T) { + got, err := decodeEncryptedContent(asn1.RawValue{ + Class: asn1.ClassContextSpecific, + Tag: 0, + IsCompound: true, + Bytes: []byte{0x04, 0x02, 0x01}, + }) + + require.Error(t, err) + require.Nil(t, got) +} + +func TestBERToDER_RejectsMalformedInput(t *testing.T) { + tests := []struct { + name string + input []byte + }{ + {name: "unterminated indefinite object", input: []byte{0x30, 0x80, 0x02, 0x01, 0x01}}, + {name: "indefinite primitive", input: []byte{0x04, 0x80, 0x00, 0x00}}, + {name: "truncated high tag", input: []byte{0x1f, 0x81}}, + {name: "non-minimal high tag", input: []byte{0x1f, 0x80, 0x01, 0x00}}, + {name: "long tag", input: []byte{0x1f, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x01, 0x00}}, + {name: "truncated length", input: []byte{0x04}}, + {name: "non-minimal long length", input: []byte{0x04, 0x81, 0x01, 0x00}}, + {name: "leading-zero long length", input: []byte{0x04, 0x82, 0x00, 0x80}}, + {name: "truncated primitive", input: []byte{0x04, 0x02, 0x01}}, + {name: "unexpected end-of-contents", input: []byte{0x30, 0x02, 0x00, 0x00}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := berToDER(tt.input) + + require.Error(t, err) + require.Nil(t, got) + }) + } +} + +func TestBERToDER_MaximumOutputSize(t *testing.T) { + der, err := berToDER(maxSizeBERObject()) + + require.NoError(t, err) + require.Len(t, der, maxDERObjectSize()) +} + +func FuzzDecrypt(f *testing.F) { + privateKey := decodeTestPrivateKey(f) + validCiphertext, err := base64.StdEncoding.DecodeString(testCiphertextForRecipient) + require.NoError(f, err) + f.Add(validCiphertext) + f.Add([]byte{}) + f.Add([]byte{0x30, 0x80, 0x00, 0x00}) + + f.Fuzz(func(t *testing.T, ciphertext []byte) { + plaintext, err := Decrypt(privateKey, ciphertext) + if err != nil { + require.ErrorIs(t, err, ErrDecrypt) + require.Nil(t, plaintext) + } + }) +} + +func FuzzBERToDER(f *testing.F) { + f.Add([]byte{0x02, 0x01, 0x01}) + f.Add([]byte{0x30, 0x80, 0x02, 0x01, 0x01, 0x00, 0x00}) + f.Add([]byte{}) + f.Add(maxSizeBERObject()) + + f.Fuzz(func(t *testing.T, input []byte) { + der, err := berToDER(input) + if err == nil { + require.NotEmpty(t, der) + require.LessOrEqual(t, len(der), maxDERObjectSize()) + } + }) +} + +func maxDERObjectSize() int { + return maxBERSize + maxBERTagSize + len(appendDERLength(nil, maxBERSize)) +} + +func maxSizeBERObject() []byte { + // Each nested indefinite object adds one byte when converted to DER, which + // offsets the maximum-size outer tag, indefinite length, and EOC marker. + const nestedConstructedObjects = maxBERTagSize + 3 + + input := make([]byte, 0, maxBERSize) + input = append(input, 0x3f) + input = append(input, bytes.Repeat([]byte{0x81}, maxBERTagSize-2)...) + input = append(input, 0x01, 0x80) + for range nestedConstructedObjects { + input = append(input, 0x30, 0x80) + } + + const primitiveHeaderSize = 5 + eocSize := 2 * (nestedConstructedObjects + 1) + payloadSize := maxBERSize - len(input) - primitiveHeaderSize - eocSize + input = append(input, 0x04, 0x83, byte(payloadSize>>16), byte(payloadSize>>8), byte(payloadSize)) + input = append(input, make([]byte, payloadSize)...) + input = append(input, make([]byte, eocSize)...) + return input +} + +func rewriteEncryptedContent(t *testing.T, ciphertext []byte, mutate func([]byte) asn1.RawValue) []byte { + t.Helper() + + der, err := berToDER(ciphertext) + require.NoError(t, err) + + var envelope contentInfo + rest, err := asn1.Unmarshal(der, &envelope) + require.NoError(t, err) + require.Empty(t, rest) + + decodedCiphertext, err := decodeEncryptedContent( + envelope.Content.EncryptedContentInfo.EncryptedContent, + ) + require.NoError(t, err) + + envelope.Content.EncryptedContentInfo.EncryptedContent = mutate( + decodedCiphertext, + ) + mutated, err := asn1.Marshal(envelope) + require.NoError(t, err) + return mutated +} + +func marshalConstructedEncryptedContent(t *testing.T, fragments ...[]byte) []byte { + t.Helper() + + rawFragments := make([][]byte, 0, len(fragments)) + for _, fragment := range fragments { + raw, err := asn1.Marshal(fragment) + require.NoError(t, err) + rawFragments = append(rawFragments, raw) + } + return marshalRawConstructedEncryptedContent(rawFragments...) +} + +func marshalRawConstructedEncryptedContent(fragments ...[]byte) []byte { + content := bytes.Join(fragments, nil) + encoded := []byte{0xa0} + encoded = appendDERLength(encoded, len(content)) + encoded = append(encoded, content...) + return encoded +} + +func decodeTestPrivateKey(t testing.TB) *rsa.PrivateKey { + t.Helper() + block, _ := pem.Decode([]byte(testPrivateKeyPEM)) + require.NotNil(t, block) + key, err := x509.ParsePKCS1PrivateKey(block.Bytes) + require.NoError(t, err) + return key +} + +func buildTestOAEPAlgorithm(t testing.TB, pSourceOID asn1.ObjectIdentifier, label []byte) algorithmIdentifier { + t.Helper() + null := asn1.RawValue{Class: asn1.ClassUniversal, Tag: asn1.TagNull} + hash := algorithmIdentifier{Algorithm: oidSHA256, Parameters: null} + hashDER, err := asn1.Marshal(hash) + require.NoError(t, err) + labelDER, err := asn1.Marshal(label) + require.NoError(t, err) + paramsDER, err := asn1.Marshal(rsaOAEPParameters{ + HashAlgorithm: hash, + MaskGenAlgorithm: algorithmIdentifier{Algorithm: oidMGF1, Parameters: asn1.RawValue{FullBytes: hashDER}}, + PSourceAlgorithm: algorithmIdentifier{Algorithm: pSourceOID, Parameters: asn1.RawValue{FullBytes: labelDER}}, + }) + require.NoError(t, err) + return algorithmIdentifier{ + Algorithm: oidRSAESOAEP, + Parameters: asn1.RawValue{FullBytes: paramsDER}, + } +} + +const aesBlockSize = 16 diff --git a/internal/enclave/provider/keystore/keystore.go b/internal/enclave/provider/keystore/keystore.go deleted file mode 100644 index b108d98..0000000 --- a/internal/enclave/provider/keystore/keystore.go +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright (c) 2026, Circle Internet Group, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//go:generate mockgen -source $GOFILE -destination keystore_mock.go -package $GOPACKAGE keystore - -// Package keystore provides a keystore provider. -package keystore - -import ( - "encoding/hex" - "sync" - - "github.com/circlefin/arc-remote-signer/internal/enclave/common/crypto" -) - -var _ Provider = (*ProviderImpl)(nil) - -// Provider is the interface for the keystore provider. -type Provider interface { - Set(ciphertext []byte, secretKey crypto.Key) error - Get(ciphertext []byte) (secretKey crypto.Key) -} - -// ProviderImpl is the implementation of the keystore provider. -type ProviderImpl struct { - mu sync.RWMutex - store map[string]crypto.Key -} - -// New returns a new keystore provider. -func New() *ProviderImpl { - return &ProviderImpl{ - store: make(map[string]crypto.Key), - } -} - -// Set sets the secret key. -func (p *ProviderImpl) Set(ciphertext []byte, secretKey crypto.Key) error { - p.mu.Lock() - defer p.mu.Unlock() - p.store[hex.EncodeToString(ciphertext)] = secretKey - return nil -} - -// Get gets the secret key. Returns nil if the secret key is not found. -func (p *ProviderImpl) Get(ciphertext []byte) crypto.Key { - p.mu.RLock() - defer p.mu.RUnlock() - secretKey, ok := p.store[hex.EncodeToString(ciphertext)] - if !ok { - return nil - } - return secretKey -} diff --git a/internal/enclave/provider/keystore/keystore_mock.go b/internal/enclave/provider/keystore/keystore_mock.go deleted file mode 100644 index adf3b0f..0000000 --- a/internal/enclave/provider/keystore/keystore_mock.go +++ /dev/null @@ -1,63 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: keystore.go - -// Package keystore is a generated GoMock package. -package keystore - -import ( - reflect "reflect" - - crypto "github.com/circlefin/arc-remote-signer/internal/enclave/common/crypto" - gomock "github.com/golang/mock/gomock" -) - -// MockProvider is a mock of Provider interface. -type MockProvider struct { - ctrl *gomock.Controller - recorder *MockProviderMockRecorder -} - -// MockProviderMockRecorder is the mock recorder for MockProvider. -type MockProviderMockRecorder struct { - mock *MockProvider -} - -// NewMockProvider creates a new mock instance. -func NewMockProvider(ctrl *gomock.Controller) *MockProvider { - mock := &MockProvider{ctrl: ctrl} - mock.recorder = &MockProviderMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockProvider) EXPECT() *MockProviderMockRecorder { - return m.recorder -} - -// Get mocks base method. -func (m *MockProvider) Get(ciphertext []byte) crypto.Key { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Get", ciphertext) - ret0, _ := ret[0].(crypto.Key) - return ret0 -} - -// Get indicates an expected call of Get. -func (mr *MockProviderMockRecorder) Get(ciphertext interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockProvider)(nil).Get), ciphertext) -} - -// Set mocks base method. -func (m *MockProvider) Set(ciphertext []byte, secretKey crypto.Key) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Set", ciphertext, secretKey) - ret0, _ := ret[0].(error) - return ret0 -} - -// Set indicates an expected call of Set. -func (mr *MockProviderMockRecorder) Set(ciphertext, secretKey interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Set", reflect.TypeOf((*MockProvider)(nil).Set), ciphertext, secretKey) -} diff --git a/internal/enclave/provider/keystore/keystore_test.go b/internal/enclave/provider/keystore/keystore_test.go deleted file mode 100644 index 136fd29..0000000 --- a/internal/enclave/provider/keystore/keystore_test.go +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright (c) 2026, Circle Internet Group, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package keystore - -import ( - "testing" - - "github.com/circlefin/arc-remote-signer/internal/enclave/common/crypto/bls" - "github.com/stretchr/testify/suite" -) - -// KeystoreTestSuite is the test suite for the keystore provider. -type KeystoreTestSuite struct { - suite.Suite - provider *ProviderImpl -} - -// SetupTest sets up the test environment. -func (suite *KeystoreTestSuite) SetupTest() { - provider := New() - suite.provider = provider -} - -func (suite *KeystoreTestSuite) TestKeystoreProvider() { - suite.Run("SetAndGet", func() { - ciphertext := []byte("test_ciphertext") - secretKey, err := bls.New() - suite.Require().NoError(err) - - err = suite.provider.Set(ciphertext, secretKey) - suite.Require().NoError(err) - - retrievedKey := suite.provider.Get(ciphertext) - suite.Require().NotNil(retrievedKey) - suite.Equal(secretKey, retrievedKey) - }) - - suite.Run("GetNonExistentKey", func() { - ciphertext := []byte("non_existent_ciphertext") - retrievedKey := suite.provider.Get(ciphertext) - suite.Require().Nil(retrievedKey) - }) - - suite.Run("SetMultipleKeys", func() { - ciphertext1 := []byte("ciphertext1") - secretKey1, err := bls.New() - suite.Require().NoError(err) - ciphertext2 := []byte("ciphertext2") - secretKey2, err := bls.New() - suite.Require().NoError(err) - - err1 := suite.provider.Set(ciphertext1, secretKey1) - suite.Require().NoError(err1) - err2 := suite.provider.Set(ciphertext2, secretKey2) - suite.Require().NoError(err2) - - retrievedKey1 := suite.provider.Get(ciphertext1) - suite.Require().NotNil(retrievedKey1) - suite.Equal(secretKey1, retrievedKey1) - - retrievedKey2 := suite.provider.Get(ciphertext2) - suite.Require().NotNil(retrievedKey2) - suite.Equal(secretKey2, retrievedKey2) - }) - - suite.Run("ConcurrentAccess", func() { - ciphertext := []byte("concurrent_ciphertext") - secretKey, err := bls.New() - suite.Require().NoError(err) - - err = suite.provider.Set(ciphertext, secretKey) - suite.Require().NoError(err) - - done := make(chan bool, 10) - for i := 0; i < 10; i++ { - go func() { - retrievedKey := suite.provider.Get(ciphertext) - var result bool - if retrievedKey != nil && secretKey == retrievedKey { - result = true - } else { - result = false - } - done <- result - }() - } - results := make([]bool, 10) - for i := 0; i < 10; i++ { - results[i] = <-done - } - for _, result := range results { - suite.Require().True(result, "Concurrent access test failed") - } - }) -} - -// TestKeystoreProvider runs the test suite. -func TestKeystoreProvider(t *testing.T) { - suite.Run(t, new(KeystoreTestSuite)) -} diff --git a/internal/enclave/public/public.go b/internal/enclave/public/public.go index f36f7c4..0188474 100644 --- a/internal/enclave/public/public.go +++ b/internal/enclave/public/public.go @@ -18,7 +18,6 @@ import ( "fmt" "buf.build/go/protovalidate" - "github.com/circlefin/arc-remote-signer/internal/common/config" grpcserver "github.com/circlefin/arc-remote-signer/internal/common/grpc/server" "github.com/circlefin/arc-remote-signer/internal/common/lifecycle" "github.com/circlefin/arc-remote-signer/proto/pb" @@ -29,7 +28,6 @@ import ( // CreateServerParams is a param struct passed in to create a new public server. type CreateServerParams struct { ServiceName string - Env config.Environment EnclaveService pb.EnclaveServiceServer NitroEnclaveEnabled bool } @@ -43,7 +41,6 @@ func New(cfg *grpcserver.Config, params CreateServerParams) (lifecycle.Runnable, grpcServer := grpcserver.NewServer(grpcserver.RequiredEngineParams{ ServiceName: params.ServiceName, - Env: params.Env, APIStatsService: nil, UnaryInterceptors: []grpc.UnaryServerInterceptor{ protovalidatemw.UnaryServerInterceptor(validator), diff --git a/internal/enclave/public/public_test.go b/internal/enclave/public/public_test.go index d77db95..d0ae964 100644 --- a/internal/enclave/public/public_test.go +++ b/internal/enclave/public/public_test.go @@ -20,14 +20,15 @@ import ( "testing" "time" - "github.com/circlefin/arc-remote-signer/internal/common/config" grpcServer "github.com/circlefin/arc-remote-signer/internal/common/grpc/server" "github.com/circlefin/arc-remote-signer/proto/pb" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "google.golang.org/grpc" + "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials/insecure" grpcHealth "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/status" ) func TestNew_ReturnsRunnableWithServiceName(t *testing.T) { @@ -38,7 +39,6 @@ func TestNew_ReturnsRunnableWithServiceName(t *testing.T) { runnable, err := New(cfg, CreateServerParams{ ServiceName: "enclave.public", - Env: config.Dev, EnclaveService: &pb.UnimplementedEnclaveServiceServer{}, NitroEnclaveEnabled: false, }) @@ -55,7 +55,6 @@ func TestNew_ReturnsErrorWhenPortIsInvalid(t *testing.T) { runnable, err := New(cfg, CreateServerParams{ ServiceName: "enclave.public", - Env: config.Dev, EnclaveService: &pb.UnimplementedEnclaveServiceServer{}, NitroEnclaveEnabled: false, }) @@ -83,7 +82,6 @@ func TestServer_StartsAndAcceptsConnections(t *testing.T) { runnable, err := New(cfg, CreateServerParams{ ServiceName: "enclave.public", - Env: config.Dev, EnclaveService: &pb.UnimplementedEnclaveServiceServer{}, NitroEnclaveEnabled: false, }) @@ -129,3 +127,92 @@ func TestServer_StartsAndAcceptsConnections(t *testing.T) { require.Fail(t, "server did not stop after shutdown") } } + +// TestServer_Initialize_ProtovalidateRejectsMalformedRequests verifies the +// protovalidate interceptor wired up in New rejects malformed +// InitializeRequest payloads with InvalidArgument before the handler's +// Unimplemented path runs. This covers the interceptor wiring itself; the +// handler-level behaviour is covered by the service package tests. +func TestServer_Initialize_ProtovalidateRejectsMalformedRequests(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + addr := listener.Addr().String() + _, portStr, err := net.SplitHostPort(addr) + require.NoError(t, err) + require.NoError(t, listener.Close()) + + port, err := strconv.Atoi(portStr) + require.NoError(t, err) + + cfg := &grpcServer.Config{ + Host: "127.0.0.1", + Port: port, + } + + runnable, err := New(cfg, CreateServerParams{ + ServiceName: "enclave.public", + EnclaveService: &pb.UnimplementedEnclaveServiceServer{}, + NitroEnclaveEnabled: false, + }) + require.NoError(t, err) + require.NotNil(t, runnable) + + serverErr := make(chan error, 1) + go func() { + serverErr <- runnable.Run() + }() + t.Cleanup(func() { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := runnable.Shutdown(shutdownCtx); err != nil { + t.Errorf("failed to shutdown server: %v", err) + } + select { + case err := <-serverErr: + if err != nil { + t.Errorf("server returned error: %v", err) + } + case <-time.After(2 * time.Second): + t.Errorf("server did not stop after shutdown") + } + }) + + conn, err := grpc.NewClient(addr, grpc.WithTransportCredentials(insecure.NewCredentials())) + require.NoError(t, err, "failed to create client") + t.Cleanup(func() { assert.NoError(t, conn.Close()) }) + + client := pb.NewEnclaveServiceClient(conn) + + tests := map[string]struct { + req *pb.InitializeRequest + }{ + "NilCredentials": { + req: &pb.InitializeRequest{}, + }, + "EmptySessionToken": { + req: &pb.InitializeRequest{ + Credentials: &pb.AwsCredentials{ + AccessKeyId: "AKIA-test", + SecretAccessKey: "secret", + SessionToken: "", + Region: "us-east-1", + }, + }, + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + resp, err := client.Initialize(ctx, tt.req) + require.Error(t, err) + require.Nil(t, resp) + st, ok := status.FromError(err) + require.True(t, ok, "error must be a gRPC status") + require.Equal(t, codes.InvalidArgument, st.Code(), + "protovalidate interceptor must reject malformed request with InvalidArgument") + }) + } +} diff --git a/internal/enclave/runtime_dev.go b/internal/enclave/runtime_dev.go new file mode 100644 index 0000000..66884fe --- /dev/null +++ b/internal/enclave/runtime_dev.go @@ -0,0 +1,44 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !prod + +package enclave + +import "github.com/circlefin/arc-remote-signer/internal/enclave/provider/awsproxy" + +// Run starts the enclave service with the configured transport. +func Run(cfg *Config) error { + return run(cfg, cfg.NitroEnclave.Enabled, buildAWSProxy) +} + +type transportKind int + +const ( + transportVsock transportKind = iota + transportTCP +) + +func selectTransport(cfg *Config) transportKind { + if cfg.NitroEnclave.Enabled { + return transportVsock + } + return transportTCP +} + +func buildAWSProxy(cfg *Config) (awsproxy.Provider, error) { + if selectTransport(cfg) == transportVsock { + return buildAwsProxyVsock(cfg) + } + return awsproxy.New(cfg.Awsproxy, awsproxy.WithTCPTransport()) +} diff --git a/internal/enclave/runtime_prod_linux_cgo.go b/internal/enclave/runtime_prod_linux_cgo.go new file mode 100644 index 0000000..7232cd1 --- /dev/null +++ b/internal/enclave/runtime_prod_linux_cgo.go @@ -0,0 +1,21 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux && cgo && prod + +package enclave + +// RunNitro starts the enclave service with Nitro and VSOCK. +func RunNitro(cfg *Config) error { + return run(cfg, true, buildAwsProxyVsock) +} diff --git a/internal/enclave/service/enclave/enclave.go b/internal/enclave/service/enclave/enclave.go index a3280f1..aaaed83 100644 --- a/internal/enclave/service/enclave/enclave.go +++ b/internal/enclave/service/enclave/enclave.go @@ -16,15 +16,11 @@ package enclave import ( "context" - "crypto/sha256" - "encoding/binary" - "fmt" + "errors" - "github.com/circlefin/arc-remote-signer/internal/common/crypto/aes" - "github.com/circlefin/arc-remote-signer/internal/enclave/common/crypto" - aesCommon "github.com/circlefin/arc-remote-signer/internal/enclave/common/crypto/aes" + enclaveCrypto "github.com/circlefin/arc-remote-signer/internal/enclave/common/crypto" + "github.com/circlefin/arc-remote-signer/internal/enclave/provider/awskms" "github.com/circlefin/arc-remote-signer/internal/enclave/provider/enclave" - "github.com/circlefin/arc-remote-signer/internal/enclave/provider/keystore" "github.com/circlefin/arc-remote-signer/proto/pb" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -34,89 +30,184 @@ import ( type Service struct { pb.UnimplementedEnclaveServiceServer nitroEnclaveEnabled bool - keystore keystore.Provider enclavePvd enclave.Provider + kmsFactory awskms.Factory + awsproxyEndpoint string + gate *initGate } -// New creates a new enclave service instance. -func New(nitroEnclaveEnabled bool, keystore keystore.Provider, enclavePvd enclave.Provider) *Service { - return &Service{ +func (s *Service) activeKey() (enclaveCrypto.Key, error) { + if s.gate == nil { + return nil, status.Error(codes.FailedPrecondition, "enclave is not initialized") + } + state := s.gate.state() + if state == nil || state.key == nil { + return nil, status.Error(codes.FailedPrecondition, "enclave is not initialized") + } + return state.key, nil +} + +// New creates a new enclave service instance. Each initialization attempt +// creates an ephemeral KMS client through awsproxyEndpoint. The attempt +// completes key generation or recovery before it publishes the Ready state. +// +// nitroEnclaveEnabled controls the proxy transport and the KMS TLS strategy. +// Standard AWS KMS keeps the AWS endpoint and uses end-to-end TLS. LocalStack +// uses awsproxyEndpoint as the SDK base endpoint and uses plain HTTP. The +// Initialize request selects the KMS backend. Nitro mode rejects LocalStack. +func New( + nitroEnclaveEnabled bool, + enclavePvd enclave.Provider, + awskmsFactory awskms.Factory, + awsproxyEndpoint string, +) *Service { + s := &Service{ nitroEnclaveEnabled: nitroEnclaveEnabled, - keystore: keystore, enclavePvd: enclavePvd, + kmsFactory: awskmsFactory, + awsproxyEndpoint: awsproxyEndpoint, } + s.gate = newInitGate(s.runInit) + return s } -// GetAttestation returns the enclave attestation document. -func (s *Service) GetAttestation(_ context.Context, _ *pb.GetAttestationRequest) (*pb.GetAttestationResponse, error) { - if s.enclavePvd == nil { - return nil, status.Error(codes.FailedPrecondition, "enclave is not enabled") +// newWithGate is the test-friendly constructor that lets tests inject a +// custom runInit callback (e.g. one that fails) without exposing a +// settable field on Service after construction. Production code uses New. +// +// Callers must keep nitroEnclaveEnabled consistent with the runInit callback. +// New derives both values from the same flag, but this test seam accepts them +// independently. +func newWithGate( + nitroEnclaveEnabled bool, + enclavePvd enclave.Provider, + runInit func(context.Context, *pb.InitializeRequest) error, +) *Service { + wrappedRunInit := func(ctx context.Context, req *pb.InitializeRequest) (*readyState, error) { + if err := runInit(ctx, req); err != nil { + return nil, err + } + return &readyState{ + response: &pb.InitializeResponse{PublicKey: []byte("test-public-key")}, + }, nil + } + return &Service{ + nitroEnclaveEnabled: nitroEnclaveEnabled, + enclavePvd: enclavePvd, + gate: newInitGate(wrappedRunInit), } - return &pb.GetAttestationResponse{ - AttestationDocument: s.enclavePvd.AttestationDocument(), - }, nil } -// GenerateKey creates a new encrypted private key and returns its public key material. -func (s *Service) GenerateKey(_ context.Context, req *pb.GenerateKeyRequest) (*pb.GenerateKeyResponse, error) { +// Initialize completes one key transaction. Concurrent calls share one +// attempt. After success, matching requests receive cloned responses. An +// existing-key request can replay the envelope from a successful generation +// response. Other key sources fail. A failed attempt does not publish the +// Ready state, and a later call can retry. +// +// Each attempt creates an ephemeral AWS KMS client through the injected +// factory. The attempt uses the client only for key generation or recovery. +// +// Note: AwsCredentials fields arrive as proto-generated strings, which +// Go cannot zero out. Documenting the limitation here; switching the +// proto field to bytes (or adopting a Zeroize-style wrapper) is a +// follow-up improvement. +func (s *Service) Initialize(ctx context.Context, req *pb.InitializeRequest) (*pb.InitializeResponse, error) { if req == nil { return nil, status.Error(codes.InvalidArgument, "request is nil") } - alg, err := toAlgorithm(req.Algorithm) - if err != nil { - return nil, status.Error(codes.InvalidArgument, err.Error()) + if req.Credentials == nil { + return nil, status.Error(codes.InvalidArgument, "credentials are required") } - - // Get the plaintext data key from cache or decrypt it from enclave ciphertext. - plainDataKey, err := s.loadDataKey(req.EnclaveEncryptedDataKey) + if len(req.KmsKeyArns) == 0 { + return nil, status.Error(codes.InvalidArgument, "kms_key_arns is required") + } + if s.nitroEnclaveEnabled && req.KmsLocalstackEnabled { + return nil, status.Error(codes.InvalidArgument, "LocalStack is not available in Nitro mode") + } + // Validate every host-supplied region before the idempotence gate. Invalid + // retries must not inherit the result of an earlier successful request. + if err := awskms.ValidateRegions(req.Credentials.Region, req.KmsKeyArns); err != nil { + return nil, runInitStatusError(err) + } + fingerprint, err := fingerprintKeySource(req) if err != nil { return nil, err } - - // Generate a new signing key. - secretKey, err := crypto.NewSecretKey(alg) + resp, err := s.gate.ensureInitialized(ctx, req, fingerprint) if err != nil { - return nil, status.Error(codes.Internal, "failed to generate secret key") + return nil, runInitStatusError(err) } + return resp, nil +} - secretKeyBytes, err := secretKey.Serialize() - if err != nil { - return nil, status.Error(codes.Internal, "failed to serialize secret key") +// toStatusError maps a raw KMS or enclave SDK error to a gRPC status, preserving any +// code the inner call already chose. Context cancellation/deadline take +// precedence so client disconnects are not mis-classified as Internal; +// recognized KMS errors are mapped via awskms.StatusFromError; errors that +// already carry a gRPC status are passed through unchanged; everything else +// becomes Internal. The original message is preserved so operators can see +// what went wrong (host and enclave are the same trust boundary). +// +// A non-empty prefix is prepended to the mapped KMS status and the Internal +// fallback for call-site context (e.g. "kms decrypt data key"). Context codes +// are left unprefixed since they are self-explanatory. +func toStatusError(prefix string, err error) error { + switch { + case errors.Is(err, context.Canceled): + return status.Error(codes.Canceled, err.Error()) + case errors.Is(err, context.DeadlineExceeded): + return status.Error(codes.DeadlineExceeded, err.Error()) + } + // An invalid KMS ARN or credentials region is client input. Classify these + // errors as InvalidArgument instead of using the Internal fallback. + if errors.Is(err, awskms.ErrInvalidARN) || errors.Is(err, awskms.ErrInvalidRegion) { + if prefix == "" { + return status.Error(codes.InvalidArgument, err.Error()) + } + return status.Errorf(codes.InvalidArgument, "%s: %v", prefix, err) } - - // Derive the corresponding public key. - publicKeyBytes, err := secretKey.PublicKey() - if err != nil { - return nil, status.Error(codes.Internal, "failed to get public key") + if mapped := awskms.StatusFromError(err); mapped != err { + if prefix == "" { + return mapped + } + st := status.Convert(mapped) + return status.Errorf(st.Code(), "%s: %s", prefix, st.Message()) } - - // Encrypt private key bytes with AES-GCM using the plaintext data key. - cipherData, nonce, err := aes.EncryptGCM(plainDataKey, secretKeyBytes) - if err != nil { - return nil, status.Error(codes.Internal, "failed to encrypt secret key") + if _, ok := status.FromError(err); ok { + return err } - - // Cache the key for future requests in this process. - err = s.keystore.Set(buildCacheKey(cipherData, plainDataKey, nonce), secretKey) - if err != nil { - return nil, status.Error(codes.Internal, "failed to store secret key") + if prefix == "" { + return status.Error(codes.Internal, err.Error()) } + return status.Errorf(codes.Internal, "%s: %v", prefix, err) +} - return &pb.GenerateKeyResponse{ - EncryptedPrivateKey: cipherData, - Nonce: nonce, - PublicKey: publicKeyBytes, - }, nil +// runInitStatusError maps a runInit error to a gRPC status. It shares +// toStatusError with the envelope path (kmsStatusError) so the Initialize and +// read/generate RPCs surface context/KMS errors identically. runInit is the +// only caller and returns raw (fmt.Errorf-wrapped) errors β€” unlike +// generateEnvelope, which returns straight to its RPC and so maps to a status +// itself via kmsStatusError β€” so this is the single mapping point for the +// Initialize path and no prefix is added here. +func runInitStatusError(err error) error { + return toStatusError("", err) } -// GetPublicKey derives the public key from encrypted key material. -func (s *Service) GetPublicKey(ctx context.Context, req *pb.GetPublicKeyRequest) (*pb.GetPublicKeyResponse, error) { - alg, err := toAlgorithm(req.Algorithm) - if err != nil { +// GenerateKey is disabled. Initialize is the only operation that can install a +// signing key. +func (s *Service) GenerateKey(_ context.Context, req *pb.GenerateKeyRequest) (*pb.GenerateKeyResponse, error) { + if req == nil { + return nil, status.Error(codes.InvalidArgument, "request is nil") + } + if _, err := toAlgorithm(req.Algorithm); err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) } + return nil, status.Error(codes.FailedPrecondition, "key generation is available only through Initialize") +} - secretKey, err := s.resolveSecretKey(ctx, alg, req.EncryptedKeyMaterial) +// GetPublicKey returns and attests the installed Ready-state key. +func (s *Service) GetPublicKey(_ context.Context, _ *pb.GetPublicKeyRequest) (*pb.GetPublicKeyResponse, error) { + secretKey, err := s.activeKey() if err != nil { return nil, err } @@ -126,24 +217,30 @@ func (s *Service) GetPublicKey(ctx context.Context, req *pb.GetPublicKeyRequest) return nil, status.Error(codes.Internal, "failed to get public key") } + var attestationDocument []byte + if s.nitroEnclaveEnabled { + if s.enclavePvd == nil { + return nil, status.Error(codes.Internal, "enclave provider is nil while nitro enclave is enabled") + } + attestationDocument, err = s.enclavePvd.Attest(publicKey) + if err != nil { + return nil, status.Error(codes.Internal, "failed to attest public key") + } + } + return &pb.GetPublicKeyResponse{ - PublicKey: publicKey, + PublicKey: publicKey, + AttestationDocument: attestationDocument, }, nil } -// SignMessage signs the request message with the decrypted secret key. -func (s *Service) SignMessage(ctx context.Context, req *pb.SignMessageRequest) (*pb.SignMessageResponse, error) { - alg, err := toAlgorithm(req.Algorithm) - if err != nil { - return nil, status.Error(codes.InvalidArgument, err.Error()) - } - - secretKey, err := s.resolveSecretKey(ctx, alg, req.EncryptedKeyMaterial) +// SignMessage signs with the installed Ready-state key. +func (s *Service) SignMessage(_ context.Context, req *pb.SignMessageRequest) (*pb.SignMessageResponse, error) { + secretKey, err := s.activeKey() if err != nil { return nil, err } - // Sign with the resolved key. signature, err := secretKey.SignMessage(req.Message) if err != nil { return nil, status.Error(codes.Internal, "failed to sign message") @@ -153,106 +250,3 @@ func (s *Service) SignMessage(ctx context.Context, req *pb.SignMessageRequest) ( Signature: signature, }, nil } - -func (s *Service) resolveSecretKey(ctx context.Context, alg crypto.Algorithm, material *pb.EncryptedKeyMaterial) (crypto.Key, error) { - if material == nil { - return nil, status.Error(codes.InvalidArgument, "encrypted key material is nil") - } - - plainDataKey, err := s.loadDataKey(material.EnclaveEncryptedDataKey) - if err != nil { - return nil, err - } - return s.loadSecretKey(ctx, alg, material.EncryptedPrivateKey, plainDataKey, material.Nonce) -} - -func (s *Service) loadDataKey(enclaveEncryptedDataKey []byte) ([]byte, error) { - // Return cached data key if available. - if dataKey := s.keystore.Get(enclaveEncryptedDataKey); dataKey != nil { - return dataKey.Serialize() - } - - // In Nitro mode, decrypt the enclave-encrypted data key. - plainDataKey := enclaveEncryptedDataKey - if s.nitroEnclaveEnabled { - if s.enclavePvd == nil { - return nil, status.Error(codes.FailedPrecondition, "enclave provider is nil while nitro enclave is enabled") - } - var err error - plainDataKey, err = s.enclavePvd.DecryptKMSEnvelopedKey(enclaveEncryptedDataKey) - if err != nil { - return nil, status.Error(codes.Internal, "failed to decrypt data key") - } - } - - // Validate and deserialize AES key bytes. - dataKey, err := aesCommon.Deserialize(plainDataKey) - if err != nil { - return nil, status.Error(codes.InvalidArgument, "failed to deserialize data key") - } - - // Cache deserialized key material. - if err := s.keystore.Set(enclaveEncryptedDataKey, dataKey); err != nil { - return nil, status.Error(codes.Internal, "failed to cache data key") - } - - return dataKey.Serialize() -} - -func (s *Service) loadSecretKey(_ context.Context, alg crypto.Algorithm, encryptedPrivateKey, dataKey, nonce []byte) (crypto.Key, error) { - // Return cached decrypted secret key if available. - cacheKey := buildCacheKey(encryptedPrivateKey, dataKey, nonce) - if secretKey := s.keystore.Get(cacheKey); secretKey != nil { - return secretKey, nil - } - - // Decrypt private key bytes with AES-GCM. - secretKeyPlaintext, err := aes.DecryptGCM(dataKey, encryptedPrivateKey, nonce) - if err != nil { - return nil, status.Error(codes.InvalidArgument, "failed to decrypt secret key") - } - - // Deserialize private key according to the requested algorithm. - secretKey, err := crypto.DeserializeSecretKey(alg, secretKeyPlaintext) - if err != nil { - return nil, status.Error(codes.InvalidArgument, "failed to deserialize secret key") - } - - // Cache decrypted key for subsequent calls. - err = s.keystore.Set(cacheKey, secretKey) - if err != nil { - return nil, status.Error(codes.Internal, "failed to cache secret key") - } - - return secretKey, nil -} - -// NOTE: The following helper functions (buildCacheKey, toAlgorithm) are duplicated from -// internal/enclave/service/key/key.go. This duplication is temporary - the old HTTP-based -// key service will be removed in a follow-up PR once this gRPC service is fully wired up. - -// buildCacheKey builds a cache key from the given arguments. -// We use a composite key (encryptedPrivateKey + dataKey + nonce) instead of just encryptedPrivateKey -// to ensure the caller possesses both the encrypted private key AND the correct data key. -// This prevents unauthorized signing attempts with only a leaked encryptedPrivateKey. -func buildCacheKey(args ...[]byte) []byte { - hash := sha256.New() - lenBuf := make([]byte, 4) - for _, arg := range args { - binary.BigEndian.PutUint32(lenBuf, uint32(len(arg))) - hash.Write(lenBuf) - hash.Write(arg) - } - return hash.Sum(nil) -} - -func toAlgorithm(algorithm pb.Algorithm) (crypto.Algorithm, error) { - switch algorithm { - case pb.Algorithm_ALGORITHM_BLS: - return crypto.AlgorithmBLS, nil - case pb.Algorithm_ALGORITHM_ED25519: - return crypto.AlgorithmEd25519, nil - default: - return "", fmt.Errorf("invalid algorithm: %s", algorithm) - } -} diff --git a/internal/enclave/service/enclave/enclave_test.go b/internal/enclave/service/enclave/enclave_test.go index 3771f96..03dda20 100644 --- a/internal/enclave/service/enclave/enclave_test.go +++ b/internal/enclave/service/enclave/enclave_test.go @@ -15,83 +15,46 @@ package enclave import ( "context" - "fmt" + "errors" + "sync" + "sync/atomic" "testing" + "time" - commonAES "github.com/circlefin/arc-remote-signer/internal/common/crypto/aes" - "github.com/circlefin/arc-remote-signer/internal/common/crypto/rand" + "github.com/aws/smithy-go" enclaveCrypto "github.com/circlefin/arc-remote-signer/internal/enclave/common/crypto" - aesCrypto "github.com/circlefin/arc-remote-signer/internal/enclave/common/crypto/aes" + "github.com/circlefin/arc-remote-signer/internal/enclave/provider/awskms" enclavePvd "github.com/circlefin/arc-remote-signer/internal/enclave/provider/enclave" - "github.com/circlefin/arc-remote-signer/internal/enclave/provider/keystore" "github.com/circlefin/arc-remote-signer/proto/pb" "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) +// EnclaveServiceTestSuite covers request validation, the Initialize gate, and +// the pure helpers. The SecretEnvelope generate/read data paths (KMS decrypt, +// caching, Nitro in-process RSA decrypt) are covered in envelope_test.go against the +// fakeKmsProvider seam. type EnclaveServiceTestSuite struct { suite.Suite - ctrl *gomock.Controller - keystore *keystore.MockProvider - enclave *enclavePvd.MockProvider - service *Service - neService *Service - - dataKey []byte - secretKey enclaveCrypto.Key - encryptedKey []byte - nonce []byte - secretCacheKey []byte - encryptedKeyMtrl *pb.EncryptedKeyMaterial + ctrl *gomock.Controller + enclave *enclavePvd.MockProvider + service *Service } func (s *EnclaveServiceTestSuite) SetupTest() { s.ctrl = gomock.NewController(s.T()) - s.keystore = keystore.NewMockProvider(s.ctrl) s.enclave = enclavePvd.NewMockProvider(s.ctrl) - s.service = New(false, s.keystore, s.enclave) - s.neService = New(true, s.keystore, s.enclave) - - var err error - s.dataKey = rand.MustGenerateRandomBytes(32) - s.secretKey, err = enclaveCrypto.NewSecretKey(enclaveCrypto.AlgorithmEd25519) - s.Require().NoError(err) - - secretKeyBytes, err := s.secretKey.Serialize() - s.Require().NoError(err) - s.encryptedKey, s.nonce, err = commonAES.EncryptGCM(s.dataKey, secretKeyBytes) - s.Require().NoError(err) - - s.secretCacheKey = buildCacheKey(s.encryptedKey, s.dataKey, s.nonce) - s.encryptedKeyMtrl = &pb.EncryptedKeyMaterial{ - EncryptedPrivateKey: s.encryptedKey, - EnclaveEncryptedDataKey: s.dataKey, - Nonce: s.nonce, - } + s.service = New(false, s.enclave, nil, "") } func (s *EnclaveServiceTestSuite) TearDownTest() { s.ctrl.Finish() } -func (s *EnclaveServiceTestSuite) TestGetAttestationDocument() { - s.enclave.EXPECT().AttestationDocument().Return([]byte("doc")) - - resp, err := s.service.GetAttestation(context.Background(), &pb.GetAttestationRequest{}) - s.NoError(err) - s.NotNil(resp) - s.Equal([]byte("doc"), resp.AttestationDocument) - - withoutEnclave := New(false, s.keystore, nil) - resp, err = withoutEnclave.GetAttestation(context.Background(), &pb.GetAttestationRequest{}) - s.Error(err) - s.Equal(codes.FailedPrecondition, status.Code(err)) - s.Nil(resp) -} - func (s *EnclaveServiceTestSuite) TestGenerateKey_NilRequest() { resp, err := s.service.GenerateKey(context.Background(), nil) @@ -111,337 +74,457 @@ func (s *EnclaveServiceTestSuite) TestGenerateKey_InvalidAlgorithm() { s.Equal(codes.InvalidArgument, status.Code(err)) } -func (s *EnclaveServiceTestSuite) TestGenerateKey_Success() { - s.keystore.EXPECT().Get(s.dataKey).Return(nil) - s.keystore.EXPECT().Set(s.dataKey, gomock.Any()).Return(nil) - s.keystore.EXPECT().Set(gomock.Any(), gomock.Any()).Return(nil) - - resp, err := s.service.GenerateKey(context.Background(), &pb.GenerateKeyRequest{ - Algorithm: pb.Algorithm_ALGORITHM_ED25519, - EnclaveEncryptedDataKey: s.dataKey, - }) +func (s *EnclaveServiceTestSuite) TestToAlgorithm() { + tests := []struct { + name string + input pb.Algorithm + wantAlg enclaveCrypto.Algorithm + wantError bool + }{ + {"bls maps correctly", pb.Algorithm_ALGORITHM_BLS, enclaveCrypto.AlgorithmBLS, false}, + {"ed25519 maps correctly", pb.Algorithm_ALGORITHM_ED25519, enclaveCrypto.AlgorithmEd25519, false}, + {"unspecified returns error", pb.Algorithm_ALGORITHM_UNSPECIFIED, enclaveCrypto.Algorithm(""), true}, + } - s.NoError(err) - s.NotNil(resp) - s.NotEmpty(resp.PublicKey) - s.NotEmpty(resp.EncryptedPrivateKey) - s.NotEmpty(resp.Nonce) + for _, tt := range tests { + s.Run(tt.name, func() { + alg, err := toAlgorithm(tt.input) + if tt.wantError { + s.Error(err) + } else { + s.NoError(err) + } + s.Equal(tt.wantAlg, alg) + }) + } } -func (s *EnclaveServiceTestSuite) TestGenerateKey_LoadDataKeyErrorWithNitro() { - s.keystore.EXPECT().Get(s.dataKey).Return(nil) - s.enclave.EXPECT().DecryptKMSEnvelopedKey(s.dataKey).Return(nil, fmt.Errorf("decrypt failed")) - - resp, err := s.neService.GenerateKey(context.Background(), &pb.GenerateKeyRequest{ - Algorithm: pb.Algorithm_ALGORITHM_ED25519, - EnclaveEncryptedDataKey: s.dataKey, - }) - - s.Nil(resp) - s.Error(err) - s.Equal(codes.Internal, status.Code(err)) - s.ErrorContains(err, "failed to decrypt data key") +// validInitReq is the request shape Initialize tests use when they need a +// well-formed payload. The credentials values are irrelevant β€” the +// runInit callbacks injected by these tests do not actually call AWS. +// KmsKeyArn must parse via aws-sdk arn.Parse to clear Service.Initialize's +// sanity check; the value is a syntactically valid ARN, never dialed. +func (s *EnclaveServiceTestSuite) validInitReq() *pb.InitializeRequest { + s.T().Helper() + return &pb.InitializeRequest{ + Credentials: &pb.AwsCredentials{ + AccessKeyId: "AKIA-test", + SecretAccessKey: "secret", + SessionToken: "session", + Region: "us-east-1", + }, + KmsKeyArns: []string{"arn:aws:kms:us-east-1:000000000000:key/00000000-0000-0000-0000-000000000000"}, + KeySource: &pb.InitializeRequest_GenerateNew{ + GenerateNew: pb.Algorithm_ALGORITHM_ED25519, + }, + } } -func (s *EnclaveServiceTestSuite) TestGetPublicKey_InvalidAlgorithm() { - resp, err := s.service.GetPublicKey(context.Background(), &pb.GetPublicKeyRequest{ - Algorithm: pb.Algorithm_ALGORITHM_UNSPECIFIED, - }) - - s.Nil(resp) - s.Error(err) - s.Equal(codes.InvalidArgument, status.Code(err)) +// newUninitializedService builds a fresh Service with the given runInit +// callback. Init-gate tests use this so they can inject failing or +// instrumented callbacks; the suite-level s.service is sufficient for +// tests that do not care about the gate's runInit behaviour. +func (s *EnclaveServiceTestSuite) newUninitializedService( + runInit func(context.Context, *pb.InitializeRequest) error, +) *Service { + s.T().Helper() + return newWithGate(false, s.enclave, runInit) } -func (s *EnclaveServiceTestSuite) TestGetPublicKey_SuccessFromCache() { - dataKeyObj, err := aesCrypto.Deserialize(s.dataKey) - s.Require().NoError(err) - - s.keystore.EXPECT().Get(s.dataKey).Return(dataKeyObj) - s.keystore.EXPECT().Get(s.secretCacheKey).Return(s.secretKey) - - resp, err := s.service.GetPublicKey(context.Background(), &pb.GetPublicKeyRequest{ - Algorithm: pb.Algorithm_ALGORITHM_ED25519, - EncryptedKeyMaterial: s.encryptedKeyMtrl, +// TestInitialize_RequestValidation covers the synchronous validation +// errors that fire before the init gate runs. +func (s *EnclaveServiceTestSuite) TestInitialize_RequestValidation() { + svc := s.newUninitializedService(func(context.Context, *pb.InitializeRequest) error { + s.FailNow("runInit must not be called when request validation fails") + return nil }) - s.NoError(err) - s.NotNil(resp) - s.NotEmpty(resp.PublicKey) -} - -func (s *EnclaveServiceTestSuite) TestGetPublicKey_ResolveSecretKeyError() { - s.keystore.EXPECT().Get(s.dataKey).Return(nil) - s.keystore.EXPECT().Set(s.dataKey, gomock.Any()).Return(fmt.Errorf("set failed")) - - resp, err := s.service.GetPublicKey(context.Background(), &pb.GetPublicKeyRequest{ - Algorithm: pb.Algorithm_ALGORITHM_ED25519, - EncryptedKeyMaterial: s.encryptedKeyMtrl, - }) + validCreds := &pb.AwsCredentials{ + AccessKeyId: "AKIA-test", + SecretAccessKey: "secret", + SessionToken: "session", + Region: "us-east-1", + } - s.Nil(resp) - s.Error(err) - s.ErrorContains(err, "failed to cache data key") + tests := map[string]struct { + req *pb.InitializeRequest + wantCode codes.Code + }{ + "NilReq": {req: nil, wantCode: codes.InvalidArgument}, + "NilCredentials": {req: &pb.InitializeRequest{Credentials: nil}, wantCode: codes.InvalidArgument}, + "NilArns": { + // Defense-in-depth against in-process callers bypassing protovalidate + // (which enforces min_items=1 at the wire boundary). + req: &pb.InitializeRequest{Credentials: validCreds, KmsKeyArns: nil}, + wantCode: codes.InvalidArgument, + }, + "EmptyArnList": { + req: &pb.InitializeRequest{Credentials: validCreds, KmsKeyArns: []string{}}, + wantCode: codes.InvalidArgument, + }, + // MalformedArn covers the reviewer-flagged path (PR #299 review + // thread): arn.Parse rejects "not-an-arn", handler short-circuits + // with InvalidArgument rather than falling through to the KMS + // factory and surfacing as Internal. + "MalformedArn": { + req: &pb.InitializeRequest{Credentials: validCreds, KmsKeyArns: []string{"not-an-arn"}}, + wantCode: codes.InvalidArgument, + }, + "EmptyArnInList": { + req: &pb.InitializeRequest{Credentials: validCreds, KmsKeyArns: []string{""}}, + wantCode: codes.InvalidArgument, + }, + "SecondArnMalformed": { + // First ARN parses fine, second fails β€” loop must check every entry. + req: &pb.InitializeRequest{ + Credentials: validCreds, + KmsKeyArns: []string{ + "arn:aws:kms:us-east-1:000000000000:key/abc", + "not-an-arn", + }, + }, + wantCode: codes.InvalidArgument, + }, + } + for name, tt := range tests { + s.Run(name, func() { + resp, err := svc.Initialize(context.Background(), tt.req) + s.Require().Error(err) + s.Nil(resp) + st, ok := status.FromError(err) + s.Require().True(ok) + s.Equal(tt.wantCode, st.Code()) + s.False(svc.gate.initialized(), "gate must not latch when validation fails") + }) + } } -func (s *EnclaveServiceTestSuite) TestGetPublicKey_NilEncryptedKeyMaterial() { - resp, err := s.service.GetPublicKey(context.Background(), &pb.GetPublicKeyRequest{ - Algorithm: pb.Algorithm_ALGORITHM_ED25519, - EncryptedKeyMaterial: nil, +// TestInitialize_HappyPath verifies a single successful Initialize call +// latches the gate and returns an empty response. +func (s *EnclaveServiceTestSuite) TestInitialize_HappyPath() { + var calls atomic.Int32 + svc := s.newUninitializedService(func(context.Context, *pb.InitializeRequest) error { + calls.Add(1) + return nil }) - s.Nil(resp) - s.Equal(codes.InvalidArgument, status.Code(err)) - s.ErrorContains(err, "encrypted key material is nil") + resp, err := svc.Initialize(context.Background(), s.validInitReq()) + s.Require().NoError(err) + s.NotNil(resp) + s.True(svc.gate.initialized(), "gate must latch after successful Initialize") + s.Equal(int32(1), calls.Load(), "runInit must run exactly once") } -func (s *EnclaveServiceTestSuite) TestSignMessage_NilEncryptedKeyMaterial() { - resp, err := s.service.SignMessage(context.Background(), &pb.SignMessageRequest{ - Algorithm: pb.Algorithm_ALGORITHM_ED25519, - EncryptedKeyMaterial: nil, - Message: []byte("x"), +func (s *EnclaveServiceTestSuite) TestInitialize_NitroRejectsLocalstack() { + var called atomic.Bool + svc := newWithGate(true, s.enclave, func(context.Context, *pb.InitializeRequest) error { + called.Store(true) + return nil }) + req := s.validInitReq() + req.KmsLocalstackEnabled = true + + resp, err := svc.Initialize(context.Background(), req) s.Nil(resp) s.Equal(codes.InvalidArgument, status.Code(err)) - s.ErrorContains(err, "encrypted key material is nil") + s.False(called.Load(), "runInit must not run for LocalStack in Nitro mode") } -func (s *EnclaveServiceTestSuite) TestSignMessage_InvalidAlgorithm() { - resp, err := s.service.SignMessage(context.Background(), &pb.SignMessageRequest{ - Algorithm: pb.Algorithm_ALGORITHM_UNSPECIFIED, - Message: []byte("msg"), +// TestInitialize_MultipleArnsAllValid verifies the ARN-validation loop +// clears a multi-region ARN slice (every entry parses) and the request +// flows through to runInit unchanged. +func (s *EnclaveServiceTestSuite) TestInitialize_MultipleArnsAllValid() { + var capturedArns []string + svc := s.newUninitializedService(func(_ context.Context, req *pb.InitializeRequest) error { + capturedArns = req.KmsKeyArns + return nil }) - s.Nil(resp) - s.Error(err) - s.Equal(codes.InvalidArgument, status.Code(err)) -} + req := s.validInitReq() + req.KmsKeyArns = []string{ + "arn:aws:kms:us-east-1:000000000000:key/abc", + "arn:aws:kms:us-west-2:000000000000:key/def", + } -func (s *EnclaveServiceTestSuite) TestSignMessage_SuccessFromCache() { - dataKeyObj, err := aesCrypto.Deserialize(s.dataKey) + resp, err := svc.Initialize(context.Background(), req) s.Require().NoError(err) - - message := []byte("payload") - s.keystore.EXPECT().Get(s.dataKey).Return(dataKeyObj) - s.keystore.EXPECT().Get(s.secretCacheKey).Return(s.secretKey) - - resp, err := s.service.SignMessage(context.Background(), &pb.SignMessageRequest{ - Algorithm: pb.Algorithm_ALGORITHM_ED25519, - EncryptedKeyMaterial: s.encryptedKeyMtrl, - Message: message, - }) - - s.NoError(err) s.NotNil(resp) - s.NotEmpty(resp.Signature) + s.Equal(req.KmsKeyArns, capturedArns, "runInit must observe the full ARN slice unchanged") } -func (s *EnclaveServiceTestSuite) TestNilEnclavePvdWithNitroEnabled() { - svc := New(true, s.keystore, nil) - s.keystore.EXPECT().Get(s.dataKey).Return(nil).Times(3) - - s.Run("generate key returns failed precondition", func() { - resp, err := svc.GenerateKey(context.Background(), &pb.GenerateKeyRequest{ - Algorithm: pb.Algorithm_ALGORITHM_ED25519, - EnclaveEncryptedDataKey: s.dataKey, - }) - - s.Nil(resp) - s.Error(err) - s.Equal(codes.FailedPrecondition, status.Code(err)) - s.Contains(err.Error(), "enclave provider is nil while nitro enclave is enabled") +// TestInitialize_Idempotent verifies a repeated Initialize call after a +// successful one returns OK without invoking runInit again. +func (s *EnclaveServiceTestSuite) TestInitialize_Idempotent() { + var calls atomic.Int32 + svc := s.newUninitializedService(func(context.Context, *pb.InitializeRequest) error { + calls.Add(1) + return nil }) - s.Run("get public key returns failed precondition", func() { - resp, err := svc.GetPublicKey(context.Background(), &pb.GetPublicKeyRequest{ - Algorithm: pb.Algorithm_ALGORITHM_ED25519, - EncryptedKeyMaterial: s.encryptedKeyMtrl, - }) - - s.Nil(resp) - s.Error(err) - s.Equal(codes.FailedPrecondition, status.Code(err)) - s.Contains(err.Error(), "enclave provider is nil while nitro enclave is enabled") - }) - - s.Run("sign message returns failed precondition", func() { - resp, err := svc.SignMessage(context.Background(), &pb.SignMessageRequest{ - Algorithm: pb.Algorithm_ALGORITHM_ED25519, - EncryptedKeyMaterial: s.encryptedKeyMtrl, - Message: []byte("payload"), - }) + for i := 0; i < 3; i++ { + resp, err := svc.Initialize(context.Background(), s.validInitReq()) + s.Require().NoError(err, "call %d", i) + s.NotNil(resp) + } + s.Equal(int32(1), calls.Load(), "runInit must run exactly once across repeated Initialize calls") +} - s.Nil(resp) - s.Error(err) - s.Equal(codes.FailedPrecondition, status.Code(err)) - s.Contains(err.Error(), "enclave provider is nil while nitro enclave is enabled") +// TestInitialize_Concurrent verifies many simultaneous Initialize calls +// collapse onto a single runInit and all callers observe success. This +// exercises both layers of the gate's exactly-once guarantee: the +// singleflight collapse for callers that pile up while the leader is +// still running, and the inner g.done.Load() re-check for callers that +// arrive *after* a prior singleflight batch has completed and its key +// has been deleted from the singleflight map. Without the inner check, +// such late callers would enter a fresh sf.Do entry and re-invoke +// runInit; the assertion `runInit must run exactly once` would fail +// under unfavourable scheduling. +func (s *EnclaveServiceTestSuite) TestInitialize_Concurrent() { + const goroutines = 50 + + var calls atomic.Int32 + start := make(chan struct{}) + svc := s.newUninitializedService(func(context.Context, *pb.InitializeRequest) error { + calls.Add(1) + // Hold the work briefly so concurrent callers genuinely race + // against the singleflight entry rather than serialising on + // the atomic.Bool fast-path. + <-start + return nil }) -} -func (s *EnclaveServiceTestSuite) TestGenerateKey_NitroEnabledSuccess() { - s.keystore.EXPECT().Get(s.dataKey).Return(nil) - s.enclave.EXPECT().DecryptKMSEnvelopedKey(s.dataKey).Return(s.dataKey, nil) - s.keystore.EXPECT().Set(s.dataKey, gomock.Any()).Return(nil) - s.keystore.EXPECT().Set(gomock.Any(), gomock.Any()).Return(nil) + var wg sync.WaitGroup + errs := make([]error, goroutines) + for i := 0; i < goroutines; i++ { + wg.Add(1) + go func(idx int) { + defer wg.Done() + _, errs[idx] = svc.Initialize(context.Background(), s.validInitReq()) + }(i) + } - resp, err := s.neService.GenerateKey(context.Background(), &pb.GenerateKeyRequest{ - Algorithm: pb.Algorithm_ALGORITHM_ED25519, - EnclaveEncryptedDataKey: s.dataKey, - }) + close(start) + wg.Wait() - s.NoError(err) - s.NotNil(resp) - s.NotEmpty(resp.PublicKey) - s.NotEmpty(resp.EncryptedPrivateKey) - s.NotEmpty(resp.Nonce) + for i, err := range errs { + s.Require().NoError(err, "goroutine %d", i) + } + s.True(svc.gate.initialized()) + s.Equal(int32(1), calls.Load(), "runInit must run exactly once under concurrent Initialize") } -func (s *EnclaveServiceTestSuite) TestGetPublicKey_NitroEnabledSuccess() { - s.keystore.EXPECT().Get(s.dataKey).Return(nil) - s.enclave.EXPECT().DecryptKMSEnvelopedKey(s.dataKey).Return(s.dataKey, nil) - s.keystore.EXPECT().Set(s.dataKey, gomock.Any()).Return(nil) - s.keystore.EXPECT().Get(s.secretCacheKey).Return(nil) - s.keystore.EXPECT().Set(s.secretCacheKey, gomock.Any()).Return(nil) - - resp, err := s.neService.GetPublicKey(context.Background(), &pb.GetPublicKeyRequest{ - Algorithm: pb.Algorithm_ALGORITHM_ED25519, - EncryptedKeyMaterial: s.encryptedKeyMtrl, +// TestInitialize_FailureAllowsRetry verifies a failed Initialize does not +// latch the gate, and the next caller's request runs runInit again. +func (s *EnclaveServiceTestSuite) TestInitialize_FailureAllowsRetry() { + var calls atomic.Int32 + wantErr := errors.New("simulated KMS failure") + svc := s.newUninitializedService(func(context.Context, *pb.InitializeRequest) error { + calls.Add(1) + if calls.Load() == 1 { + return wantErr + } + return nil }) - s.NoError(err) + // First Initialize fails β€” gate stays unlatched. + resp, err := svc.Initialize(context.Background(), s.validInitReq()) + s.Require().Error(err) + s.Nil(resp) + st, ok := status.FromError(err) + s.Require().True(ok) + s.Equal(codes.Internal, st.Code()) + s.Contains(st.Message(), "simulated KMS failure") + s.False(svc.gate.initialized(), "gate must not latch on runInit failure") + + // Second Initialize succeeds β€” gate now latches. + resp, err = svc.Initialize(context.Background(), s.validInitReq()) + s.Require().NoError(err) s.NotNil(resp) - s.NotEmpty(resp.PublicKey) + s.True(svc.gate.initialized()) + s.Equal(int32(2), calls.Load(), "runInit must run twice: once failing, once succeeding") } -func (s *EnclaveServiceTestSuite) TestSignMessage_NitroEnabledSuccess() { - s.keystore.EXPECT().Get(s.dataKey).Return(nil) - s.enclave.EXPECT().DecryptKMSEnvelopedKey(s.dataKey).Return(s.dataKey, nil) - s.keystore.EXPECT().Set(s.dataKey, gomock.Any()).Return(nil) - s.keystore.EXPECT().Get(s.secretCacheKey).Return(nil) - s.keystore.EXPECT().Set(s.secretCacheKey, gomock.Any()).Return(nil) - - message := []byte("payload") - resp, err := s.neService.SignMessage(context.Background(), &pb.SignMessageRequest{ - Algorithm: pb.Algorithm_ALGORITHM_ED25519, - EncryptedKeyMaterial: s.encryptedKeyMtrl, - Message: message, +func (s *EnclaveServiceTestSuite) TestInitialize_InvalidCredentialsRegionReturnsInvalidArgument() { + svc := s.newUninitializedService(func(_ context.Context, req *pb.InitializeRequest) error { + _, err := awskms.BuildConfig(req.Credentials, "http://127.0.0.1:9000", false) + return err }) + req := s.validInitReq() + req.Credentials.Region = "us@east-1" - s.NoError(err) - s.NotNil(resp) - s.NotEmpty(resp.Signature) -} - -func (s *EnclaveServiceTestSuite) TestLoadDataKey_CacheSetError() { - s.keystore.EXPECT().Get(s.dataKey).Return(nil) - s.keystore.EXPECT().Set(s.dataKey, gomock.Any()).Return(fmt.Errorf("set failed")) + resp, err := svc.Initialize(context.Background(), req) - plainKey, err := s.service.loadDataKey(s.dataKey) - s.Nil(plainKey) - s.Error(err) - s.ErrorContains(err, "failed to cache data key") + s.Require().Error(err) + s.Nil(resp) + s.Equal(codes.InvalidArgument, status.Code(err)) + s.False(svc.gate.initialized()) } -func (s *EnclaveServiceTestSuite) TestLoadDataKey_DeserializeError() { - invalidKey := rand.MustGenerateRandomBytes(16) - s.keystore.EXPECT().Get(invalidKey).Return(nil) +func (s *EnclaveServiceTestSuite) TestInitialize_RejectsUnsupportedRegionsAfterInitialization() { + tests := map[string]func(*pb.InitializeRequest){ + "credentials region": func(req *pb.InitializeRequest) { + req.Credentials.Region = "s3" + }, + "ARN region": func(req *pb.InitializeRequest) { + req.KmsKeyArns = []string{"arn:aws:kms:s3:123456789012:key/abc"} + }, + } - plainKey, err := s.service.loadDataKey(invalidKey) - s.Nil(plainKey) - s.Error(err) - s.Equal(codes.InvalidArgument, status.Code(err)) - s.ErrorContains(err, "failed to deserialize data key") + for name, mutate := range tests { + s.Run(name, func() { + var calls atomic.Int32 + svc := s.newUninitializedService(func(context.Context, *pb.InitializeRequest) error { + calls.Add(1) + return nil + }) + + resp, err := svc.Initialize(context.Background(), s.validInitReq()) + s.Require().NoError(err) + s.NotNil(resp) + + invalidReq := s.validInitReq() + mutate(invalidReq) + resp, err = svc.Initialize(context.Background(), invalidReq) + + s.Require().Error(err) + s.Nil(resp) + s.Equal(codes.InvalidArgument, status.Code(err)) + s.Equal(int32(1), calls.Load(), "invalid retry must be rejected before the init gate") + }) + } } -func (s *EnclaveServiceTestSuite) TestLoadSecretKey() { - s.Run("DecryptError", func() { - invalidEncrypted := rand.MustGenerateRandomBytes(32) - invalidNonce := rand.MustGenerateRandomBytes(12) - expectedCacheKey := buildCacheKey(invalidEncrypted, s.dataKey, invalidNonce) +// TestInitialize_LeaderContextDetached verifies that caller cancellation does +// not cancel the shared initialization operation. The gate timeout continues +// to bound the operation. +func (s *EnclaveServiceTestSuite) TestInitialize_LeaderContextDetached() { + runInitStarted := make(chan struct{}) + allowReturn := make(chan struct{}) + runInitReturned := make(chan struct{}) + var runInitCtxErr atomic.Value // error observed by runInit at return + svc := s.newUninitializedService(func(ctx context.Context, _ *pb.InitializeRequest) error { + close(runInitStarted) + <-allowReturn + runInitCtxErr.Store(errOrNil(ctx.Err())) + close(runInitReturned) + return nil + }) - s.keystore.EXPECT().Get(expectedCacheKey).Return(nil) + leaderCtx, cancelLeader := context.WithCancel(context.Background()) + + var wg sync.WaitGroup + wg.Add(1) + var leaderErr error + go func() { + defer wg.Done() + _, leaderErr = svc.Initialize(leaderCtx, s.validInitReq()) + }() + + <-runInitStarted // wait until runInit is inside the closure + cancelLeader() // simulate leader's client disconnecting + close(allowReturn) // let runInit finish + <-runInitReturned + + wg.Wait() + + // runInit must have observed a non-cancelled ctx at the point of return, + // proving the detach worked. + s.Equal(errNil{}, runInitCtxErr.Load(), + "runInit must see a non-cancelled context even after leader cancel") + + s.Equal(codes.Canceled, status.Code(leaderErr), + "caller cancellation must stop only that caller's wait") + _, retryErr := svc.Initialize(context.Background(), s.validInitReq()) + s.Require().NoError(retryErr, + "the shared initialization must still complete for a later caller") + s.True(svc.gate.initialized(), + "gate must latch when runInit returns nil even if leader was cancelled") +} - key, err := s.service.loadSecretKey(context.Background(), enclaveCrypto.AlgorithmEd25519, invalidEncrypted, s.dataKey, invalidNonce) - s.Nil(key) - s.Error(err) - s.ErrorContains(err, "failed to decrypt secret key") +func (s *EnclaveServiceTestSuite) TestInitialize_LeaderDeadlineDoesNotCancelSharedOperation() { + runInitStarted := make(chan struct{}) + allowReturn := make(chan struct{}) + svc := s.newUninitializedService(func(ctx context.Context, _ *pb.InitializeRequest) error { + close(runInitStarted) + <-allowReturn + return ctx.Err() }) - s.Run("DeserializeError", func() { - // Encrypt garbage (10 bytes) β€” valid AES ciphertext, invalid Ed25519 key size. - garbageKey := rand.MustGenerateRandomBytes(10) - encryptedGarbage, nonce, err := commonAES.EncryptGCM(s.dataKey, garbageKey) - s.Require().NoError(err) - expectedCacheKey := buildCacheKey(encryptedGarbage, s.dataKey, nonce) - - s.keystore.EXPECT().Get(expectedCacheKey).Return(nil) - - key, err := s.service.loadSecretKey( - context.Background(), - enclaveCrypto.AlgorithmEd25519, - encryptedGarbage, - s.dataKey, - nonce, - ) - - s.Nil(key) - s.Require().Error(err) - s.Equal(codes.InvalidArgument, status.Code(err)) - s.ErrorContains(err, "failed to deserialize secret key") - }) + expiredCtx, cancel := context.WithDeadline(context.Background(), time.Now().Add(-time.Second)) + defer cancel() + leaderDone := make(chan error, 1) + go func() { + _, err := svc.Initialize(expiredCtx, s.validInitReq()) + leaderDone <- err + }() + <-runInitStarted + + waiterDone := make(chan error, 1) + go func() { + _, err := svc.Initialize(context.Background(), s.validInitReq()) + waiterDone <- err + }() + + close(allowReturn) + + s.Equal(codes.DeadlineExceeded, status.Code(<-leaderDone)) + s.Require().NoError(<-waiterDone) + s.True(svc.gate.initialized()) +} - s.Run("CacheSetError", func() { - s.keystore.EXPECT().Get(s.secretCacheKey).Return(nil) - s.keystore.EXPECT().Set(s.secretCacheKey, gomock.Any()).Return(fmt.Errorf("set failed")) - - key, err := s.service.loadSecretKey( - context.Background(), - enclaveCrypto.AlgorithmEd25519, - s.encryptedKey, - s.dataKey, - s.nonce, - ) - - s.Nil(key) - s.Require().Error(err) - s.Equal(codes.Internal, status.Code(err)) - s.ErrorContains(err, "failed to cache secret key") +func (s *EnclaveServiceTestSuite) TestInitialize_CanceledWaiterDoesNotBlockSharedOperation() { + runInitStarted := make(chan struct{}) + allowReturn := make(chan struct{}) + svc := s.newUninitializedService(func(context.Context, *pb.InitializeRequest) error { + close(runInitStarted) + <-allowReturn + return nil }) -} -func (s *EnclaveServiceTestSuite) TestBuildCacheKey_DeterministicAndUnique() { - k1 := buildCacheKey([]byte("a"), []byte("b")) - k2 := buildCacheKey([]byte("a"), []byte("b")) - k3 := buildCacheKey([]byte("ab")) + leaderDone := make(chan error, 1) + go func() { + _, err := svc.Initialize(context.Background(), s.validInitReq()) + leaderDone <- err + }() + <-runInitStarted + + waiterCtx, cancelWaiter := context.WithCancel(context.Background()) + cancelWaiter() + waiterDone := make(chan error, 1) + go func() { + _, err := svc.Initialize(waiterCtx, s.validInitReq()) + waiterDone <- err + }() + + s.Equal(codes.Canceled, status.Code(<-waiterDone)) + select { + case err := <-leaderDone: + s.T().Fatalf("shared initialization returned before release: %v", err) + default: + } - s.Equal(k1, k2) - s.NotEqual(k1, k3) + close(allowReturn) + s.Require().NoError(<-leaderDone) } -func (s *EnclaveServiceTestSuite) TestToAlgorithm() { - tests := []struct { - name string - input pb.Algorithm - wantAlg enclaveCrypto.Algorithm - wantError bool - }{ - {"bls maps correctly", pb.Algorithm_ALGORITHM_BLS, enclaveCrypto.AlgorithmBLS, false}, - {"ed25519 maps correctly", pb.Algorithm_ALGORITHM_ED25519, enclaveCrypto.AlgorithmEd25519, false}, - {"unspecified returns error", pb.Algorithm_ALGORITHM_UNSPECIFIED, enclaveCrypto.Algorithm(""), true}, - } +// errOrNil and errNil let atomic.Value store a typed "no error" sentinel, +// since atomic.Value rejects nil interface values. +type errNil struct{} - for _, tt := range tests { - s.Run(tt.name, func() { - alg, err := toAlgorithm(tt.input) - if tt.wantError { - s.Error(err) - } else { - s.NoError(err) - } - s.Equal(tt.wantAlg, alg) - }) +func errOrNil(err error) any { + if err == nil { + return errNil{} } + return err +} + +func TestRunInitStatusError_KmsBranch(t *testing.T) { + kmsErr := &smithy.GenericAPIError{Code: "AccessDeniedException", Message: "no kms"} + got := runInitStatusError(kmsErr) + st, ok := status.FromError(got) + require.True(t, ok) + require.Equal(t, codes.PermissionDenied, st.Code()) } func TestEnclaveServiceTestSuite(t *testing.T) { diff --git a/internal/enclave/service/enclave/envelope.go b/internal/enclave/service/enclave/envelope.go new file mode 100644 index 0000000..c571ec9 --- /dev/null +++ b/internal/enclave/service/enclave/envelope.go @@ -0,0 +1,217 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enclave + +import ( + "context" + "crypto/sha256" + "encoding/binary" + "fmt" + + "github.com/circlefin/arc-remote-signer/internal/common/crypto/aes" + "github.com/circlefin/arc-remote-signer/internal/enclave/common/crypto" + aesCommon "github.com/circlefin/arc-remote-signer/internal/enclave/common/crypto/aes" + "github.com/circlefin/arc-remote-signer/internal/enclave/provider/awskms" + "github.com/circlefin/arc-remote-signer/proto/pb" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// kmsStatusError maps a raw KMS or enclave SDK error from the envelope read/generate +// path to a gRPC status, prefixed with the call site (e.g. "kms decrypt data +// key") for context. It delegates to toStatusError so this path surfaces the +// same context-cancel/deadline and awskms.StatusFromError mapping as the +// Initialize path (runInitStatusError). +func kmsStatusError(prefix string, err error) error { + return toStatusError(prefix, err) +} + +func fingerprintKeySource(req *pb.InitializeRequest) (keySourceFingerprint, error) { + var parts [][]byte + switch source := req.GetKeySource().(type) { + case *pb.InitializeRequest_ExistingKey: + env := source.ExistingKey + if env == nil || + len(env.GetKmsEncryptedDataKey()) == 0 || + len(env.GetEncryptedPrivateKey()) == 0 || + len(env.GetNonce()) == 0 { + return keySourceFingerprint{}, status.Error(codes.InvalidArgument, "existing_key is incomplete") + } + if _, err := toAlgorithm(env.GetAlgorithm()); err != nil { + return keySourceFingerprint{}, status.Error(codes.InvalidArgument, err.Error()) + } + parts = [][]byte{ + []byte("initialize-existing-v1"), + {byte(env.GetAlgorithm())}, + env.GetKmsEncryptedDataKey(), + env.GetEncryptedPrivateKey(), + env.GetNonce(), + } + case *pb.InitializeRequest_GenerateNew: + if _, err := toAlgorithm(source.GenerateNew); err != nil { + return keySourceFingerprint{}, status.Error(codes.InvalidArgument, err.Error()) + } + algorithm := make([]byte, 4) + binary.BigEndian.PutUint32(algorithm, uint32(source.GenerateNew)) + parts = [][]byte{[]byte("initialize-generate-v1"), algorithm} + default: + return keySourceFingerprint{}, status.Error(codes.InvalidArgument, "key_source is required") + } + + var fingerprint keySourceFingerprint + copy(fingerprint[:], hashKeySourceFields(parts...)) + return fingerprint, nil +} + +func (s *Service) decryptDataKey( + ctx context.Context, + pvd awskms.Provider, + kmsEncryptedDataKey []byte, +) ([]byte, error) { + if pvd == nil { + return nil, status.Error(codes.FailedPrecondition, "kms provider not initialized") + } + plaintext, ciphertextForRecipient, err := pvd.Decrypt(ctx, kmsEncryptedDataKey) + if err != nil { + return nil, kmsStatusError("kms decrypt data key", err) + } + if s.nitroEnclaveEnabled { + if s.enclavePvd == nil { + return nil, status.Error(codes.FailedPrecondition, "enclave provider is nil while nitro enclave is enabled") + } + plaintext, err = s.enclavePvd.DecryptKMSEnvelopedKey(ciphertextForRecipient) + if err != nil { + return nil, status.Error(codes.Internal, "key recovery failed") + } + } + // Deserialize->Serialize round-trips the KMS/enclave-produced key bytes through + // the aes type to validate and canonicalize them before downstream AES-GCM + // use, so a malformed data key fails here with a clear error rather than + // deeper in DecryptGCM. The bytes are KMS/enclave output (not raw client input), + // so a failure is Internal β€” matching generateEnvelope's identical step. + dataKey, err := aesCommon.Deserialize(plaintext) + if err != nil { + return nil, status.Error(codes.Internal, "key recovery failed") + } + return dataKey.Serialize() +} + +func (s *Service) loadKeyCandidate( + ctx context.Context, + pvd awskms.Provider, + env *pb.SecretEnvelope, +) (crypto.Key, error) { + alg, err := toAlgorithm(env.GetAlgorithm()) + if err != nil { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + dataKey, err := s.decryptDataKey(ctx, pvd, env.GetKmsEncryptedDataKey()) + if err != nil { + return nil, err + } + secretKeyPlaintext, err := aes.DecryptGCM(dataKey, env.GetEncryptedPrivateKey(), env.GetNonce()) + if err != nil { + return nil, status.Error(codes.Internal, "key recovery failed") + } + secretKey, err := crypto.DeserializeSecretKey(alg, secretKeyPlaintext) + if err != nil { + return nil, status.Error(codes.Internal, "key recovery failed") + } + return secretKey, nil +} + +func (s *Service) generateEnvelopeCandidate( + ctx context.Context, + pvd awskms.Provider, + pbAlg pb.Algorithm, +) (*pb.SecretEnvelope, crypto.Key, []byte, error) { + // Derive the crypto algorithm from the same proto value stamped into the + // envelope, so the key-generation algorithm and the envelope's recorded + // algorithm cannot drift. + alg, err := toAlgorithm(pbAlg) + if err != nil { + return nil, nil, nil, status.Error(codes.InvalidArgument, err.Error()) + } + if pvd == nil { + return nil, nil, nil, status.Error(codes.FailedPrecondition, "kms provider not initialized") + } + plainDataKey, kmsCipherDataKey, ciphertextForRecipient, err := pvd.GenerateDataKey(ctx) + if err != nil { + return nil, nil, nil, fmt.Errorf("kms generate data key: %w", err) + } + if s.nitroEnclaveEnabled { + if s.enclavePvd == nil { + return nil, nil, nil, status.Error(codes.FailedPrecondition, "enclave provider is nil while nitro enclave is enabled") + } + plainDataKey, err = s.enclavePvd.DecryptKMSEnvelopedKey(ciphertextForRecipient) + if err != nil { + return nil, nil, nil, status.Error(codes.Internal, "key generation failed") + } + } + dataKey, err := aesCommon.Deserialize(plainDataKey) + if err != nil { + return nil, nil, nil, status.Error(codes.Internal, "key generation failed") + } + dataKeyBytes, err := dataKey.Serialize() + if err != nil { + return nil, nil, nil, status.Errorf(codes.Internal, "serialize data key: %v", err) + } + secretKey, err := crypto.NewSecretKey(alg) + if err != nil { + return nil, nil, nil, status.Errorf(codes.Internal, "generate secret key: %v", err) + } + secretKeyBytes, err := secretKey.Serialize() + if err != nil { + return nil, nil, nil, status.Errorf(codes.Internal, "serialize secret key: %v", err) + } + publicKey, err := secretKey.PublicKey() + if err != nil { + return nil, nil, nil, status.Errorf(codes.Internal, "get public key: %v", err) + } + cipher, nonce, err := aes.EncryptGCM(dataKeyBytes, secretKeyBytes) + if err != nil { + return nil, nil, nil, status.Errorf(codes.Internal, "encrypt secret key: %v", err) + } + env := &pb.SecretEnvelope{ + Algorithm: pbAlg, + KmsEncryptedDataKey: kmsCipherDataKey, + EncryptedPrivateKey: cipher, + Nonce: nonce, + } + return env, secretKey, publicKey, nil +} + +// hashKeySourceFields hashes length-prefixed fields for a fingerprint. +func hashKeySourceFields(args ...[]byte) []byte { + hash := sha256.New() + lenBuf := make([]byte, 4) + for _, arg := range args { + binary.BigEndian.PutUint32(lenBuf, uint32(len(arg))) + hash.Write(lenBuf) + hash.Write(arg) + } + return hash.Sum(nil) +} + +// toAlgorithm maps a proto Algorithm to the enclave crypto algorithm. +func toAlgorithm(algorithm pb.Algorithm) (crypto.Algorithm, error) { + switch algorithm { + case pb.Algorithm_ALGORITHM_BLS: + return crypto.AlgorithmBLS, nil + case pb.Algorithm_ALGORITHM_ED25519: + return crypto.AlgorithmEd25519, nil + default: + return "", fmt.Errorf("invalid algorithm: %s", algorithm) + } +} diff --git a/internal/enclave/service/enclave/envelope_test.go b/internal/enclave/service/enclave/envelope_test.go new file mode 100644 index 0000000..2ae0c86 --- /dev/null +++ b/internal/enclave/service/enclave/envelope_test.go @@ -0,0 +1,125 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enclave + +import ( + "bytes" + "context" + "errors" + "fmt" + "testing" + + "github.com/circlefin/arc-remote-signer/internal/common/crypto/aes" + "github.com/circlefin/arc-remote-signer/internal/enclave/common/crypto" + enclavePvd "github.com/circlefin/arc-remote-signer/internal/enclave/provider/enclave" + "github.com/circlefin/arc-remote-signer/proto/pb" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func newEd25519Envelope(t *testing.T, dataKey []byte) (*pb.SecretEnvelope, crypto.Key) { + t.Helper() + key, err := crypto.NewSecretKey(crypto.AlgorithmEd25519) + require.NoError(t, err) + keyBytes, err := key.Serialize() + require.NoError(t, err) + ciphertext, nonce, err := aes.EncryptGCM(dataKey, keyBytes) + require.NoError(t, err) + return &pb.SecretEnvelope{ + Algorithm: pb.Algorithm_ALGORITHM_ED25519, + KmsEncryptedDataKey: []byte("kms-ciphertext"), + EncryptedPrivateKey: ciphertext, + Nonce: nonce, + }, key +} + +func TestFingerprintKeySource(t *testing.T) { + generate := generateInitRequest() + same := generateInitRequest() + different := generateInitRequest() + different.KeySource = &pb.InitializeRequest_GenerateNew{GenerateNew: pb.Algorithm_ALGORITHM_BLS} + + first, err := fingerprintKeySource(generate) + require.NoError(t, err) + second, err := fingerprintKeySource(same) + require.NoError(t, err) + other, err := fingerprintKeySource(different) + require.NoError(t, err) + require.Equal(t, first, second) + require.NotEqual(t, first, other) +} + +func TestLoadKeyCandidate_RecoversWrappedKey(t *testing.T) { + dataKey := bytes.Repeat([]byte{0x22}, 32) + envelope, wantKey := newEd25519Envelope(t, dataKey) + wantPublicKey, err := wantKey.PublicKey() + require.NoError(t, err) + pvd := &fakeKmsProvider{ + decrypt: func(_ context.Context, ciphertext []byte) ([]byte, []byte, error) { + require.Equal(t, envelope.KmsEncryptedDataKey, ciphertext) + return dataKey, nil, nil + }, + } + + gotKey, err := (&Service{}).loadKeyCandidate(context.Background(), pvd, envelope) + + require.NoError(t, err) + gotPublicKey, err := gotKey.PublicKey() + require.NoError(t, err) + require.Equal(t, wantPublicKey, gotPublicKey) +} + +func TestGenerateEnvelopeCandidate_NitroDecryptsRecipient(t *testing.T) { + dataKey := bytes.Repeat([]byte{0x33}, 32) + recipientCiphertext := []byte("recipient-ciphertext") + pvd := &fakeKmsProvider{ + generateDataKey: func(context.Context) ([]byte, []byte, []byte, error) { + return nil, []byte("kms-ciphertext"), recipientCiphertext, nil + }, + } + ctrl := gomock.NewController(t) + enclaveProvider := enclavePvd.NewMockProvider(ctrl) + enclaveProvider.EXPECT().DecryptKMSEnvelopedKey(recipientCiphertext).Return(dataKey, nil) + svc := &Service{nitroEnclaveEnabled: true, enclavePvd: enclaveProvider} + + envelope, key, publicKey, err := svc.generateEnvelopeCandidate( + context.Background(), + pvd, + pb.Algorithm_ALGORITHM_ED25519, + ) + + require.NoError(t, err) + require.NotNil(t, key) + require.NotEmpty(t, publicKey) + require.Equal(t, []byte("kms-ciphertext"), envelope.GetKmsEncryptedDataKey()) +} + +func TestKMSStatusError(t *testing.T) { + tests := map[string]struct { + err error + wantCode codes.Code + }{ + "canceled": {err: context.Canceled, wantCode: codes.Canceled}, + "deadline": {err: context.DeadlineExceeded, wantCode: codes.DeadlineExceeded}, + "internal": {err: errors.New("boom"), wantCode: codes.Internal}, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + got := kmsStatusError("kms operation", fmt.Errorf("wrapped: %w", tt.err)) + require.Equal(t, tt.wantCode, status.Code(got)) + }) + } +} diff --git a/internal/enclave/service/enclave/initgate.go b/internal/enclave/service/enclave/initgate.go new file mode 100644 index 0000000..ad49fba --- /dev/null +++ b/internal/enclave/service/enclave/initgate.go @@ -0,0 +1,133 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enclave + +import ( + "context" + "errors" + "sync/atomic" + "time" + + enclaveCrypto "github.com/circlefin/arc-remote-signer/internal/enclave/common/crypto" + "github.com/circlefin/arc-remote-signer/proto/pb" + "golang.org/x/sync/singleflight" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" +) + +type keySourceFingerprint [32]byte + +const defaultInitializationTimeout = 30 * time.Second + +// readyState is published only after the complete key lifecycle succeeds. +type readyState struct { + fingerprint keySourceFingerprint + generatedEnvelopeFingerprint keySourceFingerprint + hasGeneratedEnvelopeFingerprint bool + key enclaveCrypto.Key + response *pb.InitializeResponse +} + +// runInitFunc builds a candidate state without publishing it. +type runInitFunc func(context.Context, *pb.InitializeRequest) (*readyState, error) + +type initGate struct { + sf singleflight.Group + ready atomic.Pointer[readyState] + runInit runInitFunc + runTimeout time.Duration +} + +// newInitGate creates an initialization gate. +func newInitGate(runInit runInitFunc) *initGate { + return &initGate{runInit: runInit, runTimeout: defaultInitializationTimeout} +} + +// initialized reports whether a complete Ready state was published. +func (g *initGate) initialized() bool { + return g.ready.Load() != nil +} + +func (g *initGate) state() *readyState { + return g.ready.Load() +} + +// ensureInitialized publishes one complete Ready state. +func (g *initGate) ensureInitialized( + ctx context.Context, + req *pb.InitializeRequest, + fingerprint keySourceFingerprint, +) (*pb.InitializeResponse, error) { + if state := g.ready.Load(); state != nil { + return responseForFingerprint(state, fingerprint) + } + resultCh := g.sf.DoChan("initialize", func() (any, error) { + if state := g.ready.Load(); state != nil { + return state, nil + } + runCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), g.runTimeout) + defer cancel() + state, err := g.runInit(runCtx, req) + if err != nil { + return nil, err + } + if state == nil || state.response == nil { + return nil, errors.New("initialize returned an empty ready state") + } + state.fingerprint = fingerprint + if envelope := state.response.GetSecretEnvelope(); envelope != nil { + recoveryFingerprint, err := fingerprintKeySource(&pb.InitializeRequest{ + KeySource: &pb.InitializeRequest_ExistingKey{ExistingKey: envelope}, + }) + if err != nil { + return nil, err + } + state.generatedEnvelopeFingerprint = recoveryFingerprint + state.hasGeneratedEnvelopeFingerprint = true + } + g.ready.Store(state) + return state, nil + }) + + var result singleflight.Result + select { + case result = <-resultCh: + case <-ctx.Done(): + return nil, ctx.Err() + } + if result.Err != nil { + return nil, result.Err + } + state, ok := result.Val.(*readyState) + if !ok || state == nil { + return nil, errors.New("initialize returned an invalid ready state") + } + return responseForFingerprint(state, fingerprint) +} + +func responseForFingerprint( + state *readyState, + fingerprint keySourceFingerprint, +) (*pb.InitializeResponse, error) { + if state.fingerprint == fingerprint { + return proto.Clone(state.response).(*pb.InitializeResponse), nil + } + if state.hasGeneratedEnvelopeFingerprint && state.generatedEnvelopeFingerprint == fingerprint { + response := proto.Clone(state.response).(*pb.InitializeResponse) + response.SecretEnvelope = nil + return response, nil + } + return nil, status.Error(codes.FailedPrecondition, "enclave is already initialized with a different key source") +} diff --git a/internal/enclave/service/enclave/initialization_test.go b/internal/enclave/service/enclave/initialization_test.go new file mode 100644 index 0000000..e98b8af --- /dev/null +++ b/internal/enclave/service/enclave/initialization_test.go @@ -0,0 +1,458 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enclave + +import ( + "bytes" + "context" + "errors" + "sync" + "sync/atomic" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/smithy-go" + "github.com/circlefin/arc-remote-signer/internal/enclave/common/crypto" + "github.com/circlefin/arc-remote-signer/internal/enclave/provider/awskms" + enclavePvd "github.com/circlefin/arc-remote-signer/internal/enclave/provider/enclave" + "github.com/circlefin/arc-remote-signer/proto/pb" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" +) + +func newAtomicInitService( + t *testing.T, + nitro bool, + pvd awskms.Provider, + enclaveProvider enclavePvd.Provider, +) *Service { + t.Helper() + factory := func(context.Context, aws.Config, []string) (awskms.Provider, error) { + return pvd, nil + } + return newServiceForTest(t, nitro, enclaveProvider, factory, "http://127.0.0.1:9000") +} + +func generateInitRequest() *pb.InitializeRequest { + req := validInitReq() + req.KeySource = &pb.InitializeRequest_GenerateNew{ + GenerateNew: pb.Algorithm_ALGORITHM_ED25519, + } + return req +} + +func TestInitialize_GeneratesAndInstallsKeyAtomically(t *testing.T) { + dataKey := bytes.Repeat([]byte{0x11}, 32) + var kmsCalls atomic.Int32 + pvd := &fakeKmsProvider{ + generateDataKey: func(context.Context) ([]byte, []byte, []byte, error) { + kmsCalls.Add(1) + return dataKey, []byte("kms-ciphertext"), nil, nil + }, + } + svc := newAtomicInitService(t, false, pvd, nil) + + resp, err := svc.Initialize(context.Background(), generateInitRequest()) + + require.NoError(t, err) + require.NotEmpty(t, resp.GetPublicKey()) + require.Equal(t, pb.Algorithm_ALGORITHM_ED25519, resp.GetSecretEnvelope().GetAlgorithm()) + signResp, err := svc.SignMessage(context.Background(), &pb.SignMessageRequest{Message: []byte("message")}) + require.NoError(t, err) + valid, err := crypto.VerifySignedMessage( + crypto.AlgorithmEd25519, + signResp.GetSignature(), + []byte("message"), + resp.GetPublicKey(), + ) + require.NoError(t, err) + require.True(t, valid) + require.Equal(t, int32(1), kmsCalls.Load(), "signing after Ready must not call KMS") +} + +func TestInitialize_RecoversExistingKey(t *testing.T) { + dataKey := bytes.Repeat([]byte{0x22}, 32) + env, key := newEd25519Envelope(t, dataKey) + wantPublicKey, err := key.PublicKey() + require.NoError(t, err) + var decryptCalls atomic.Int32 + pvd := &fakeKmsProvider{ + decrypt: func(context.Context, []byte) ([]byte, []byte, error) { + decryptCalls.Add(1) + return dataKey, nil, nil + }, + } + svc := newAtomicInitService(t, false, pvd, nil) + req := validInitReq() + req.KeySource = &pb.InitializeRequest_ExistingKey{ExistingKey: env} + + resp, err := svc.Initialize(context.Background(), req) + + require.NoError(t, err) + require.Equal(t, wantPublicKey, resp.GetPublicKey()) + require.Nil(t, resp.GetSecretEnvelope()) + _, err = svc.GetPublicKey(context.Background(), &pb.GetPublicKeyRequest{}) + require.NoError(t, err) + require.Equal(t, int32(1), decryptCalls.Load()) +} + +func TestInitialize_MapsDecryptKMSError(t *testing.T) { + dataKey := bytes.Repeat([]byte{0x24}, 32) + env, _ := newEd25519Envelope(t, dataKey) + kmsErr := &smithy.GenericAPIError{ + Code: "AccessDeniedException", + Message: "policy denied the request", + } + pvd := &fakeKmsProvider{ + decrypt: func(context.Context, []byte) ([]byte, []byte, error) { + return nil, nil, kmsErr + }, + } + svc := newAtomicInitService(t, false, pvd, nil) + req := validInitReq() + req.KeySource = &pb.InitializeRequest_ExistingKey{ExistingKey: env} + + resp, err := svc.Initialize(context.Background(), req) + + require.Nil(t, resp) + require.Equal(t, codes.PermissionDenied, status.Code(err)) + require.Contains(t, status.Convert(err).Message(), "kms decrypt data key") +} + +func TestInitialize_FailureDoesNotPublishReadyAndCanRetry(t *testing.T) { + dataKey := bytes.Repeat([]byte{0x33}, 32) + recipientCiphertext := []byte("recipient-ciphertext") + pvd := &fakeKmsProvider{ + generateDataKey: func(context.Context) ([]byte, []byte, []byte, error) { + return nil, []byte("kms-ciphertext"), recipientCiphertext, nil + }, + } + ctrl := gomock.NewController(t) + enclaveProvider := enclavePvd.NewMockProvider(ctrl) + enclaveProvider.EXPECT().DecryptKMSEnvelopedKey(recipientCiphertext).Return(dataKey, nil).Times(2) + enclaveProvider.EXPECT().Attest(gomock.Any()).Return(nil, errors.New("attestation failed")) + enclaveProvider.EXPECT().Attest(gomock.Any()).Return([]byte("attestation"), nil) + svc := newAtomicInitService(t, true, pvd, enclaveProvider) + + resp, err := svc.Initialize(context.Background(), generateInitRequest()) + require.Nil(t, resp) + require.Equal(t, codes.Internal, status.Code(err)) + signResp, signErr := svc.SignMessage( + context.Background(), + &pb.SignMessageRequest{Message: []byte("message")}, + ) + require.Nil(t, signResp) + require.Equal(t, codes.FailedPrecondition, status.Code(signErr)) + + resp, err = svc.Initialize(context.Background(), generateInitRequest()) + require.NoError(t, err) + require.NotEmpty(t, resp.GetPublicKey()) + require.Equal(t, []byte("attestation"), resp.GetAttestationDocument()) +} + +func TestInitialize_RetryCreatesAttemptScopedProviderAndPublishesOnlySuccess(t *testing.T) { + dataKey := bytes.Repeat([]byte{0x34}, 32) + providers := []*fakeKmsProvider{ + { + generateDataKey: func(context.Context) ([]byte, []byte, []byte, error) { + return nil, nil, nil, &smithy.GenericAPIError{ + Code: "ExpiredTokenException", + Message: "expired", + } + }, + }, + { + generateDataKey: func(context.Context) ([]byte, []byte, []byte, error) { + return dataKey, []byte("kms-ciphertext"), nil, nil + }, + }, + } + var factoryCalls atomic.Int32 + factory := func(context.Context, aws.Config, []string) (awskms.Provider, error) { + index := int(factoryCalls.Add(1) - 1) + return providers[index], nil + } + svc := newServiceForTest(t, false, nil, factory, "http://127.0.0.1:9000") + + resp, err := svc.Initialize(context.Background(), generateInitRequest()) + + require.Nil(t, resp) + require.Equal(t, codes.Unauthenticated, status.Code(err)) + require.False(t, svc.gate.initialized()) + require.Nil(t, svc.gate.state()) + + resp, err = svc.Initialize(context.Background(), generateInitRequest()) + + require.NoError(t, err) + require.NotEmpty(t, resp.GetPublicKey()) + require.True(t, svc.gate.initialized()) + require.Equal(t, int32(2), factoryCalls.Load()) +} + +func TestInitialize_GeneratesAndInstallsBLSKey(t *testing.T) { + dataKey := bytes.Repeat([]byte{0x44}, 32) + pvd := &fakeKmsProvider{ + generateDataKey: func(context.Context) ([]byte, []byte, []byte, error) { + return dataKey, []byte("kms-ciphertext"), nil, nil + }, + } + svc := newAtomicInitService(t, false, pvd, nil) + req := validInitReq() + req.KeySource = &pb.InitializeRequest_GenerateNew{GenerateNew: pb.Algorithm_ALGORITHM_BLS} + + resp, err := svc.Initialize(context.Background(), req) + + require.NoError(t, err) + require.Equal(t, pb.Algorithm_ALGORITHM_BLS, resp.GetSecretEnvelope().GetAlgorithm()) + signResp, err := svc.SignMessage( + context.Background(), + &pb.SignMessageRequest{Message: []byte("message")}, + ) + require.NoError(t, err) + valid, err := crypto.VerifySignedMessage( + crypto.AlgorithmBLS, + signResp.GetSignature(), + []byte("message"), + resp.GetPublicKey(), + ) + require.NoError(t, err) + require.True(t, valid) +} + +func TestInitialize_GeneratedEnvelopeReplaysAsExistingSource(t *testing.T) { + dataKey := bytes.Repeat([]byte{0x55}, 32) + var generateCalls atomic.Int32 + var decryptCalls atomic.Int32 + pvd := &fakeKmsProvider{ + generateDataKey: func(context.Context) ([]byte, []byte, []byte, error) { + generateCalls.Add(1) + return dataKey, []byte("kms-ciphertext"), nil, nil + }, + decrypt: func(context.Context, []byte) ([]byte, []byte, error) { + decryptCalls.Add(1) + return dataKey, nil, nil + }, + } + svc := newAtomicInitService(t, false, pvd, nil) + + generated, err := svc.Initialize(context.Background(), generateInitRequest()) + require.NoError(t, err) + recoveryReq := validInitReq() + recoveryReq.KeySource = &pb.InitializeRequest_ExistingKey{ + ExistingKey: proto.Clone(generated.GetSecretEnvelope()).(*pb.SecretEnvelope), + } + + recovered, err := svc.Initialize(context.Background(), recoveryReq) + + require.NoError(t, err) + require.Equal(t, generated.GetPublicKey(), recovered.GetPublicKey()) + require.Nil(t, recovered.GetSecretEnvelope()) + require.Equal(t, int32(1), generateCalls.Load()) + require.Equal(t, int32(0), decryptCalls.Load(), "host restart replay must use the installed key") +} + +func TestInitialize_GeneratedEnvelopeRejectsDifferentExistingSource(t *testing.T) { + dataKey := bytes.Repeat([]byte{0x56}, 32) + var generateCalls atomic.Int32 + var decryptCalls atomic.Int32 + pvd := &fakeKmsProvider{ + generateDataKey: func(context.Context) ([]byte, []byte, []byte, error) { + generateCalls.Add(1) + return dataKey, []byte("kms-ciphertext"), nil, nil + }, + decrypt: func(context.Context, []byte) ([]byte, []byte, error) { + decryptCalls.Add(1) + return dataKey, nil, nil + }, + } + svc := newAtomicInitService(t, false, pvd, nil) + + generated, err := svc.Initialize(context.Background(), generateInitRequest()) + require.NoError(t, err) + differentEnvelope := proto.Clone(generated.GetSecretEnvelope()).(*pb.SecretEnvelope) + differentEnvelope.Nonce[0] ^= 0xff + conflictReq := validInitReq() + conflictReq.KeySource = &pb.InitializeRequest_ExistingKey{ExistingKey: differentEnvelope} + + resp, err := svc.Initialize(context.Background(), conflictReq) + + require.Nil(t, resp) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + require.Equal(t, int32(1), generateCalls.Load()) + require.Equal(t, int32(0), decryptCalls.Load()) +} + +func TestInitialize_ConflictingKeySourceAfterReadyFailsWithoutKMS(t *testing.T) { + dataKey := bytes.Repeat([]byte{0x66}, 32) + var kmsCalls atomic.Int32 + pvd := &fakeKmsProvider{ + generateDataKey: func(context.Context) ([]byte, []byte, []byte, error) { + kmsCalls.Add(1) + return dataKey, []byte("kms-ciphertext"), nil, nil + }, + } + svc := newAtomicInitService(t, false, pvd, nil) + _, err := svc.Initialize(context.Background(), generateInitRequest()) + require.NoError(t, err) + conflict := validInitReq() + conflict.KeySource = &pb.InitializeRequest_GenerateNew{GenerateNew: pb.Algorithm_ALGORITHM_BLS} + + resp, err := svc.Initialize(context.Background(), conflict) + + require.Nil(t, resp) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + require.Equal(t, int32(1), kmsCalls.Load()) +} + +func TestInitialize_ResponseReplayIsImmutable(t *testing.T) { + dataKey := bytes.Repeat([]byte{0x77}, 32) + pvd := &fakeKmsProvider{ + generateDataKey: func(context.Context) ([]byte, []byte, []byte, error) { + return dataKey, []byte("kms-ciphertext"), nil, nil + }, + } + svc := newAtomicInitService(t, false, pvd, nil) + req := generateInitRequest() + first, err := svc.Initialize(context.Background(), req) + require.NoError(t, err) + wantReplay := proto.Clone(first).(*pb.InitializeResponse) + first.PublicKey[0] ^= 0xff + first.SecretEnvelope.Nonce[0] ^= 0xff + + replayed, err := svc.Initialize( + context.Background(), + proto.Clone(req).(*pb.InitializeRequest), + ) + + require.NoError(t, err) + require.True(t, proto.Equal(wantReplay, replayed)) +} + +func TestInitialize_ConcurrentDifferentSourcesPublishOneWinner(t *testing.T) { + dataKey := bytes.Repeat([]byte{0x88}, 32) + started := make(chan struct{}) + release := make(chan struct{}) + var kmsCalls atomic.Int32 + pvd := &fakeKmsProvider{ + generateDataKey: func(context.Context) ([]byte, []byte, []byte, error) { + if kmsCalls.Add(1) == 1 { + close(started) + } + <-release + return dataKey, []byte("kms-ciphertext"), nil, nil + }, + } + svc := newAtomicInitService(t, false, pvd, nil) + requests := []*pb.InitializeRequest{generateInitRequest(), validInitReq()} + requests[1].KeySource = &pb.InitializeRequest_GenerateNew{ + GenerateNew: pb.Algorithm_ALGORITHM_BLS, + } + waiting := make(chan struct{}) + waitingContext := &observedDoneContext{ + Context: context.Background(), + observed: waiting, + } + responses := make([]*pb.InitializeResponse, len(requests)) + errs := make([]error, len(requests)) + var waitGroup sync.WaitGroup + waitGroup.Add(len(requests)) + go func() { + defer waitGroup.Done() + responses[0], errs[0] = svc.Initialize(context.Background(), requests[0]) + }() + <-started + go func() { + defer waitGroup.Done() + responses[1], errs[1] = svc.Initialize(waitingContext, requests[1]) + }() + <-waiting + close(release) + waitGroup.Wait() + + var successCount, conflictCount int + for i := range requests { + switch status.Code(errs[i]) { + case codes.OK: + successCount++ + require.NotEmpty(t, responses[i].GetPublicKey()) + case codes.FailedPrecondition: + conflictCount++ + require.Nil(t, responses[i]) + default: + require.Failf(t, "unexpected result", "request %d returned %v", i, errs[i]) + } + } + require.Equal(t, 1, successCount) + require.Equal(t, 1, conflictCount) + require.Equal(t, int32(1), kmsCalls.Load()) +} + +func TestInitialize_RequiresExactlyOneValidKeySource(t *testing.T) { + dataKey := bytes.Repeat([]byte{0x99}, 32) + var kmsCalls atomic.Int32 + pvd := &fakeKmsProvider{ + generateDataKey: func(context.Context) ([]byte, []byte, []byte, error) { + kmsCalls.Add(1) + return dataKey, []byte("kms-ciphertext"), nil, nil + }, + } + svc := newAtomicInitService(t, false, pvd, nil) + + tests := map[string]*pb.InitializeRequest{ + "missing source": func() *pb.InitializeRequest { + req := validInitReq() + req.KeySource = nil + return req + }(), + "unspecified generate algorithm": func() *pb.InitializeRequest { + req := validInitReq() + req.KeySource = &pb.InitializeRequest_GenerateNew{ + GenerateNew: pb.Algorithm_ALGORITHM_UNSPECIFIED, + } + return req + }(), + "incomplete existing envelope": func() *pb.InitializeRequest { + req := validInitReq() + req.KeySource = &pb.InitializeRequest_ExistingKey{ + ExistingKey: &pb.SecretEnvelope{Algorithm: pb.Algorithm_ALGORITHM_ED25519}, + } + return req + }(), + } + + for name, req := range tests { + t.Run(name, func(t *testing.T) { + resp, err := svc.Initialize(context.Background(), req) + require.Nil(t, resp) + require.Equal(t, codes.InvalidArgument, status.Code(err)) + }) + } + require.Equal(t, int32(0), kmsCalls.Load()) +} + +type observedDoneContext struct { + context.Context + once sync.Once + observed chan struct{} +} + +func (c *observedDoneContext) Done() <-chan struct{} { + c.once.Do(func() { + close(c.observed) + }) + return nil +} diff --git a/internal/enclave/service/enclave/initrunner.go b/internal/enclave/service/enclave/initrunner.go new file mode 100644 index 0000000..92975a9 --- /dev/null +++ b/internal/enclave/service/enclave/initrunner.go @@ -0,0 +1,81 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enclave + +import ( + "context" + "fmt" + + "github.com/circlefin/arc-remote-signer/internal/enclave/common/crypto" + "github.com/circlefin/arc-remote-signer/internal/enclave/provider/awskms" + "github.com/circlefin/arc-remote-signer/proto/pb" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// runInit builds a candidate Ready state without publishing partial state. +func (s *Service) runInit(ctx context.Context, req *pb.InitializeRequest) (*readyState, error) { + awsCfg, err := awskms.BuildConfig(req.Credentials, s.awsproxyEndpoint, req.KmsLocalstackEnabled) + if err != nil { + return nil, fmt.Errorf("build aws config: %w", err) + } + pvd, err := s.kmsFactory(ctx, awsCfg, req.KmsKeyArns) + if err != nil { + return nil, fmt.Errorf("build kms provider: %w", err) + } + + var ( + key crypto.Key + publicKey []byte + envelope *pb.SecretEnvelope + ) + switch source := req.GetKeySource().(type) { + case *pb.InitializeRequest_ExistingKey: + key, err = s.loadKeyCandidate(ctx, pvd, source.ExistingKey) + if err != nil { + return nil, err + } + publicKey, err = key.PublicKey() + if err != nil { + return nil, status.Error(codes.Internal, "failed to get public key") + } + case *pb.InitializeRequest_GenerateNew: + envelope, key, publicKey, err = s.generateEnvelopeCandidate(ctx, pvd, source.GenerateNew) + if err != nil { + return nil, err + } + default: + return nil, status.Error(codes.InvalidArgument, "key_source is required") + } + + var attestationDocument []byte + if s.nitroEnclaveEnabled { + if s.enclavePvd == nil { + return nil, status.Error(codes.Internal, "enclave provider is nil while nitro enclave is enabled") + } + attestationDocument, err = s.enclavePvd.Attest(publicKey) + if err != nil { + return nil, status.Error(codes.Internal, "failed to attest public key") + } + } + + return &readyState{ + key: key, + response: &pb.InitializeResponse{ + PublicKey: publicKey, + AttestationDocument: attestationDocument, + SecretEnvelope: envelope, + }, + }, nil +} diff --git a/internal/enclave/service/enclave/initrunner_integration_test.go b/internal/enclave/service/enclave/initrunner_integration_test.go new file mode 100644 index 0000000..2d903b6 --- /dev/null +++ b/internal/enclave/service/enclave/initrunner_integration_test.go @@ -0,0 +1,133 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build integration + +package enclave + +import ( + "context" + "net" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/circlefin/arc-remote-signer/internal/enclave/common/crypto" + "github.com/circlefin/arc-remote-signer/internal/enclave/provider/awskms" + "github.com/circlefin/arc-remote-signer/proto/pb" + "github.com/stretchr/testify/require" +) + +// LocalStack fixture values mirror the awskms integration test +// (internal/enclave/provider/awskms/awskms_integration_test.go) and the docker-compose +// at deployments/docker-compose.yaml. The KMS key alias is seeded by +// deployments/localstack_scripts/create-kms-keys.sh. +const ( + localstackEndpoint = "http://localhost:4566" + localstackKmsArn = "arn:aws:kms:us-east-1:000000000000:alias/dev-multi-region-crypto" + localstackInvalidArn = "arn:aws:kms:us-east-1:000000000000:alias/does-not-exist" +) + +// newLocalstackFactory returns an awskms.Factory that wires a real plaintext +// provider against LocalStack. It matches the development path while avoiding +// the host-side proxy processes. LocalStack does not support Nitro +// RecipientInfo. +func newLocalstackFactory(t *testing.T) awskms.Factory { + t.Helper() + return func(callCtx context.Context, awsCfg aws.Config, arns []string) (awskms.Provider, error) { + return awskms.NewForDevelopment( + callCtx, + &awskms.Config{Arns: arns, ConnectTimeout: 3000}, + awsCfg, + ) + } +} + +func validIntegrationReq(t *testing.T, arn string) *pb.InitializeRequest { + t.Helper() + return &pb.InitializeRequest{ + Credentials: &pb.AwsCredentials{ + AccessKeyId: "test", + SecretAccessKey: "test", + SessionToken: "test", + Region: "us-east-1", + }, + KmsKeyArns: []string{arn}, + KmsLocalstackEnabled: true, + KeySource: &pb.InitializeRequest_GenerateNew{ + GenerateNew: pb.Algorithm_ALGORITHM_ED25519, + }, + } +} + +// TestNewRunInit_Integration_HappyPath exercises the full runInit +// closure against LocalStack: awskms.BuildConfig produces an aws.Config that +// points at LocalStack, the factory builds a real KMS provider, and +// GenerateDataKey returns successfully. +// +// This test points awsproxyEndpoint at LocalStack. It exercises a direct +// SDK-to-LocalStack dial instead of the production awsproxy(TCP) -> +// vsockproxy -> LocalStack chain. That keeps the runInit logic test +// hermetic (no proxy processes to stand up). The proxied chain itself is +// covered end-to-end elsewhere in CI: the compose smoke job (make test-all +// -> up -> smoke.sh, which starts the host app and Initializes the enclave +// through the wired proxies) and the dedicated smoke_eif job +// (scripts/smoke-eif.sh on a real Nitro runner). +func TestNewRunInit_Integration_HappyPath(t *testing.T) { + s := newServiceForTest(t, false, nil, newLocalstackFactory(t), localstackEndpoint) + _, err := s.runInit(context.Background(), validIntegrationReq(t, localstackKmsArn)) + require.NoError(t, err, "runInit must succeed against LocalStack with a valid ARN") +} + +// TestNewRunInit_Integration_InvalidArn verifies that runInit surfaces a +// non-nil error when LocalStack rejects the ARN. The error is returned +// raw β€” mapping to a gRPC status happens in runInitStatusError, not +// inside runInit. +func TestNewRunInit_Integration_InvalidArn(t *testing.T) { + s := newServiceForTest(t, false, nil, newLocalstackFactory(t), localstackEndpoint) + _, err := s.runInit(context.Background(), validIntegrationReq(t, localstackInvalidArn)) + require.Error(t, err, "runInit must propagate KMS errors when the ARN is unknown") +} + +// TestNewRunInit_Integration_InvalidEndpoint verifies that runInit +// surfaces a transport error when awsproxy is unreachable. We point at +// 127.0.0.1:65535 which is essentially never bound, so the SDK's HTTP +// dial fails fast with connection refused. The error must satisfy +// net.Error somewhere in the wrap chain so awskms.StatusFromError's net.Error +// branch will translate it to codes.Unavailable in production. +func TestNewRunInit_Integration_InvalidEndpoint(t *testing.T) { + s := newServiceForTest(t, false, nil, newLocalstackFactory(t), "http://127.0.0.1:65535") + _, err := s.runInit(context.Background(), validIntegrationReq(t, localstackKmsArn)) + require.Error(t, err, "runInit must surface a transport error when the endpoint is unreachable") + var netErr net.Error + require.ErrorAs(t, err, &netErr, + "SDK transport failure must wrap a net.Error so awskms.StatusFromError's net.Error branch fires in production") +} + +// TestService_Envelope_Integration_GenerateThenSign generates a key +// in-enclave via a real LocalStack KMS GenerateDataKey, then signs with the +// returned SecretEnvelope end to end and verifies the signature. +func TestService_Envelope_Integration_GenerateThenSign(t *testing.T) { + ctx := context.Background() + s := newServiceForTest(t, false, nil, newLocalstackFactory(t), localstackEndpoint) + initResp, err := s.Initialize(ctx, validIntegrationReq(t, localstackKmsArn)) + require.NoError(t, err, "Initialize must succeed against LocalStack") + + msg := []byte("integration-message") + sig, err := s.SignMessage(ctx, &pb.SignMessageRequest{Message: msg}) + require.NoError(t, err) + require.NotEmpty(t, sig.Signature) + + ok, err := crypto.VerifySignedMessage(crypto.AlgorithmEd25519, sig.Signature, msg, initResp.PublicKey) + require.NoError(t, err) + require.True(t, ok, "signature must verify against the generated public key") +} diff --git a/internal/enclave/service/enclave/initrunner_test.go b/internal/enclave/service/enclave/initrunner_test.go new file mode 100644 index 0000000..84ac8cf --- /dev/null +++ b/internal/enclave/service/enclave/initrunner_test.go @@ -0,0 +1,227 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enclave + +import ( + "bytes" + "context" + "errors" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/smithy-go" + "github.com/circlefin/arc-remote-signer/internal/enclave/provider/awskms" + "github.com/circlefin/arc-remote-signer/internal/enclave/provider/enclave" + "github.com/circlefin/arc-remote-signer/proto/pb" + "github.com/stretchr/testify/require" +) + +// fakeKmsProvider implements awskms.Provider for enclave-side tests. +// We deliberately do not use the gomock generated under +// internal/enclave/provider/awskms/awskms_mock.go to keep these tests +// decoupled from the generated mock. +type fakeKmsProvider struct { + generateDataKey func(ctx context.Context) ([]byte, []byte, []byte, error) + decrypt func(ctx context.Context, ciphertext []byte) ([]byte, []byte, error) +} + +func (f *fakeKmsProvider) GenerateDataKey(ctx context.Context) ([]byte, []byte, []byte, error) { + return f.generateDataKey(ctx) +} + +func (f *fakeKmsProvider) Decrypt(ctx context.Context, ciphertext []byte) ([]byte, []byte, error) { + return f.decrypt(ctx, ciphertext) +} + +// newServiceForTest builds a Service whose gate runs the real runInit +// method, with an injectable KMS factory and awsproxy endpoint. Test-only. +func newServiceForTest( + t *testing.T, + nitroEnclaveEnabled bool, + enclavePvd enclave.Provider, + factory awskms.Factory, + awsproxyEndpoint string, +) *Service { + t.Helper() + s := &Service{ + nitroEnclaveEnabled: nitroEnclaveEnabled, + enclavePvd: enclavePvd, + kmsFactory: factory, + awsproxyEndpoint: awsproxyEndpoint, + } + s.gate = newInitGate(s.runInit) + return s +} + +// validInitReq returns a well-formed Initialize request shape for tests. +func validInitReq() *pb.InitializeRequest { + return &pb.InitializeRequest{ + Credentials: &pb.AwsCredentials{ + AccessKeyId: "AKIA-test", + SecretAccessKey: "secret", + SessionToken: "session", + Region: "us-east-1", + }, + KmsKeyArns: []string{"arn:aws:kms:us-east-1:123456789012:key/abc"}, + KeySource: &pb.InitializeRequest_GenerateNew{ + GenerateNew: pb.Algorithm_ALGORITHM_ED25519, + }, + } +} + +func TestNewRunInit_HappyPath(t *testing.T) { + type ctxKey struct{} + var calledArns []string + var calledRegion string + var calledCtxValue any + factory := func(callCtx context.Context, awsCfg aws.Config, arns []string) (awskms.Provider, error) { + calledArns = arns + calledRegion = awsCfg.Region + calledCtxValue = callCtx.Value(ctxKey{}) + return &fakeKmsProvider{ + generateDataKey: func(_ context.Context) ([]byte, []byte, []byte, error) { + return bytes.Repeat([]byte{0x11}, 32), []byte("cipher"), []byte("recipient"), nil + }, + }, nil + } + + s := newServiceForTest(t, false, nil, factory, "http://127.0.0.1:9000") + ctx := context.WithValue(context.Background(), ctxKey{}, "marker") + _, err := s.runInit(ctx, validInitReq()) + require.NoError(t, err) + require.Equal(t, []string{"arn:aws:kms:us-east-1:123456789012:key/abc"}, calledArns) + require.Equal(t, "us-east-1", calledRegion) + require.Equal(t, "marker", calledCtxValue, + "factory must receive the runInit per-call ctx (carries deadline + values)") +} + +func TestRunInitSelectsKMSBackend(t *testing.T) { + tests := map[string]struct { + localstack bool + wantBase string + }{ + "LocalStack": {localstack: true, wantBase: "http://127.0.0.1:9000"}, + "AWS KMS": {}, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + var gotBase string + factory := func(_ context.Context, awsCfg aws.Config, _ []string) (awskms.Provider, error) { + if awsCfg.BaseEndpoint != nil { + gotBase = *awsCfg.BaseEndpoint + } + return &fakeKmsProvider{ + generateDataKey: func(context.Context) ([]byte, []byte, []byte, error) { + return bytes.Repeat([]byte{0x11}, 32), []byte("cipher"), nil, nil + }, + }, nil + } + svc := newServiceForTest( + t, + false, + nil, + factory, + "http://127.0.0.1:9000", + ) + req := validInitReq() + req.KmsLocalstackEnabled = tt.localstack + + _, err := svc.runInit(context.Background(), req) + require.NoError(t, err) + require.Equal(t, tt.wantBase, gotBase) + }) + } +} + +func TestNewRunInit_BuildConfigError_NilCreds(t *testing.T) { + factory := func(_ context.Context, _ aws.Config, _ []string) (awskms.Provider, error) { + t.Fatal("factory must not be called when awskms.BuildConfig fails") + return nil, nil + } + s := newServiceForTest(t, false, nil, factory, "http://127.0.0.1:9000") + req := &pb.InitializeRequest{Credentials: nil, KmsKeyArns: []string{"arn"}} + _, err := s.runInit(context.Background(), req) + require.Error(t, err) + require.Contains(t, err.Error(), "build aws config") +} + +func TestNewRunInit_BuildConfigError_EmptyEndpoint(t *testing.T) { + factory := func(_ context.Context, _ aws.Config, _ []string) (awskms.Provider, error) { + t.Fatal("factory must not be called when awskms.BuildConfig fails") + return nil, nil + } + s := newServiceForTest(t, false, nil, factory, "") + _, err := s.runInit(context.Background(), validInitReq()) + require.Error(t, err) + require.Contains(t, err.Error(), "build aws config") +} + +func TestNewRunInit_FactoryError(t *testing.T) { + wantErr := errors.New("factory boom") + factory := func(_ context.Context, _ aws.Config, _ []string) (awskms.Provider, error) { + return nil, wantErr + } + s := newServiceForTest(t, false, nil, factory, "http://127.0.0.1:9000") + _, err := s.runInit(context.Background(), validInitReq()) + require.Error(t, err) + require.ErrorIs(t, err, wantErr) + require.Contains(t, err.Error(), "build kms provider") +} + +func TestNewRunInit_GenerateDataKeyError_PassThrough(t *testing.T) { + // Tests that runInit returns the raw SDK error; mapping to gRPC code + // happens later in runInitStatusError, not inside runInit. + kmsErr := &smithy.GenericAPIError{Code: "AccessDeniedException", Message: "denied"} + factory := func(_ context.Context, _ aws.Config, _ []string) (awskms.Provider, error) { + return &fakeKmsProvider{ + generateDataKey: func(_ context.Context) ([]byte, []byte, []byte, error) { + return nil, nil, nil, kmsErr + }, + }, nil + } + s := newServiceForTest(t, false, nil, factory, "http://127.0.0.1:9000") + _, err := s.runInit(context.Background(), validInitReq()) + require.ErrorIs(t, err, kmsErr) +} + +func TestNewRunInit_ContextCancelled(t *testing.T) { + factory := func(_ context.Context, _ aws.Config, _ []string) (awskms.Provider, error) { + return &fakeKmsProvider{ + generateDataKey: func(ctx context.Context) ([]byte, []byte, []byte, error) { + <-ctx.Done() + return nil, nil, nil, ctx.Err() + }, + }, nil + } + s := newServiceForTest(t, false, nil, factory, "http://127.0.0.1:9000") + ctx, cancel := context.WithCancel(context.Background()) + cancel() // pre-cancelled + _, err := s.runInit(ctx, validInitReq()) + require.ErrorIs(t, err, context.Canceled) +} + +func TestRunInit_RetainsKMSProvider(t *testing.T) { + want := &fakeKmsProvider{ + generateDataKey: func(_ context.Context) ([]byte, []byte, []byte, error) { + return bytes.Repeat([]byte{0x11}, 32), []byte("cipher"), []byte("recipient"), nil + }, + } + factory := func(_ context.Context, _ aws.Config, _ []string) (awskms.Provider, error) { + return want, nil + } + s := newServiceForTest(t, false, nil, factory, "http://127.0.0.1:9000") + _, err := s.Initialize(context.Background(), validInitReq()) + require.NoError(t, err) +} diff --git a/internal/smoke/public/attestation_dev_test.go b/internal/smoke/public/attestation_dev_test.go new file mode 100644 index 0000000..52f5e33 --- /dev/null +++ b/internal/smoke/public/attestation_dev_test.go @@ -0,0 +1,18 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build smoke && !nitro_smoke + +package smoke + +const expectNitroAttestation = false diff --git a/internal/smoke/public/attestation_nitro_test.go b/internal/smoke/public/attestation_nitro_test.go new file mode 100644 index 0000000..da16c42 --- /dev/null +++ b/internal/smoke/public/attestation_nitro_test.go @@ -0,0 +1,18 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build smoke && nitro_smoke + +package smoke + +const expectNitroAttestation = true diff --git a/internal/smoke/public/public_test.go b/internal/smoke/public/public_test.go index 00d8791..7587db3 100644 --- a/internal/smoke/public/public_test.go +++ b/internal/smoke/public/public_test.go @@ -68,6 +68,11 @@ func (suite *TestSuite) TestPublicKey() { suite.Require().NoError(err, "PublicKey endpoint failed") suite.Require().NotNil(result, "PublicKey response is nil") suite.Require().NotEmpty(result.PublicKey, "PublicKey is empty") + if expectNitroAttestation { + suite.Require().NotEmpty(result.AttestationDocument, "Attestation document is empty") + } else { + suite.Require().Empty(result.AttestationDocument, "Attestation document is not empty") + } } // TestSign tests the Sign gRPC endpoint. diff --git a/internal/vsockproxy/config.go b/internal/vsockproxy/config.go new file mode 100644 index 0000000..8850488 --- /dev/null +++ b/internal/vsockproxy/config.go @@ -0,0 +1,116 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package vsockproxy is the standalone host-side bridge between vsock +// (production) or TCP (dev/CI) listeners and AWS service endpoints. It +// runs as its own process via the `app run-vsockproxy` subcommand. +package vsockproxy + +import ( + "fmt" +) + +// configName is the ApplicationConfig.GetName() result; surfaces in +// logging. +const configName = "nitro-enclave-signer-vsockproxy" + +// DefaultBasePort is the default vsock port the host-side proxy listens +// on. Additional services occupy consecutive ports (base+1, ...). +const DefaultBasePort uint32 = 10316 + +// DefaultMaxConns mirrors gateway-tee-signer's socat max-children=50 +// default and bounds in-flight connections per service. +const DefaultMaxConns = 50 + +// endpointTemplates maps each entry in byteproxy.AWSServiceNames to +// the AWS endpoint template used for that service. The "{region}" +// placeholder is substituted with the per-connection region selected by +// the enclave KMS client. Keyed by service name so +// a drift between this map and byteproxy.AWSServiceNames is caught at +// startup. +var endpointTemplates = map[string]string{ + "kms": "kms.{region}.amazonaws.com:443", +} + +// EndpointTemplate returns the endpoint template registered for the +// given service. Exported so tests and operators can inspect the +// mapping; New performs the actual {region} substitution and parity +// check at startup. +func EndpointTemplate(service string) (string, error) { + tmpl, ok := endpointTemplates[service] + if !ok { + return "", fmt.Errorf("vsockproxy: no endpoint template registered for service %q", service) + } + return tmpl, nil +} + +// Config contains the settings for the host-side proxy. +type Config struct { + Vsockproxy Vsockproxy `mapstructure:"vsockproxy"` + Provider ProviderConfig `mapstructure:"provider"` +} + +// Vsockproxy holds the host-side proxy settings. +type Vsockproxy struct { + // BasePort is the vsock or TCP port for the first service. + BasePort uint32 `mapstructure:"basePort"` + + // MaxConns caps in-flight connections per service. + MaxConns int `mapstructure:"maxConns"` +} + +// ProviderConfig mirrors the subset of app.Config.Provider that the +// vsockproxy process needs to consume. +type ProviderConfig struct { + Enclave ProviderEnclaveConfig `mapstructure:"enclave"` + AWSKMS ProviderAWSKMSConfig `mapstructure:"awskms"` +} + +// ProviderEnclaveConfig mirrors the subset of the enclave provider +// config that the vsockproxy process needs to consume. +type ProviderEnclaveConfig struct { + NitroEnclave ProviderNitroEnclaveConfig `mapstructure:"nitroEnclave"` +} + +// ProviderNitroEnclaveConfig mirrors the NitroEnclave block from the +// enclave provider config. Only `enabled` is read here; CID/Port belong +// to the app process. +type ProviderNitroEnclaveConfig struct { + Enabled bool `mapstructure:"enabled"` +} + +// ProviderAWSKMSConfig contains the KMS settings that the proxy needs. +// ARN regions form the allowlist for standard AWS KMS routes. +type ProviderAWSKMSConfig struct { + Localstack ProviderAWSKMSLocalstackConfig `mapstructure:"localstack"` + Arns []string `mapstructure:"arns"` +} + +// ProviderAWSKMSLocalstackConfig selects LocalStack as the KMS backend. +type ProviderAWSKMSLocalstackConfig struct { + Enabled bool `mapstructure:"enabled"` +} + +// NewConfig returns Config defaults suitable for a host run. +func NewConfig() *Config { + return &Config{ + Vsockproxy: Vsockproxy{ + BasePort: DefaultBasePort, + MaxConns: DefaultMaxConns, + }, + } +} + +// GetName implements config.ApplicationConfig so LoadConfig can target +// this Config. +func (c *Config) GetName() string { return configName } diff --git a/internal/vsockproxy/config_test.go b/internal/vsockproxy/config_test.go new file mode 100644 index 0000000..a9d603c --- /dev/null +++ b/internal/vsockproxy/config_test.go @@ -0,0 +1,81 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vsockproxy + +import ( + "testing" + + "github.com/circlefin/arc-remote-signer/internal/common/config" + "github.com/stretchr/testify/require" +) + +func TestNewConfig_Defaults(t *testing.T) { + cfg := NewConfig() + + require.NotNil(t, cfg) + require.Equal(t, DefaultBasePort, cfg.Vsockproxy.BasePort) + require.Equal(t, DefaultMaxConns, cfg.Vsockproxy.MaxConns) + require.False(t, cfg.Provider.Enclave.NitroEnclave.Enabled, + "nitroEnclave.enabled default must mirror the app provider default (false)") + require.Empty(t, cfg.Provider.AWSKMS.Arns, + "awskms.arns default must be empty so callers supply a real list") + + require.Equal(t, uint32(10316), DefaultBasePort, + "DefaultBasePort must match gateway-tee-signer's aws_proxy_port default") + require.Equal(t, 50, DefaultMaxConns, + "DefaultMaxConns must match gateway-tee-signer's socat max-children=50") +} + +func TestConfig_GetName(t *testing.T) { + cfg := NewConfig() + require.Equal(t, "nitro-enclave-signer-vsockproxy", cfg.GetName()) +} + +// TestLoadConfig_EnvOverridesYaml pins the APP_VSOCKPROXY_* viper path +// production deployments depend on: every override goes through +// config.LoadConfig, which merges defaults + app.yaml + env in that +// order. A regression in the mapstructure tags, EnvKeyReplacer, or +// the Vsockproxy struct shape would silently leave compose / prod on +// stale values without this assertion firing. +func TestLoadConfig_EnvOverridesYaml(t *testing.T) { + t.Setenv("APP_VSOCKPROXY_BASEPORT", "20000") + t.Setenv("APP_VSOCKPROXY_MAXCONNS", "100") + t.Setenv("APP_PROVIDER_ENCLAVE_NITROENCLAVE_ENABLED", "true") + t.Setenv("APP_PROVIDER_AWSKMS_LOCALSTACK_ENABLED", "true") + + cfg := NewConfig() + config.LoadConfig(cfg, "../../configs/app.yaml") + + require.Equal(t, uint32(20000), cfg.Vsockproxy.BasePort) + require.Equal(t, 100, cfg.Vsockproxy.MaxConns) + require.True(t, cfg.Provider.Enclave.NitroEnclave.Enabled, + "APP_PROVIDER_ENCLAVE_NITROENCLAVE_ENABLED must drive the transport selector in run.go") + require.True(t, cfg.Provider.AWSKMS.Localstack.Enabled) +} + +func TestEndpointTemplate(t *testing.T) { + t.Run("KnownService", func(t *testing.T) { + tmpl, err := EndpointTemplate("kms") + require.NoError(t, err) + require.Contains(t, tmpl, "{region}") + require.Contains(t, tmpl, "kms.") + require.Contains(t, tmpl, ":443") + }) + + t.Run("UnknownService", func(t *testing.T) { + _, err := EndpointTemplate("nonexistent-service") + require.Error(t, err) + require.Contains(t, err.Error(), "nonexistent-service") + }) +} diff --git a/internal/vsockproxy/options.go b/internal/vsockproxy/options.go new file mode 100644 index 0000000..354a8a1 --- /dev/null +++ b/internal/vsockproxy/options.go @@ -0,0 +1,56 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vsockproxy + +import ( + "context" + "net" + + "github.com/circlefin/arc-remote-signer/internal/common/byteproxy" +) + +// listenFunc binds a listener for the given service port. +type listenFunc func(port uint32) (net.Listener, error) + +// dialFunc opens an upstream connection. The endpoint string is opaque to +// the byteproxy framework; the wrapper decides what it means. +type dialFunc func(ctx context.Context, endpoint string) (net.Conn, error) + +// options carries injection points and the selected transport. +type options struct { + listen listenFunc + dial dialFunc + transport byteproxy.TransportKind + // tcpLocalstackEndpoint is the upstream dial target in TCP mode; ignored in vsock mode. + tcpLocalstackEndpoint string +} + +// Option customises New. Transport-selection options +// (WithVsockTransport / WithTCPTransport) and test seams (WithListen / +// WithDial) all use this single type. +type Option func(*options) + +// WithListen overrides the listener for tests. Production callers should +// rely on the default produced by the selected transport. Must be passed +// AFTER any transport-selection option (WithVsockTransport / WithTCPTransport), +// which install listen/dial unconditionally and would otherwise overwrite +// this override. +func WithListen(fn listenFunc) Option { return func(o *options) { o.listen = fn } } + +// WithDial overrides the upstream dialer for tests. Production callers +// should rely on the default produced by the selected transport. Must be +// passed AFTER any transport-selection option (WithVsockTransport / +// WithTCPTransport), which install listen/dial unconditionally and would +// otherwise overwrite this override. +func WithDial(fn dialFunc) Option { return func(o *options) { o.dial = fn } } diff --git a/internal/vsockproxy/ready.go b/internal/vsockproxy/ready.go new file mode 100644 index 0000000..fb7772c --- /dev/null +++ b/internal/vsockproxy/ready.go @@ -0,0 +1,68 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vsockproxy + +import ( + "fmt" + "os" + "path/filepath" +) + +// readyFileEnv names the env var the launcher (docker/run.sh) sets to a +// readiness sentinel path. vsockproxy creates the file only after every +// listener is bound β€” which happens synchronously in New β€” so the parent +// can gate enclave startup on the proxy actually listening instead of +// probing the kernel (vsock listeners are not enumerable from the host via +// /proc or the tools shipped in the image). An empty/unset value disables +// the signal, so compose, smoke, and local runs are unaffected. +const readyFileEnv = "VSOCKPROXY_READY_FILE" + +// createReadyTemp creates a temp file in dir that signalReady renames into +// place. It is a package var so tests can exercise the close-failure branch, +// which is otherwise impossible to trigger deterministically for a freshly +// created empty file. +var createReadyTemp = func(dir string) (*os.File, error) { + return os.CreateTemp(dir, "vsockproxy.ready-*") +} + +// signalReady publishes the readiness sentinel when readyFileEnv is set. +// It MUST be called only after New returns successfully, i.e. once all +// listeners are bound. A failure is returned so the caller aborts rather +// than leaving the parent waiting on a file that never appears. +// +// The sentinel is written to a temp file in the same directory and renamed +// into place only after Close succeeds. Rename is atomic within a +// filesystem, so docker/run.sh can never observe the sentinel before it is +// fully written β€” a failure at any step leaves no sentinel, so the parent +// cannot false-ready for a proxy that did not come up cleanly. +func signalReady() error { + path := os.Getenv(readyFileEnv) + if path == "" { + return nil + } + f, err := createReadyTemp(filepath.Dir(path)) + if err != nil { + return fmt.Errorf("vsockproxy: write readiness file %q: %w", path, err) + } + tmp := f.Name() + if err := f.Close(); err != nil { + _ = os.Remove(tmp) + return fmt.Errorf("vsockproxy: close readiness file %q: %w", path, err) + } + if err := os.Rename(tmp, path); err != nil { + _ = os.Remove(tmp) + return fmt.Errorf("vsockproxy: publish readiness file %q: %w", path, err) + } + return nil +} diff --git a/internal/vsockproxy/ready_test.go b/internal/vsockproxy/ready_test.go new file mode 100644 index 0000000..cbf8b37 --- /dev/null +++ b/internal/vsockproxy/ready_test.go @@ -0,0 +1,75 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vsockproxy + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestSignalReady(t *testing.T) { + t.Run("unset env is a no-op", func(t *testing.T) { + t.Setenv(readyFileEnv, "") + require.NoError(t, signalReady()) + }) + + t.Run("creates sentinel file", func(t *testing.T) { + path := filepath.Join(t.TempDir(), "vsockproxy.ready") + t.Setenv(readyFileEnv, path) + + require.NoError(t, signalReady()) + require.FileExists(t, path) + }) + + t.Run("unwritable path errors", func(t *testing.T) { + // A path whose parent directory does not exist cannot be created; + // the error must propagate so the launcher does not block on a + // sentinel that will never appear. + path := filepath.Join(t.TempDir(), "no-such-dir", "vsockproxy.ready") + t.Setenv(readyFileEnv, path) + + err := signalReady() + require.ErrorContains(t, err, path) + }) + + t.Run("close failure leaves no sentinel and wraps error", func(t *testing.T) { + // Force the close-failure branch via the createReadyTemp seam: the + // returned file is already closed, so signalReady's Close errors + // before it can rename the temp into place. The sentinel must never + // appear and the leftover temp must be removed. + dir := t.TempDir() + path := filepath.Join(dir, "vsockproxy.ready") + orig := createReadyTemp + t.Cleanup(func() { createReadyTemp = orig }) + createReadyTemp = func(d string) (*os.File, error) { + f, err := os.CreateTemp(d, "vsockproxy.ready-*") + require.NoError(t, err) + require.NoError(t, f.Close()) // pre-close so the next Close fails + return f, nil + } + t.Setenv(readyFileEnv, path) + + err := signalReady() + require.ErrorContains(t, err, path) + require.ErrorContains(t, err, "close") + require.NoFileExists(t, path, "sentinel must not appear when close fails") + + entries, rerr := os.ReadDir(dir) + require.NoError(t, rerr) + require.Empty(t, entries, "leftover temp file must be removed on close failure") + }) +} diff --git a/internal/vsockproxy/routing.go b/internal/vsockproxy/routing.go new file mode 100644 index 0000000..a009ed4 --- /dev/null +++ b/internal/vsockproxy/routing.go @@ -0,0 +1,156 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vsockproxy + +import ( + "context" + "errors" + "fmt" + "net" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws/arn" + "github.com/circlefin/arc-remote-signer/internal/common/byteproxy" +) + +const routeReadTimeout = time.Second + +type routeResolver struct { + allowedRegions map[string]struct{} + useLocalstack bool + options *options +} + +func newRouteResolver(cfg *Config, o *options) (*routeResolver, error) { + if cfg == nil { + return nil, errors.New("vsockproxy: nil config") + } + if err := ensureServiceParity(byteproxy.AWSServiceNames, endpointTemplates); err != nil { + return nil, err + } + if o.transport != byteproxy.TransportVsock && o.transport != byteproxy.TransportTCP { + return nil, fmt.Errorf("vsockproxy: unhandled transport kind %d", o.transport) + } + if cfg.Provider.AWSKMS.Localstack.Enabled { + if o.transport != byteproxy.TransportTCP { + return nil, errors.New("vsockproxy: LocalStack requires TCP transport") + } + return &routeResolver{useLocalstack: true, options: o}, nil + } + if len(cfg.Provider.AWSKMS.Arns) == 0 { + return nil, errors.New("vsockproxy: provider.awskms.arns has no entry to derive allowed regions from") + } + regions := make(map[string]struct{}, len(cfg.Provider.AWSKMS.Arns)) + for i, value := range cfg.Provider.AWSKMS.Arns { + parsed, err := arn.Parse(value) + if err != nil { + return nil, fmt.Errorf("vsockproxy: provider.awskms.arns[%d] is not a valid ARN: %w", i, err) + } + if parsed.Service != "kms" { + return nil, fmt.Errorf("vsockproxy: provider.awskms.arns[%d] is not a KMS ARN", i) + } + if parsed.Region == "" { + return nil, fmt.Errorf("vsockproxy: provider.awskms.arns[%d] has no region", i) + } + regions[parsed.Region] = struct{}{} + } + return &routeResolver{allowedRegions: regions, options: o}, nil +} + +func (r *routeResolver) resolve(route byteproxy.AWSRoute) (string, error) { + tmpl, ok := endpointTemplates[route.Service] + if !ok { + return "", fmt.Errorf("vsockproxy: route service %q is not supported", route.Service) + } + if r.useLocalstack { + return r.options.tcpLocalstackEndpoint, nil + } + if _, ok := r.allowedRegions[route.Region]; !ok { + return "", fmt.Errorf("vsockproxy: route region %q is not configured", route.Region) + } + return strings.ReplaceAll(tmpl, "{region}", route.Region), nil +} + +func (r *routeResolver) dial( + ctx context.Context, + service string, + inbound net.Conn, +) (net.Conn, error) { + route, err := r.readRoute(ctx, inbound) + if err != nil { + return nil, err + } + if route.Service != service { + return nil, fmt.Errorf("vsockproxy: route service %q does not match listener service %q", route.Service, service) + } + endpoint, err := r.resolve(route) + if err != nil { + return nil, err + } + upstream, err := r.options.dial(ctx, endpoint) + if err != nil { + return nil, fmt.Errorf("vsockproxy: dial %s: %w", endpoint, err) + } + return upstream, nil +} + +// readRoute reads exactly one AWS route header from inbound, bounded by both +// routeReadTimeout and ctx cancellation. bridge passes the shutdown ctx so a +// connection stalled mid-header does not delay Shutdown past a ctx cancel: +// on cancellation the pending read is unblocked (deadline moved to now) and +// the reader goroutine is reaped before returning. +func (r *routeResolver) readRoute(ctx context.Context, inbound net.Conn) (byteproxy.AWSRoute, error) { + if err := inbound.SetReadDeadline(time.Now().Add(routeReadTimeout)); err != nil { + return byteproxy.AWSRoute{}, fmt.Errorf("vsockproxy: set route read deadline: %w", err) + } + type result struct { + route byteproxy.AWSRoute + err error + } + resultCh := make(chan result, 1) + go func() { + route, err := byteproxy.ReadAWSRoute(inbound) + resultCh <- result{route: route, err: err} + }() + + select { + case <-ctx.Done(): + // Force the blocked ReadAWSRoute to return so the goroutine exits, then + // reap it to avoid a leak before reporting the cancellation. + _ = inbound.SetReadDeadline(time.Now()) + <-resultCh + return byteproxy.AWSRoute{}, fmt.Errorf("vsockproxy: route read cancelled: %w", ctx.Err()) + case res := <-resultCh: + if resetErr := inbound.SetReadDeadline(time.Time{}); resetErr != nil && res.err == nil { + return byteproxy.AWSRoute{}, fmt.Errorf("vsockproxy: clear route read deadline: %w", resetErr) + } + if res.err != nil { + return byteproxy.AWSRoute{}, fmt.Errorf("vsockproxy: read route: %w", res.err) + } + return res.route, nil + } +} + +func ensureServiceParity(services []string, templates map[string]string) error { + for _, name := range services { + if _, ok := templates[name]; !ok { + return fmt.Errorf( + "vsockproxy: byteproxy.AWSServiceNames includes %q but no endpoint template is registered; "+ + "add an entry to vsockproxy/config.go endpointTemplates", name, + ) + } + } + return nil +} diff --git a/internal/vsockproxy/routing_test.go b/internal/vsockproxy/routing_test.go new file mode 100644 index 0000000..3594c8a --- /dev/null +++ b/internal/vsockproxy/routing_test.go @@ -0,0 +1,231 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vsockproxy + +import ( + "context" + "errors" + "net" + "testing" + + "github.com/circlefin/arc-remote-signer/internal/common/byteproxy" + "github.com/stretchr/testify/require" +) + +func TestRouteResolverSelectsRequestedAllowedRegion(t *testing.T) { + cfg := &Config{Provider: ProviderConfig{AWSKMS: ProviderAWSKMSConfig{Arns: []string{ + "arn:aws:kms:us-east-1:123456789012:key/first", + "arn:aws:kms:us-west-2:123456789012:key/second", + }}}} + resolver, err := newRouteResolver(cfg, &options{transport: byteproxy.TransportVsock}) + require.NoError(t, err) + + endpoint, err := resolver.resolve(byteproxy.AWSRoute{Service: "kms", Region: "us-west-2"}) + require.NoError(t, err) + require.Equal(t, "kms.us-west-2.amazonaws.com:443", endpoint) +} + +func TestRouteResolverSelectsBackendInTCPMode(t *testing.T) { + tests := map[string]struct { + localstack bool + want string + }{ + "LocalStack": {localstack: true, want: "localstack:4566"}, + "AWS KMS": {want: "kms.us-west-2.amazonaws.com:443"}, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + cfg := &Config{Provider: ProviderConfig{AWSKMS: ProviderAWSKMSConfig{ + Arns: []string{"arn:aws:kms:us-west-2:123456789012:key/test"}, + Localstack: ProviderAWSKMSLocalstackConfig{ + Enabled: tt.localstack, + }, + }}} + resolver, err := newRouteResolver(cfg, &options{ + transport: byteproxy.TransportTCP, + tcpLocalstackEndpoint: "localstack:4566", + }) + require.NoError(t, err) + + endpoint, err := resolver.resolve(byteproxy.AWSRoute{Service: "kms", Region: "us-west-2"}) + require.NoError(t, err) + require.Equal(t, tt.want, endpoint) + }) + } +} + +func TestRouteResolverUsesConfiguredLocalstackTarget(t *testing.T) { + cfg := &Config{Provider: ProviderConfig{AWSKMS: ProviderAWSKMSConfig{ + Localstack: ProviderAWSKMSLocalstackConfig{Enabled: true}, + }}} + resolver, err := newRouteResolver(cfg, &options{ + transport: byteproxy.TransportTCP, + tcpLocalstackEndpoint: "test-localstack:4566", + }) + require.NoError(t, err) + + endpoint, err := resolver.resolve(byteproxy.AWSRoute{Service: "kms", Region: "us-east-1"}) + require.NoError(t, err) + require.Equal(t, "test-localstack:4566", endpoint) +} + +func TestRouteResolverRejectsLocalstackWithVsock(t *testing.T) { + cfg := &Config{Provider: ProviderConfig{AWSKMS: ProviderAWSKMSConfig{ + Localstack: ProviderAWSKMSLocalstackConfig{Enabled: true}, + }}} + + _, err := newRouteResolver(cfg, &options{transport: byteproxy.TransportVsock}) + require.ErrorContains(t, err, "LocalStack requires TCP transport") +} + +func TestRouteResolverRejectsRegionOutsideARNAllowlist(t *testing.T) { + cfg := &Config{Provider: ProviderConfig{AWSKMS: ProviderAWSKMSConfig{Arns: []string{ + "arn:aws:kms:us-east-1:123456789012:key/first", + }}}} + resolver, err := newRouteResolver(cfg, &options{transport: byteproxy.TransportVsock}) + require.NoError(t, err) + + _, err = resolver.resolve(byteproxy.AWSRoute{Service: "kms", Region: "eu-west-1"}) + require.ErrorContains(t, err, "not configured") +} + +func TestRouteResolverRejectsServiceMismatch(t *testing.T) { + cfg := &Config{Provider: ProviderConfig{AWSKMS: ProviderAWSKMSConfig{Arns: []string{ + "arn:aws:kms:us-east-1:123456789012:key/first", + }}}} + resolver, err := newRouteResolver(cfg, &options{transport: byteproxy.TransportVsock}) + require.NoError(t, err) + + _, err = resolver.resolve(byteproxy.AWSRoute{Service: "secretsmanager", Region: "us-east-1"}) + require.ErrorContains(t, err, "service") +} + +func TestRouteResolverDialReadsInboundRoute(t *testing.T) { + client, inbound := net.Pipe() + t.Cleanup(func() { _ = client.Close() }) + t.Cleanup(func() { _ = inbound.Close() }) + upstream, upstreamPeer := net.Pipe() + t.Cleanup(func() { _ = upstream.Close() }) + t.Cleanup(func() { _ = upstreamPeer.Close() }) + + dialed := make(chan string, 1) + cfg := &Config{Provider: ProviderConfig{AWSKMS: ProviderAWSKMSConfig{Arns: []string{ + "arn:aws:kms:us-east-1:123456789012:key/first", + "arn:aws:kms:us-west-2:123456789012:key/second", + }}}} + resolver, err := newRouteResolver(cfg, &options{ + transport: byteproxy.TransportVsock, + dial: func(_ context.Context, endpoint string) (net.Conn, error) { + dialed <- endpoint + return upstream, nil + }, + }) + require.NoError(t, err) + + writeErr := make(chan error, 1) + go func() { + writeErr <- byteproxy.WriteAWSRoute(client, byteproxy.AWSRoute{ + Service: "kms", + Region: "us-west-2", + }) + }() + gotConn, err := resolver.dial(context.Background(), "kms", inbound) + require.NoError(t, err) + require.Same(t, upstream, gotConn) + require.NoError(t, <-writeErr) + require.Equal(t, "kms.us-west-2.amazonaws.com:443", <-dialed) +} + +func TestRouteResolverDialErrorIncludesResolvedEndpoint(t *testing.T) { + client, inbound := net.Pipe() + t.Cleanup(func() { _ = client.Close() }) + t.Cleanup(func() { _ = inbound.Close() }) + + cfg := &Config{Provider: ProviderConfig{AWSKMS: ProviderAWSKMSConfig{Arns: []string{ + "arn:aws:kms:us-west-2:123456789012:key/second", + }}}} + resolver, err := newRouteResolver(cfg, &options{ + transport: byteproxy.TransportVsock, + dial: func(_ context.Context, _ string) (net.Conn, error) { + return nil, errors.New("connection refused") + }, + }) + require.NoError(t, err) + + writeErr := make(chan error, 1) + go func() { + writeErr <- byteproxy.WriteAWSRoute(client, byteproxy.AWSRoute{ + Service: "kms", + Region: "us-west-2", + }) + }() + _, err = resolver.dial(context.Background(), "kms", inbound) + require.ErrorContains(t, err, "kms.us-west-2.amazonaws.com:443") + require.NoError(t, <-writeErr) +} + +func TestRouteResolverDialRejectsServiceMismatch(t *testing.T) { + client, inbound := net.Pipe() + t.Cleanup(func() { _ = client.Close() }) + t.Cleanup(func() { _ = inbound.Close() }) + + cfg := &Config{Provider: ProviderConfig{AWSKMS: ProviderAWSKMSConfig{Arns: []string{ + "arn:aws:kms:us-east-1:123456789012:key/first", + }}}} + resolver, err := newRouteResolver(cfg, &options{ + transport: byteproxy.TransportVsock, + dial: func(_ context.Context, _ string) (net.Conn, error) { + return nil, errors.New("dial must not be reached on service mismatch") + }, + }) + require.NoError(t, err) + + // The inbound route declares a service other than the listener's ("kms"), + // so dial must reject it before resolving an endpoint or dialing upstream. + writeErr := make(chan error, 1) + go func() { + writeErr <- byteproxy.WriteAWSRoute(client, byteproxy.AWSRoute{ + Service: "secretsmanager", + Region: "us-east-1", + }) + }() + _, err = resolver.dial(context.Background(), "kms", inbound) + require.ErrorContains(t, err, "does not match listener service") + require.NoError(t, <-writeErr) +} + +func TestRouteResolverDialCancelsRouteReadOnContext(t *testing.T) { + client, inbound := net.Pipe() + t.Cleanup(func() { _ = client.Close() }) + t.Cleanup(func() { _ = inbound.Close() }) + + cfg := &Config{Provider: ProviderConfig{AWSKMS: ProviderAWSKMSConfig{Arns: []string{ + "arn:aws:kms:us-east-1:123456789012:key/first", + }}}} + resolver, err := newRouteResolver(cfg, &options{ + transport: byteproxy.TransportVsock, + dial: func(_ context.Context, _ string) (net.Conn, error) { + return nil, errors.New("dial must not be reached when the route read is cancelled") + }, + }) + require.NoError(t, err) + + // No route is ever written to client, so the header read blocks; a cancelled + // ctx must abort it (well before the 1s routeReadTimeout) rather than hang. + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err = resolver.dial(ctx, "kms", inbound) + require.ErrorIs(t, err, context.Canceled) +} diff --git a/internal/vsockproxy/run.go b/internal/vsockproxy/run.go new file mode 100644 index 0000000..da3da5b --- /dev/null +++ b/internal/vsockproxy/run.go @@ -0,0 +1,69 @@ +//go:build linux + +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vsockproxy + +import ( + "context" + "time" + + "github.com/circlefin/arc-remote-signer/internal/common/lifecycle" + "github.com/circlefin/arc-remote-signer/internal/common/logging" +) + +// Run constructs the vsockproxy, registers it with a lifecycle manager, +// and blocks on Manager.Run until SIGINT or SIGTERM. The transport +// option is derived from cfg.Provider.Enclave.NitroEnclave.Enabled, the +// same toggle the app process consumes. +// +// Returns an error only if New fails; the cobra subcommand surfaces it +// via RunE (exit 1). lifecycle.Manager.Run log.Fatals on runtime +// runnable failures, which is fine for a single-runnable process. +func Run(cfg *Config) error { + applyBuildPolicy(cfg) + + logger := logging.Get(configName) + logger.Info(context.Background(), "starting vsockproxy", logging.Entries{ + "nitroEnclaveEnabled": cfg.Provider.Enclave.NitroEnclave.Enabled, + "basePort": cfg.Vsockproxy.BasePort, + }) + + p, err := New(cfg, runtimeTransportOption(cfg)) + if err != nil { + return err + } + // Release the listeners New bound on every post-construction return path + // (e.g. a signalReady failure below). Shutdown is idempotent, so the + // lifecycle-driven shutdown after Run still works on the happy path; the + // bounded context caps this cleanup so it cannot outlive the lifecycle's + // own shutdown budget. + defer func() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = p.Shutdown(ctx) + }() + + // Listeners are bound by New (see byteproxy.New), so signal readiness + // now β€” before blocking on Run β€” to let the parent gate enclave startup + // on the proxy actually listening. + if err := signalReady(); err != nil { + return err + } + + lc := lifecycle.NewManager() + lc.Manage(p) + lc.Run() + return nil +} diff --git a/internal/vsockproxy/run_signal_test.go b/internal/vsockproxy/run_signal_test.go new file mode 100644 index 0000000..d3757fc --- /dev/null +++ b/internal/vsockproxy/run_signal_test.go @@ -0,0 +1,42 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !prod + +package vsockproxy + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestRun_ReadinessSignalFailureAborts verifies that Run returns an error +// when it cannot write the readiness file. +func TestRun_ReadinessSignalFailureAborts(t *testing.T) { + // Parent directory does not exist, so os.Create fails and signalReady + // returns an error carrying the path. + path := filepath.Join(t.TempDir(), "no-such-dir", "vsockproxy.ready") + t.Setenv(readyFileEnv, path) + + cfg := &Config{ + Vsockproxy: Vsockproxy{BasePort: 0, MaxConns: 1}, + Provider: ProviderConfig{AWSKMS: ProviderAWSKMSConfig{ + Arns: []string{"arn:aws:kms:us-east-1:000000000000:key/test"}, + }}, + } + + err := Run(cfg) + require.ErrorContains(t, err, path) +} diff --git a/internal/vsockproxy/run_stub.go b/internal/vsockproxy/run_stub.go new file mode 100644 index 0000000..6dd4592 --- /dev/null +++ b/internal/vsockproxy/run_stub.go @@ -0,0 +1,77 @@ +//go:build !linux + +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vsockproxy + +import ( + "context" + "errors" + "time" + + "github.com/circlefin/arc-remote-signer/internal/common/lifecycle" + "github.com/circlefin/arc-remote-signer/internal/common/logging" +) + +// errVsockRequiresLinux is returned by Run when vsock transport is +// requested on a non-Linux build, where it cannot be satisfied. +var errVsockRequiresLinux = errors.New("vsockproxy: vsock transport requires Linux") + +// Run constructs the vsockproxy on non-Linux platforms. Only TCP +// transport is available, so Run rejects NitroEnclave.Enabled up front; +// WithVsockTransport does not exist in non-Linux builds. +func Run(cfg *Config) error { + if cfg.Provider.Enclave.NitroEnclave.Enabled { + return errVsockRequiresLinux + } + + logger := logging.Get(configName) + logger.Info(context.Background(), "starting vsockproxy (non-Linux, TCP only)", logging.Entries{ + "basePort": cfg.Vsockproxy.BasePort, + }) + + p, err := New(cfg, transportOptionForConfig(cfg)) + if err != nil { + return err + } + // Release the listeners New bound on every post-construction return path + // (e.g. a signalReady failure below). Shutdown is idempotent, so the + // lifecycle-driven shutdown after Run still works on the happy path; the + // bounded context caps this cleanup so it cannot outlive the lifecycle's + // own shutdown budget. + defer func() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = p.Shutdown(ctx) + }() + + // Listeners are bound by New (see byteproxy.New), so signal readiness + // now β€” before blocking on Run β€” to let the parent gate enclave startup + // on the proxy actually listening. + if err := signalReady(); err != nil { + return err + } + + lc := lifecycle.NewManager() + lc.Manage(p) + lc.Run() + return nil +} + +// transportOptionForConfig on non-Linux always returns WithTCPTransport; +// vsock mode is rejected earlier in Run because WithVsockTransport does +// not exist in non-Linux builds. +func transportOptionForConfig(_ *Config) Option { + return WithTCPTransport() +} diff --git a/internal/vsockproxy/run_stub_test.go b/internal/vsockproxy/run_stub_test.go new file mode 100644 index 0000000..d150c9d --- /dev/null +++ b/internal/vsockproxy/run_stub_test.go @@ -0,0 +1,31 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !linux + +package vsockproxy + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestRun_NitroEnclaveEnabledErrors(t *testing.T) { + cfg := stubCfg() + cfg.Provider.Enclave.NitroEnclave.Enabled = true + + err := Run(cfg) + + require.ErrorIs(t, err, errVsockRequiresLinux) +} diff --git a/internal/vsockproxy/run_test.go b/internal/vsockproxy/run_test.go new file mode 100644 index 0000000..03c8104 --- /dev/null +++ b/internal/vsockproxy/run_test.go @@ -0,0 +1,80 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux && !prod + +package vsockproxy + +import ( + "testing" + + "github.com/circlefin/arc-remote-signer/internal/common/byteproxy" + "github.com/stretchr/testify/require" +) + +// TestTransportOptionForConfig verifies that the helper used by Run +// picks WithVsockTransport when provider.enclave.nitroEnclave.enabled is +// true and WithTCPTransport() otherwise. This is the small piece of +// logic inside Run that we test in isolation; the full Run path is +// covered by the docker-compose integration test. +func TestTransportOptionForConfig(t *testing.T) { + t.Run("VsockWhenNitroEnabled", func(t *testing.T) { + cfg := &Config{ + Provider: ProviderConfig{ + Enclave: ProviderEnclaveConfig{ + NitroEnclave: ProviderNitroEnclaveConfig{Enabled: true}, + }, + }, + } + opt := transportOptionForConfig(cfg) + require.NotNil(t, opt) + o := options{} + opt(&o) + require.Equal(t, byteproxy.TransportVsock, o.transport) + }) + + t.Run("TCPWhenNitroDisabled", func(t *testing.T) { + cfg := &Config{ + Provider: ProviderConfig{ + Enclave: ProviderEnclaveConfig{ + NitroEnclave: ProviderNitroEnclaveConfig{Enabled: false}, + }, + }, + } + opt := transportOptionForConfig(cfg) + require.NotNil(t, opt) + o := options{} + opt(&o) + require.Equal(t, byteproxy.TransportTCP, o.transport) + }) +} + +// TestRun_NewFailureReturnsError verifies that Run surfaces the New +// error directly (no log.Fatal). The config selects vsock mode but +// supplies no ARN, so route allowlist construction fails before any port is bound. +func TestRun_NewFailureReturnsError(t *testing.T) { + cfg := &Config{ + Vsockproxy: Vsockproxy{ + BasePort: 30000, + MaxConns: 4, + }, + Provider: ProviderConfig{ + Enclave: ProviderEnclaveConfig{ + NitroEnclave: ProviderNitroEnclaveConfig{Enabled: true}, + }, + }, + } + + err := Run(cfg) + require.Error(t, err) +} diff --git a/internal/vsockproxy/runtime_dev_linux.go b/internal/vsockproxy/runtime_dev_linux.go new file mode 100644 index 0000000..4af835b --- /dev/null +++ b/internal/vsockproxy/runtime_dev_linux.go @@ -0,0 +1,31 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux && !prod + +package vsockproxy + +func applyBuildPolicy(*Config) { + // The development build uses the supplied configuration. +} + +func runtimeTransportOption(cfg *Config) Option { + return transportOptionForConfig(cfg) +} + +func transportOptionForConfig(cfg *Config) Option { + if cfg.Provider.Enclave.NitroEnclave.Enabled { + return WithVsockTransport() + } + return WithTCPTransport() +} diff --git a/internal/vsockproxy/runtime_prod_linux.go b/internal/vsockproxy/runtime_prod_linux.go new file mode 100644 index 0000000..352da97 --- /dev/null +++ b/internal/vsockproxy/runtime_prod_linux.go @@ -0,0 +1,27 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux && prod + +package vsockproxy + +func applyBuildPolicy(cfg *Config) { + // The production build sends AWS requests to AWS KMS endpoints through VSOCK. + cfg.Vsockproxy.BasePort = DefaultBasePort + cfg.Provider.Enclave.NitroEnclave.Enabled = true + cfg.Provider.AWSKMS.Localstack.Enabled = false +} + +func runtimeTransportOption(*Config) Option { + return WithVsockTransport() +} diff --git a/internal/vsockproxy/runtime_prod_test.go b/internal/vsockproxy/runtime_prod_test.go new file mode 100644 index 0000000..8a13864 --- /dev/null +++ b/internal/vsockproxy/runtime_prod_test.go @@ -0,0 +1,40 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux && prod + +package vsockproxy + +import ( + "testing" + + "github.com/circlefin/arc-remote-signer/internal/common/byteproxy" + "github.com/stretchr/testify/require" +) + +func TestApplyBuildPolicy_EnforcesProductionSettings(t *testing.T) { + cfg := NewConfig() + cfg.Vsockproxy.BasePort = 1 + cfg.Provider.Enclave.NitroEnclave.Enabled = false + cfg.Provider.AWSKMS.Localstack.Enabled = true + + applyBuildPolicy(cfg) + + require.Equal(t, DefaultBasePort, cfg.Vsockproxy.BasePort) + require.True(t, cfg.Provider.Enclave.NitroEnclave.Enabled) + require.False(t, cfg.Provider.AWSKMS.Localstack.Enabled) + + options := options{} + runtimeTransportOption(cfg)(&options) + require.Equal(t, byteproxy.TransportVsock, options.transport) +} diff --git a/internal/vsockproxy/transport_tcp.go b/internal/vsockproxy/transport_tcp.go new file mode 100644 index 0000000..01da3cd --- /dev/null +++ b/internal/vsockproxy/transport_tcp.go @@ -0,0 +1,66 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vsockproxy + +import ( + "context" + "fmt" + "net" + "time" + + "github.com/circlefin/arc-remote-signer/internal/common/byteproxy" +) + +// tcpDialTimeout bounds an in-flight upstream TCP dial in TCP transport +// mode so a hung dev endpoint does not pin Shutdown past its deadline. +const tcpDialTimeout = 10 * time.Second + +// tcpListenHost is the bind host used in TCP transport mode. Dev/CI is +// the only consumer of this transport; the proxy and its caller live in +// different containers (compose, smoke), so the listener must accept +// remote dials. +const tcpListenHost = "0.0.0.0" + +// tcpLocalstackEndpoint is the LocalStack target. The compose service name +// and the default LocalStack port are fixed. +const tcpLocalstackEndpoint = "localstack:4566" + +// WithTCPTransport selects TCP transport for development and CI. It also +// sets the LocalStack target for configurations that enable LocalStack. +// The transport installs its listen and dial functions. The last transport +// option wins when a caller passes more than one option. +func WithTCPTransport() Option { + return func(o *options) { + o.transport = byteproxy.TransportTCP + o.tcpLocalstackEndpoint = tcpLocalstackEndpoint + o.listen = tcpListen(tcpListenHost) + o.dial = tcpDial + } +} + +// tcpListen returns a listenFunc that binds host: on TCP. Used +// when WithTCPTransport is selected. +func tcpListen(host string) listenFunc { + return func(port uint32) (net.Listener, error) { + return net.Listen("tcp", fmt.Sprintf("%s:%d", host, port)) + } +} + +// tcpDial opens a TCP connection to the resolved upstream endpoint. The +// ctx is forwarded from byteproxy.Shutdown so a stalled TCP handshake +// aborts promptly during lifecycle shutdown. +func tcpDial(ctx context.Context, endpoint string) (net.Conn, error) { + d := net.Dialer{Timeout: tcpDialTimeout} + return d.DialContext(ctx, "tcp", endpoint) +} diff --git a/internal/vsockproxy/transport_tcp_test.go b/internal/vsockproxy/transport_tcp_test.go new file mode 100644 index 0000000..6fbfc3c --- /dev/null +++ b/internal/vsockproxy/transport_tcp_test.go @@ -0,0 +1,57 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vsockproxy + +import ( + "context" + "net" + "testing" + + "github.com/circlefin/arc-remote-signer/internal/common/byteproxy" + "github.com/stretchr/testify/require" +) + +func TestTCPListen_BindsHost(t *testing.T) { + listen := tcpListen("127.0.0.1") + ln, err := listen(0) + require.NoError(t, err) + t.Cleanup(func() { _ = ln.Close() }) + + addr, ok := ln.Addr().(*net.TCPAddr) + require.True(t, ok, "addr must be *net.TCPAddr") + require.Equal(t, "127.0.0.1", addr.IP.String(), + "tcpListen must bind the supplied host") +} + +// TestWithTCPTransport_DefaultsToLocalstackEndpoint pins the upstream +// endpoint and listen host the TCP transport uses in dev/CI. compose and +// smoke depend on these defaults; a silent rename of the package +// constants would break those environments without any other test +// firing. +func TestWithTCPTransport_DefaultsToLocalstackEndpoint(t *testing.T) { + o := options{} + WithTCPTransport()(&o) + require.Equal(t, byteproxy.TransportTCP, o.transport) + require.Equal(t, "localstack:4566", o.tcpLocalstackEndpoint) +} + +func TestTCPDial_Succeeds(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { _ = ln.Close() }) + + conn, err := tcpDial(context.Background(), ln.Addr().String()) + require.NoError(t, err) + require.NoError(t, conn.Close()) +} diff --git a/internal/vsockproxy/transport_vsock.go b/internal/vsockproxy/transport_vsock.go new file mode 100644 index 0000000..bf6d83e --- /dev/null +++ b/internal/vsockproxy/transport_vsock.go @@ -0,0 +1,64 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux + +package vsockproxy + +import ( + "context" + "net" + "time" + + "github.com/circlefin/arc-remote-signer/internal/common/byteproxy" + "github.com/mdlayher/vsock" +) + +// vsockDialTimeout bounds the upstream TCP dial when vsockproxy bridges +// vsock-accept β†’ TCP-AWS. +const vsockDialTimeout = 10 * time.Second + +// WithVsockTransport selects the production vsock transport: vsock listen +// on the configured BasePort, TCP dial to the resolved AWS endpoint. +// References to this option do not compile on non-Linux platforms. The +// transport's listen/dial are installed unconditionally so that when more +// than one transport option is passed the last one wins coherently; the +// WithListen/WithDial test seams still override because they are applied +// after the transport option. +func WithVsockTransport() Option { + return func(o *options) { + o.transport = byteproxy.TransportVsock + o.listen = vsockListen + o.dial = vsockDial + } +} + +// vsockListen binds a VSOCK listener to the Nitro parent CID for the given port. +func vsockListen(port uint32) (net.Listener, error) { + return listenNitroParentVsock(port, listenVsockContextID) +} + +// listenVsockContextID binds directly to contextID without discovering the local CID. +// Do not replace this with vsock.Listen: it opens /dev/vsock, which is not exposed +// to the Kubernetes signer container. +func listenVsockContextID(contextID, port uint32) (net.Listener, error) { + return vsock.ListenContextID(contextID, port, &vsock.Config{}) +} + +// vsockDial opens a TCP connection to the given AWS endpoint. The ctx is +// forwarded from byteproxy.Shutdown so a stalled TCP handshake aborts +// promptly during lifecycle shutdown. +func vsockDial(ctx context.Context, endpoint string) (net.Conn, error) { + d := net.Dialer{Timeout: vsockDialTimeout} + return d.DialContext(ctx, "tcp", endpoint) +} diff --git a/internal/vsockproxy/transport_vsock_test.go b/internal/vsockproxy/transport_vsock_test.go new file mode 100644 index 0000000..de9eef2 --- /dev/null +++ b/internal/vsockproxy/transport_vsock_test.go @@ -0,0 +1,40 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vsockproxy + +import ( + "errors" + "net" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestListenNitroParentVsock_UsesNitroParentCID(t *testing.T) { + const port uint32 = 10316 + wantErr := errors.New("listener stopped") + var gotCID uint32 + var gotPort uint32 + listen := func(cid, port uint32) (net.Listener, error) { + gotCID = cid + gotPort = port + return nil, wantErr + } + + _, err := listenNitroParentVsock(port, listen) + + require.ErrorIs(t, err, wantErr) + require.Equal(t, uint32(3), gotCID) + require.Equal(t, port, gotPort) +} diff --git a/internal/vsockproxy/vsock_listener.go b/internal/vsockproxy/vsock_listener.go new file mode 100644 index 0000000..66a4bd8 --- /dev/null +++ b/internal/vsockproxy/vsock_listener.go @@ -0,0 +1,26 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vsockproxy + +import "net" + +// nitroParentCID is the AWS Nitro parent CID. It is distinct from vsock.Host +// (CID 2) and must match parentCID in the enclave-side awsproxy transport. +const nitroParentCID uint32 = 3 + +type contextIDListenFunc func(contextID, port uint32) (net.Listener, error) + +func listenNitroParentVsock(port uint32, listen contextIDListenFunc) (net.Listener, error) { + return listen(nitroParentCID, port) +} diff --git a/internal/vsockproxy/vsockproxy.go b/internal/vsockproxy/vsockproxy.go new file mode 100644 index 0000000..61e11d9 --- /dev/null +++ b/internal/vsockproxy/vsockproxy.go @@ -0,0 +1,81 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux + +package vsockproxy + +import ( + "context" + "errors" + "fmt" + "net" + + "github.com/circlefin/arc-remote-signer/internal/common/byteproxy" +) + +// proxyName is the lifecycle.Runnable name reported by both the enabled +// and disabled providers. +const proxyName = "host-vsockproxy" + +// Provider is the host-side proxy interface (byteproxy.Proxy plus the +// dropped-count observability hook). +type Provider interface { + byteproxy.Proxy +} + +// New creates a host-side proxy. The transport defaults to vsock (the +// zero value); pass WithTCPTransport() to select TCP for dev/CI. +// Passing both transport options is allowed and the last one wins. +func New(cfg *Config, opts ...Option) (Provider, error) { + if cfg == nil { + return nil, errors.New("vsockproxy: nil config") + } + + // transport's zero value is vsock; start with its listen/dial so a call + // with no transport option runs vsock. A transport option overrides both. + o := options{listen: vsockListen, dial: vsockDial} + for _, apply := range opts { + apply(&o) + } + + resolver, err := newRouteResolver(cfg, &o) + if err != nil { + return nil, err + } + + p, err := byteproxy.NewAWSProxy(proxyName, cfg.Vsockproxy.BasePort, cfg.Vsockproxy.MaxConns, + func(svc string, port uint32) (byteproxy.AWSServiceBinding, error) { + // Endpoint is informational only (byteproxy logs it). In vsock mode + // the real kms..amazonaws.com:443 target is resolved per + // connection in resolver.dial from the AWSRoute header, so record a + // descriptor here rather than the unresolved "{region}" template. + var endpoint string + if cfg.Provider.AWSKMS.Localstack.Enabled { + endpoint = o.tcpLocalstackEndpoint + } else { + endpoint = svc + " (region resolved per connection)" + } + return byteproxy.AWSServiceBinding{ + Endpoint: endpoint, + Listen: func() (net.Listener, error) { return o.listen(port) }, + Dial: func(ctx context.Context, inbound net.Conn) (net.Conn, error) { + return resolver.dial(ctx, svc, inbound) + }, + }, nil + }) + if err != nil { + return nil, fmt.Errorf("vsockproxy: %w", err) + } + return p, nil +} diff --git a/internal/vsockproxy/vsockproxy_stub.go b/internal/vsockproxy/vsockproxy_stub.go new file mode 100644 index 0000000..a199e1f --- /dev/null +++ b/internal/vsockproxy/vsockproxy_stub.go @@ -0,0 +1,72 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !linux + +package vsockproxy + +import ( + "context" + "errors" + "fmt" + "net" + + "github.com/circlefin/arc-remote-signer/internal/common/byteproxy" +) + +// proxyName is the lifecycle.Runnable name reported by the stub; matches +// the Linux build so logs are identical across platforms. +const proxyName = "host-vsockproxy" + +// Provider mirrors the Linux Provider so cross-platform callers compile. +type Provider interface { + byteproxy.Proxy +} + +// New creates a host-side proxy. On non-Linux only the TCP transport is +// available; selecting WithVsockTransport is a compile-time error +// because the option lives in the Linux-only transport_vsock.go. +func New(cfg *Config, opts ...Option) (Provider, error) { + if cfg == nil { + return nil, errors.New("vsockproxy: nil config") + } + o := options{} + for _, apply := range opts { + apply(&o) + } + if o.transport != byteproxy.TransportTCP { + return nil, errors.New("vsockproxy: non-Linux requires WithTCPTransport()") + } + resolver, err := newRouteResolver(cfg, &o) + if err != nil { + return nil, err + } + p, err := byteproxy.NewAWSProxy(proxyName, cfg.Vsockproxy.BasePort, cfg.Vsockproxy.MaxConns, + func(svc string, port uint32) (byteproxy.AWSServiceBinding, error) { + endpoint := svc + " (region resolved per connection)" + if cfg.Provider.AWSKMS.Localstack.Enabled { + endpoint = o.tcpLocalstackEndpoint + } + return byteproxy.AWSServiceBinding{ + Endpoint: endpoint, + Listen: func() (net.Listener, error) { return o.listen(port) }, + Dial: func(ctx context.Context, inbound net.Conn) (net.Conn, error) { + return resolver.dial(ctx, svc, inbound) + }, + }, nil + }) + if err != nil { + return nil, fmt.Errorf("vsockproxy: %w", err) + } + return p, nil +} diff --git a/internal/vsockproxy/vsockproxy_stub_test.go b/internal/vsockproxy/vsockproxy_stub_test.go new file mode 100644 index 0000000..94903ff --- /dev/null +++ b/internal/vsockproxy/vsockproxy_stub_test.go @@ -0,0 +1,50 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !linux + +package vsockproxy + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// stubCfg returns a valid config for the TCP stub. +func stubCfg() *Config { + return &Config{ + Vsockproxy: Vsockproxy{BasePort: 0, MaxConns: 1}, + Provider: ProviderConfig{AWSKMS: ProviderAWSKMSConfig{ + Arns: []string{"arn:aws:kms:us-east-1:000000000000:key/test"}, + }}, + } +} + +func TestNew_NilConfigErrors(t *testing.T) { + _, err := New(nil, WithTCPTransport()) + require.Error(t, err) + require.Contains(t, err.Error(), "nil config") +} + +func TestNew_NoTransportOptionErrors(t *testing.T) { + _, err := New(stubCfg()) + require.Error(t, err) + require.Contains(t, err.Error(), "non-Linux requires WithTCPTransport") +} + +func TestNew_TCPTransportSucceeds(t *testing.T) { + p, err := New(stubCfg(), WithTCPTransport()) + require.NoError(t, err) + require.NotNil(t, p) +} diff --git a/internal/vsockproxy/vsockproxy_test.go b/internal/vsockproxy/vsockproxy_test.go new file mode 100644 index 0000000..34f239d --- /dev/null +++ b/internal/vsockproxy/vsockproxy_test.go @@ -0,0 +1,317 @@ +// Copyright (c) 2026, Circle Internet Group, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux + +package vsockproxy + +import ( + "context" + "errors" + "net" + "strings" + "testing" + "time" + + "github.com/circlefin/arc-remote-signer/internal/common/byteproxy" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type VsockProxyTestSuite struct { + suite.Suite +} + +func (s *VsockProxyTestSuite) newLocalListener() net.Listener { + s.T().Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + s.Require().NoError(err) + s.T().Cleanup(func() { _ = ln.Close() }) + return ln +} + +// vsockCfg returns a vsock-mode Config with a single allowed us-east-1 ARN. +func vsockCfg(basePort uint32, maxConns int) *Config { + return &Config{ + Vsockproxy: Vsockproxy{BasePort: basePort, MaxConns: maxConns}, + Provider: ProviderConfig{ + AWSKMS: ProviderAWSKMSConfig{ + Arns: []string{"arn:aws:kms:us-east-1:000000000000:key/abc"}, + }, + }, + } +} + +func (s *VsockProxyTestSuite) TestNew() { + failingListen := func(_ uint32) (net.Listener, error) { return nil, errors.New("bind failed") } + failingDial := func(_ context.Context, _ string) (net.Conn, error) { return nil, errors.New("not used") } + okListen := func(_ uint32) (net.Listener, error) { return s.newLocalListener(), nil } + + tests := map[string]struct { + cfg *Config + opts []Option + wantErr bool + wantErrContains string + }{ + "NilConfig": { + cfg: nil, + opts: []Option{WithListen(failingListen), WithDial(failingDial)}, + wantErr: true, + }, + "InvalidMaxConns_VsockTransport": { + cfg: vsockCfg(30000, 0), + opts: []Option{WithVsockTransport(), WithListen(okListen), WithDial(failingDial)}, + wantErr: true, + }, + "BindFailure_VsockTransport": { + cfg: vsockCfg(30000, 4), + opts: []Option{WithVsockTransport(), WithListen(failingListen), WithDial(failingDial)}, + wantErr: true, + }, + "EnabledHappyPath_VsockTransport": { + cfg: vsockCfg(30000, 4), + opts: []Option{WithVsockTransport(), WithListen(okListen), WithDial(failingDial)}, + wantErr: false, + }, + "EnabledHappyPath_TCPTransport": { + cfg: vsockCfg(30000, 4), + opts: []Option{WithTCPTransport(), WithListen(okListen), WithDial(failingDial)}, + wantErr: false, + }, + // No transport option falls back to the vsock zero value; the + // vsock-mode Config (ARN present) lets region resolution succeed. + "NoTransportOption_DefaultsToVsock": { + cfg: vsockCfg(30000, 4), + opts: []Option{WithListen(okListen), WithDial(failingDial)}, + wantErr: false, + }, + // vsock wins as the last transport option, so a vsock-mode Config + // (ARN present for region resolution) is required for success. + "BothTransportOptions_LastWins_Succeeds": { + cfg: vsockCfg(30000, 4), + opts: []Option{WithTCPTransport(), WithVsockTransport(), WithListen(okListen), WithDial(failingDial)}, + wantErr: false, + }, + "NoArns_VsockTransport_Errors": { + cfg: &Config{Vsockproxy: Vsockproxy{BasePort: 30000, MaxConns: 4}}, + opts: []Option{WithVsockTransport(), WithListen(okListen), WithDial(failingDial)}, + wantErr: true, + wantErrContains: "provider.awskms.arns", + }, + "MalformedArn_VsockTransport_Errors": { + cfg: &Config{ + Vsockproxy: Vsockproxy{BasePort: 30000, MaxConns: 4}, + Provider: ProviderConfig{ + AWSKMS: ProviderAWSKMSConfig{Arns: []string{"not-an-arn"}}, + }, + }, + opts: []Option{WithVsockTransport(), WithListen(okListen), WithDial(failingDial)}, + wantErr: true, + wantErrContains: "not a valid ARN", + }, + // A syntactically valid KMS ARN whose region segment is empty parses + // cleanly but must still be rejected by the empty-region guard β€” a + // distinct path from the arn.Parse failure above. + "ValidArnNoRegion_VsockTransport_Errors": { + cfg: &Config{ + Vsockproxy: Vsockproxy{BasePort: 30000, MaxConns: 4}, + Provider: ProviderConfig{ + AWSKMS: ProviderAWSKMSConfig{Arns: []string{"arn:aws:kms::000000000000:key/test"}}, + }, + }, + opts: []Option{WithVsockTransport(), WithListen(okListen), WithDial(failingDial)}, + wantErr: true, + wantErrContains: "has no region", + }, + } + for name, tt := range tests { + s.Run(name, func() { + p, err := New(tt.cfg, tt.opts...) + if tt.wantErr { + s.Require().Error(err) + if tt.wantErrContains != "" { + s.Contains(err.Error(), tt.wantErrContains) + } + return + } + s.Require().NoError(err) + s.Require().NotNil(p) + }) + } +} + +func (s *VsockProxyTestSuite) TestNew_PortMapping() { + s.Require().NotEmpty(byteproxy.AWSServiceNames) + + var seenPorts []uint32 + listen := func(port uint32) (net.Listener, error) { + seenPorts = append(seenPorts, port) + return s.newLocalListener(), nil + } + dial := func(_ context.Context, _ string) (net.Conn, error) { return nil, errors.New("dial unused") } + + const base uint32 = 40000 + p, err := New( + vsockCfg(base, 4), + WithVsockTransport(), WithListen(listen), WithDial(dial), + ) + s.Require().NoError(err) + s.T().Cleanup(func() { _ = p.Shutdown(context.Background()) }) + + wantPorts := make([]uint32, len(byteproxy.AWSServiceNames)) + for i := range byteproxy.AWSServiceNames { + wantPorts[i] = base + uint32(i) //nolint:gosec // bounded by len(AWSServiceNames) + } + s.Equal(wantPorts, seenPorts) + s.Len(seenPorts, len(byteproxy.AWSServiceNames), "listen must be called exactly once per service") +} + +func (s *VsockProxyTestSuite) TestNew_EndpointTemplateSubstitution() { + s.Require().NotEmpty(byteproxy.AWSServiceNames) + for _, name := range byteproxy.AWSServiceNames { + tmpl, err := EndpointTemplate(name) + s.Require().NoError(err) + s.Contains(tmpl, "{region}") + } + const region = "eu-west-2" + listeners := make([]net.Listener, len(byteproxy.AWSServiceNames)) + for i := range byteproxy.AWSServiceNames { + listeners[i] = s.newLocalListener() + } + idx := 0 + listen := func(_ uint32) (net.Listener, error) { + ln := listeners[idx] + idx++ + return ln, nil + } + endpointsCh := make(chan string, len(byteproxy.AWSServiceNames)) + dial := func(_ context.Context, endpoint string) (net.Conn, error) { + endpointsCh <- endpoint + return nil, errors.New("dial unused") + } + p, err := New( + &Config{ + Vsockproxy: Vsockproxy{BasePort: 40000, MaxConns: 4}, + Provider: ProviderConfig{ + AWSKMS: ProviderAWSKMSConfig{ + Arns: []string{"arn:aws:kms:" + region + ":000000000000:key/abc"}, + }, + }, + }, + WithVsockTransport(), WithListen(listen), WithDial(dial), + ) + s.Require().NoError(err) + go func() { _ = p.Run() }() + s.T().Cleanup(func() { _ = p.Shutdown(context.Background()) }) + for i, ln := range listeners { + c, derr := net.Dial("tcp", ln.Addr().String()) + s.Require().NoError(derr) + s.T().Cleanup(func() { _ = c.Close() }) + s.Require().NoError(byteproxy.WriteAWSRoute(c, byteproxy.AWSRoute{ + Service: byteproxy.AWSServiceNames[i], + Region: region, + })) + } + seen := make([]string, 0, len(byteproxy.AWSServiceNames)) + for range byteproxy.AWSServiceNames { + select { + case ep := <-endpointsCh: + seen = append(seen, ep) + case <-time.After(2 * time.Second): + s.FailNow("timed out waiting for dial to capture endpoint") + } + } + for _, ep := range seen { + s.NotContains(ep, "{region}") + s.Contains(ep, region) + s.True(strings.HasSuffix(ep, ":443")) + } +} + +// TestEnsureServiceParity covers the drift case the function exists to +// catch: a service name listed in byteproxy.AWSServiceNames without a +// corresponding entry in endpointTemplates. The check is fed via +// parameters (rather than the package globals) precisely so this case +// is reachable from tests. +func TestEnsureServiceParity(t *testing.T) { + tests := map[string]struct { + services []string + templates map[string]string + wantErr bool + wantErrMsg string + }{ + "all services covered": { + services: []string{"kms"}, + templates: map[string]string{"kms": "kms.{region}.amazonaws.com:443"}, + wantErr: false, + }, + "service missing template": { + services: []string{"kms", "secretsmanager"}, + templates: map[string]string{"kms": "kms.{region}.amazonaws.com:443"}, + wantErr: true, + wantErrMsg: `"secretsmanager"`, + }, + "empty services list": { + services: nil, + templates: map[string]string{}, + wantErr: false, + }, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + err := ensureServiceParity(tt.services, tt.templates) + if tt.wantErr { + require.Error(t, err) + require.Contains(t, err.Error(), tt.wantErrMsg) + } else { + require.NoError(t, err) + } + }) + } +} + +// TestWithTransport_LastWins verifies that passing both transport options +// is not an error and that the last option applied selects the transport, +// including its TCP upstream endpoint, so the selected kind and its dialer +// stay coherent. +func (s *VsockProxyTestSuite) TestWithTransport_LastWins() { + s.Run("tcp last", func() { + o := options{} + WithVsockTransport()(&o) + WithTCPTransport()(&o) + s.Equal(byteproxy.TransportTCP, o.transport) + s.Equal("localstack:4566", o.tcpLocalstackEndpoint) + }) + s.Run("vsock last", func() { + o := options{} + WithTCPTransport()(&o) + WithVsockTransport()(&o) + s.Equal(byteproxy.TransportVsock, o.transport) + }) +} + +// TestNew_UnhandledTransportKind covers resolveEndpoint's default guard: +// an out-of-range transport (e.g. a future byteproxy.TransportKind added +// without a matching resolveEndpoint case) must surface an error from New. +// Injected via a raw Option so the guard stays protected from silent +// removal. A non-vsock kind skips region resolution, so no ARN is needed. +func TestNew_UnhandledTransportKind(t *testing.T) { + badTransport := Option(func(o *options) { o.transport = byteproxy.TransportKind(99) }) + _, err := New(&Config{Vsockproxy: Vsockproxy{BasePort: 30000, MaxConns: 4}}, badTransport) + require.Error(t, err) + require.Contains(t, err.Error(), "unhandled transport kind") +} + +func TestVsockProxyTestSuite(t *testing.T) { + suite.Run(t, &VsockProxyTestSuite{Suite: suite.Suite{}}) +} diff --git a/proto/arc/enclave/v1/enclave.proto b/proto/arc/enclave/v1/enclave.proto index 7c3f375..8a3856c 100644 --- a/proto/arc/enclave/v1/enclave.proto +++ b/proto/arc/enclave/v1/enclave.proto @@ -7,10 +7,70 @@ option go_package = "github.com/circlefin/arc-remote-signer/proto/pb"; import "buf/validate/validate.proto"; service EnclaveService { + rpc Initialize(InitializeRequest) returns (InitializeResponse) {} rpc GenerateKey(GenerateKeyRequest) returns (GenerateKeyResponse) {} rpc GetPublicKey(GetPublicKeyRequest) returns (GetPublicKeyResponse) {} rpc SignMessage(SignMessageRequest) returns (SignMessageResponse) {} - rpc GetAttestation(GetAttestationRequest) returns (GetAttestationResponse) {} +} + +message AwsCredentials { + string access_key_id = 1 [(buf.validate.field).string.min_len = 1]; + string secret_access_key = 2 [(buf.validate.field).string.min_len = 1]; + // session_token is required (min_len = 1). Nitro Enclave deployments + // always source credentials from STS (instance role / attestation-based + // role assumption), which mints a session token; long-lived IAM user + // credentials without a session token are intentionally rejected. + string session_token = 3 [(buf.validate.field).string.min_len = 1]; + string region = 4 [(buf.validate.field).string.min_len = 1]; +} + +message InitializeRequest { + AwsCredentials credentials = 1 [(buf.validate.field).required = true]; + // kms_key_arns is the multi-region list of KMS key ARNs the enclave + // exercises during Initialize to verify reachability of AWS KMS (via + // awsproxy) and that the host-supplied credentials authenticate. Each ARN + // gets a region-routed KMS client, and failed calls continue through the + // ordered list. At least one ARN required. + repeated string kms_key_arns = 2 [(buf.validate.field).repeated = {min_items: 1, items: {string: {min_len: 1}}}]; + // kms_localstack_enabled selects LocalStack instead of AWS KMS. Nitro + // Enclave deployments must keep this value false. + bool kms_localstack_enabled = 3; + // key_source specifies whether Initialize recovers a stored key or + // generates a new key and returns its envelope for host persistence. + oneof key_source { + option (buf.validate.oneof).required = true; + + SecretEnvelope existing_key = 4; + Algorithm generate_new = 5 [(buf.validate.field).enum = { + defined_only: true, + not_in: [0] + }]; + } +} + +message InitializeResponse { + bytes public_key = 1 [(buf.validate.field).bytes.min_len = 1]; + bytes attestation_document = 2; + // secret_envelope is present only when generate_new was requested. + SecretEnvelope secret_envelope = 3; +} + +// SecretEnvelope contains the wrapped signing key. KMS encrypts the AES data +// key. AES-GCM encrypts the private key with that data key. The host stores the +// envelope and supplies it to Initialize after a restart. +message SecretEnvelope { + Algorithm algorithm = 1 [(buf.validate.field).enum = { + defined_only: true, + not_in: [0] + }]; + // kms_encrypted_data_key is the standard KMS ciphertext of the AES data + // key. The enclave calls KMS Decrypt on it to recover the data key. + bytes kms_encrypted_data_key = 2 [(buf.validate.field).bytes.min_len = 1]; + // encrypted_private_key is the AES-GCM ciphertext of the serialized + // private key. + bytes encrypted_private_key = 3 [(buf.validate.field).bytes.min_len = 1]; + // nonce is the AES-GCM nonce for encrypted_private_key. + bytes nonce = 4 [(buf.validate.field).bytes.min_len = 1]; } enum Algorithm { @@ -20,52 +80,41 @@ enum Algorithm { } message GenerateKeyRequest { + reserved 2; + reserved "enclave_encrypted_data_key"; Algorithm algorithm = 1 [(buf.validate.field).enum = { defined_only: true, not_in: [0] }]; - bytes enclave_encrypted_data_key = 2 [(buf.validate.field).bytes.min_len = 1]; } message GenerateKeyResponse { + reserved 2, 3; + reserved "encrypted_private_key", "nonce"; bytes public_key = 1 [(buf.validate.field).bytes.min_len = 1]; - bytes encrypted_private_key = 2 [(buf.validate.field).bytes.min_len = 1]; - bytes nonce = 3 [(buf.validate.field).bytes.min_len = 1]; -} - -message EncryptedKeyMaterial { - bytes encrypted_private_key = 1 [(buf.validate.field).bytes.min_len = 1]; - bytes enclave_encrypted_data_key = 2 [(buf.validate.field).bytes.min_len = 1]; - bytes nonce = 3 [(buf.validate.field).bytes.min_len = 1]; + // secret_envelope is the ready-to-store wrapped key the host persists + // verbatim to Secrets Manager. + SecretEnvelope secret_envelope = 4 [(buf.validate.field).required = true]; } message GetPublicKeyRequest { - Algorithm algorithm = 1 [(buf.validate.field).enum = { - defined_only: true, - not_in: [0] - }]; - EncryptedKeyMaterial encrypted_key_material = 2 [(buf.validate.field).required = true]; + reserved 1, 2, 3; + reserved "algorithm", "encrypted_key_material", "secret_envelope"; } message GetPublicKeyResponse { bytes public_key = 1 [(buf.validate.field).bytes.min_len = 1]; + // attestation_document binds public_key to the Nitro enclave. It is empty + // when Nitro mode is disabled. + bytes attestation_document = 2; } message SignMessageRequest { - Algorithm algorithm = 1 [(buf.validate.field).enum = { - defined_only: true, - not_in: [0] - }]; - EncryptedKeyMaterial encrypted_key_material = 2 [(buf.validate.field).required = true]; + reserved 1, 2, 4; + reserved "algorithm", "encrypted_key_material", "secret_envelope"; bytes message = 3 [(buf.validate.field).bytes.min_len = 1]; } message SignMessageResponse { bytes signature = 1 [(buf.validate.field).bytes.min_len = 1]; } - -message GetAttestationRequest {} - -message GetAttestationResponse { - bytes attestation_document = 1 [(buf.validate.field).bytes.min_len = 1]; -} diff --git a/proto/arc/signer/v1/signer.proto b/proto/arc/signer/v1/signer.proto index c5691d3..4fdd6b0 100644 --- a/proto/arc/signer/v1/signer.proto +++ b/proto/arc/signer/v1/signer.proto @@ -15,6 +15,9 @@ service SignerService { message PublicKeyRequest {} message PublicKeyResponse { bytes public_key = 1; + // attestation_document binds public_key to the Nitro enclave. It is empty + // when Nitro mode is disabled. + bytes attestation_document = 2; } message SignRequest { diff --git a/scripts/common.sh b/scripts/common.sh index 4f394ee..ceea2f7 100755 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -21,3 +21,8 @@ export APP_ENV=${APP_ENV:-"dev"} # Gets all the development environment variables set -o allexport && source "${PROJECT_DIR}/configs/.env.dev" && set +o allexport + +# ENABLE_ENCLAVE is the single switch for Nitro mode (see configs/.env.dev). +# Derive the host app's config flag from it β€” mirroring docker/run.sh in prod β€” +# so the local app and the docker-compose services never disagree. +export APP_PROVIDER_ENCLAVE_NITROENCLAVE_ENABLED="${ENABLE_ENCLAVE:-false}" diff --git a/scripts/merge-production-coverage.sh b/scripts/merge-production-coverage.sh new file mode 100755 index 0000000..62b7cdf --- /dev/null +++ b/scripts/merge-production-coverage.sh @@ -0,0 +1,190 @@ +#!/usr/bin/env bash + +set -eu + +# Use this command: +# merge-production-coverage.sh BASE_PROFILE PRODUCTION_PROFILE PACKAGE_PREFIX +# +# PACKAGE_PREFIX is a source-directory suffix, such as internal/app. +# You can use PACKAGE_PREFIX with or without a trailing slash. +# The command rejects the same path for both profiles. +# The command rejects missing, empty, or malformed profiles. +# The command also rejects unreadable profiles. +# The command also rejects a production profile without PACKAGE_PREFIX coverage. +# Both profiles must use the same coverage mode. +# The command changes BASE_PROFILE in place only after a successful merge. +# BASE_PROFILE keeps its coverage for files in both profiles. +# The command adds production-only files below PACKAGE_PREFIX to BASE_PROFILE. +# Each error starts with a stable diagnostic identifier. +# The brackets contain context. The context format is not stable. +# Only the identifier is stable. + +report_error() { + local identifier="$1" + local context="$2" + local message="$3" + + printf '%s [%s]: %s\n' "$identifier" "$context" "$message" >&2 +} + +if [ -z "${1:-}" ]; then + report_error 'MISSING_ARGUMENT' 'BASE_PROFILE' \ + 'Give the path to the base coverage profile.' + exit 1 +fi +if [ -z "${2:-}" ]; then + report_error 'MISSING_ARGUMENT' 'PRODUCTION_PROFILE' \ + 'Give the path to the production coverage profile.' + exit 1 +fi +if [ -z "${3:-}" ]; then + report_error 'MISSING_ARGUMENT' 'PACKAGE_PREFIX' \ + 'Give the package prefix.' + exit 1 +fi + +coverage="$1" +production="$2" +package_prefix="${3%/}/" + +if [ "$coverage" = "$production" ]; then + report_error 'PROFILE_PATHS_MATCH' "$coverage" \ + 'The base and production profiles must use different paths.' + exit 1 +fi + +if [ ! -s "$coverage" ]; then + report_error 'BASE_PROFILE_MISSING_OR_EMPTY' "$coverage" \ + 'The base coverage profile is missing or empty.' + exit 1 +fi +if [ ! -s "$production" ]; then + report_error 'PRODUCTION_PROFILE_MISSING_OR_EMPTY' "$production" \ + 'The production coverage profile is missing or empty.' + exit 1 +fi +if [ ! -r "$coverage" ]; then + report_error 'BASE_PROFILE_UNREADABLE' "$coverage" \ + 'The process cannot read the base coverage profile.' + exit 1 +fi +if [ ! -r "$production" ]; then + report_error 'PRODUCTION_PROFILE_UNREADABLE' "$production" \ + 'The process cannot read the production coverage profile.' + exit 1 +fi + +coverage_mode="$(sed -n '1p' "$coverage")" +production_mode="$(sed -n '1p' "$production")" + +validate_mode_header() { + local profile="$1" + local mode="$2" + + case "$mode" in + 'mode: set' | 'mode: count' | 'mode: atomic') ;; + *) + report_error 'INVALID_MODE_HEADER' "$profile" \ + "The profile has an invalid mode header: $mode" + exit 1 + ;; + esac +} + +validate_mode_header "$coverage" "$coverage_mode" +validate_mode_header "$production" "$production_mode" + +if [ "$coverage_mode" != "$production_mode" ]; then + report_error 'COVERAGE_MODE_MISMATCH' "$coverage | $production" \ + "The profiles use different coverage modes: $coverage_mode and $production_mode." + exit 1 +fi + +additions='' +merged='' +additions_template="${coverage}.additions.XXXXXX" +merged_template="${coverage}.merged.XXXXXX" + +remove_temp_profiles() { + if [ -n "$additions" ]; then + rm -f "$additions" + fi + if [ -n "$merged" ]; then + rm -f "$merged" + fi +} + +trap remove_temp_profiles EXIT + +if ! additions="$(mktemp "$additions_template")"; then + report_error 'TEMP_PROFILE_CREATE_FAILED' "$additions_template" \ + 'The command cannot create a temporary coverage profile.' + exit 1 +fi +if ! merged="$(mktemp "$merged_template")"; then + report_error 'TEMP_PROFILE_CREATE_FAILED' "$merged_template" \ + 'The command cannot create a temporary coverage profile.' + exit 1 +fi + +merge_status=0 +awk -v package_prefix="$package_prefix" ' + function source_file(line, fields, source) { + split(line, fields, " ") + source = fields[1] + sub(/:[0-9].*$/, "", source) + return source + } + + function in_package(source) { + return index(source, package_prefix) == 1 || + index(source, "/" package_prefix) > 0 + } + + FNR == 1 { next } + + FILENAME == ARGV[1] { + existing[source_file($0)] = 1 + next + } + + { + source = source_file($0) + if (in_package(source)) { + found_package = 1 + if (!(source in existing)) { + print + } + } + } + + END { + if (!found_package) { + exit 3 + } + } +' "$coverage" "$production" >"$additions" || merge_status=$? +case "$merge_status" in + 0) ;; + 3) + report_error 'PACKAGE_COVERAGE_MISSING' "$production" \ + "The production profile does not contain coverage for $package_prefix." + exit 1 + ;; + *) + report_error 'COVERAGE_MERGE_FAILED' "$production -> $coverage" \ + 'The command cannot merge the production profile into the base profile.' + exit 1 + ;; +esac + +if ! cat "$coverage" "$additions" >"$merged"; then + report_error 'MERGED_PROFILE_WRITE_FAILED' "$merged" \ + 'The command cannot write the merged coverage profile.' + exit 1 +fi +if ! mv "$merged" "$coverage"; then + report_error 'BASE_PROFILE_REPLACE_FAILED' "$merged -> $coverage" \ + 'The command cannot replace the base coverage profile.' + exit 1 +fi diff --git a/scripts/smoke-eif.sh b/scripts/smoke-eif.sh index 294f912..917cb86 100755 --- a/scripts/smoke-eif.sh +++ b/scripts/smoke-eif.sh @@ -1,49 +1,210 @@ #!/usr/bin/env bash -# smoke-eif.sh runs smoke tests against a real Nitro Enclave EIF. -# Requires a 2xlarge-nitro runner (nitro-cli + /dev/nitro_enclaves available). +# smoke-eif.sh runs smoke tests against the built signer-with-enclave image. +# Requires a 2xlarge-nitro-dind runner (Docker + Nitro Enclaves available). # # Usage: -# EIF_PATH=./enclave.eif scripts/smoke-eif.sh +# IMAGE_REF=nitro-enclave-signer/signer-with-enclave:smoke scripts/smoke-eif.sh # # Environment variables: -# EIF_PATH Path to the enclave.eif file (required) -# ENCLAVE_CID VSOCK CID for the enclave (default: 16) -# ENCLAVE_CPU CPU count for the enclave (default: 2) -# ENCLAVE_MEMORY Memory in MiB for the enclave (default: 4096) -set -eux +# IMAGE_REF Docker image reference to run (required) +# AWS_ACCESS_KEY_ID AWS access key from CI OIDC credentials (required) +# AWS_SECRET_ACCESS_KEY AWS secret key from CI OIDC credentials (required) +# AWS_SESSION_TOKEN AWS session token from CI OIDC credentials (required) +# APP_SERVICE_SIGNER_KEYID Signer key identifier (required) +# APP_PROVIDER_AWSKMS_ARNS Comma-separated AWS KMS key ARNs (required) +# ENCLAVE_CPU CPU count for the enclave (default: 2) +# ENCLAVE_MEMORY Memory in MiB for the enclave (default: 4096) +set -eu -export PROJECT_DIR="$(cd "$(dirname "$0")/.." && pwd)" -export BIN_DIR="${PROJECT_DIR}/bin" +PROJECT_DIR="$(cd "$(dirname "$0")/.." && pwd)" +export PROJECT_DIR -: "${EIF_PATH:?EIF_PATH must be set to the path of the enclave.eif file}" -ENCLAVE_CID="${ENCLAVE_CID:-16}" +: "${IMAGE_REF:?IMAGE_REF must be set to the signer-with-enclave image to run}" +: "${AWS_ACCESS_KEY_ID:?AWS_ACCESS_KEY_ID must be set}" +: "${AWS_SECRET_ACCESS_KEY:?AWS_SECRET_ACCESS_KEY must be set}" +: "${AWS_SESSION_TOKEN:?AWS_SESSION_TOKEN must be set}" +: "${APP_SERVICE_SIGNER_KEYID:?APP_SERVICE_SIGNER_KEYID must be set}" +: "${APP_PROVIDER_AWSKMS_ARNS:?APP_PROVIDER_AWSKMS_ARNS must be set}" ENCLAVE_CPU="${ENCLAVE_CPU:-2}" ENCLAVE_MEMORY="${ENCLAVE_MEMORY:-4096}" +CONTAINER_NAME="${CONTAINER_NAME:-smoke-signer-with-enclave}" +command -v timeout >/dev/null 2>&1 || { + echo "timeout command is required on the Nitro runner" >&2 + exit 1 +} +set -x + +run_bounded() { + local timeout_s="$1" + shift + + timeout --signal=TERM --kill-after=5s "${timeout_s}s" "$@" +} + +remove_container() { + local timeout_s="$1" + local container_name="$2" + local output + local status + + output="$(run_bounded "$timeout_s" docker rm -f "$container_name" 2>&1)" && return 0 + status=$? + case "$output" in + *"No such container"* | *"No such object"*) return 0 ;; + esac + + echo "warning: failed to remove container $container_name: $output" >&2 + return "$status" +} + +terminate_enclaves() { + local cleanup_container="${CONTAINER_NAME}-nitro-cleanup" + local cleanup_exit_code + local cleanup_failed=0 + + # The Nitro device belongs to the dind daemon, so use the tested image + # rather than assuming nitro-cli and the device exist on the runner host. + if ! remove_container 15 "$cleanup_container"; then + return 1 + fi -APP_PID="" -ENCLAVE_ID="" + if ! run_bounded 15 docker run -d \ + --name "$cleanup_container" \ + --device=/dev/nitro_enclaves \ + --security-opt seccomp=unconfined \ + --entrypoint nitro-cli \ + "$IMAGE_REF" terminate-enclave --all >/dev/null; then + echo "warning: failed to start bounded Nitro enclave cleanup" >&2 + cleanup_failed=1 + elif ! cleanup_exit_code="$(run_bounded 15 docker wait "$cleanup_container")"; then + echo "warning: Nitro enclave cleanup failed or did not finish within 15s" >&2 + cleanup_failed=1 + elif [ "$cleanup_exit_code" != "0" ]; then + echo "warning: Nitro enclave cleanup exited with code $cleanup_exit_code" >&2 + cleanup_failed=1 + fi + + if [ "$cleanup_failed" -ne 0 ]; then + run_bounded 10 docker logs "$cleanup_container" >&2 || \ + echo "warning: failed to read Nitro enclave cleanup logs" >&2 + fi + + if ! remove_container 15 "$cleanup_container"; then + cleanup_failed=1 + fi + + return "$cleanup_failed" +} + +verify_image_contents() { + local entrypoint + + entrypoint="$(run_bounded 15 docker inspect --format '{{json .Config.Entrypoint}}' "$IMAGE_REF")" + test "$entrypoint" = '["/usr/local/circle/run.sh"]' + + run_bounded 30 docker run --rm \ + --entrypoint /bin/sh \ + "$IMAGE_REF" -eu -c ' + test -x /usr/local/circle/app + test -f /usr/local/circle/config.yaml + test -f /usr/local/circle/enclave.eif + test -x /usr/local/circle/run.sh + test ! -e /usr/local/circle/enclave + test ! -e /usr/local/circle/configs + test ! -e /usr/local/circle/deployments + test ! -e /usr/local/circle/run_proxy.sh + test ! -e /usr/local/circle/run_enclave.dev.sh + if command -v socat >/dev/null 2>&1; then exit 1; fi + nitro-cli --version >/dev/null + if grep -aF "dev-session-token-placeholder" /usr/local/circle/app; then exit 1; fi + if grep -aE "(localhost|localstack):4566" /usr/local/circle/app; then exit 1; fi + if grep -Eqi "localstack|baseURL|nitroEnclave|ENABLE_ENCLAVE" /usr/local/circle/config.yaml; then exit 1; fi + if grep -Eq "ENABLE_ENCLAVE|APP_PROVIDER_ENCLAVE_NITROENCLAVE_ENABLED|enclave_enabled|EXTRA_NITRO_CLI_FLAGS|--debug-mode" /usr/local/circle/run.sh; then exit 1; fi + ' +} cleanup() { - if [ -n "$APP_PID" ] && kill -0 "$APP_PID" 2>/dev/null; then - kill -TERM "$APP_PID" || true - wait "$APP_PID" 2>/dev/null || true + local status=$? + local cleanup_failed=0 + local inspect_output + local container_exists=false + local container_absent=false + local container_exit_code + + trap - EXIT INT TERM + set +e + + if inspect_output="$(run_bounded 10 docker inspect "$CONTAINER_NAME" 2>&1)"; then + container_exists=true + else + case "$inspect_output" in + *"No such container"* | *"No such object"*) container_absent=true ;; + *) + echo "warning: failed to inspect container $CONTAINER_NAME: $inspect_output" >&2 + cleanup_failed=1 + ;; + esac + fi + + if [ "$container_exists" = true ]; then + if ! run_bounded 85 docker stop --time 75 "$CONTAINER_NAME" >/dev/null; then + echo "warning: signer container did not stop gracefully" >&2 + cleanup_failed=1 + elif ! container_exit_code="$(run_bounded 10 docker inspect --format '{{.State.ExitCode}}' "$CONTAINER_NAME")"; then + echo "warning: failed to inspect signer container exit code" >&2 + cleanup_failed=1 + elif [ "$container_exit_code" != "0" ]; then + echo "warning: signer container exited with code $container_exit_code" >&2 + cleanup_failed=1 + fi + if ! run_bounded 10 docker logs "$CONTAINER_NAME"; then + echo "warning: failed to read signer container logs" >&2 + fi + fi + + # If inspect failed for an unknown reason, still attempt force-removal so a + # transient daemon error cannot leave a credential-bearing container alive. + if [ "$container_absent" = false ]; then + if ! remove_container 15 "$CONTAINER_NAME"; then + cleanup_failed=1 + fi + fi + + if ! terminate_enclaves; then + cleanup_failed=1 fi - if [ -n "$ENCLAVE_ID" ]; then - sudo nitro-cli terminate-enclave --enclave-id "$ENCLAVE_ID" 2>/dev/null || true + + if [ "$status" -eq 0 ] && [ "$cleanup_failed" -ne 0 ]; then + status=1 fi - jobs -p | xargs -r kill 2>/dev/null || true + exit "$status" } trap cleanup EXIT -# Polls a command until it succeeds or the timeout (in seconds) is reached. -# Probe stderr is shown on timeout to aid diagnosis. +handle_signal() { + local status="$1" + + trap - INT TERM + exit "$status" +} +trap 'handle_signal 130' INT +trap 'handle_signal 143' TERM + +# Polls a command while the signer container is running, until it succeeds or +# the timeout (in seconds) is reached. Probe stderr is shown on timeout. wait_for() { local desc="$1" local max="${2:-30}" shift 2 local out for i in $(seq 1 "$max"); do + if [ "$(docker inspect --format '{{.State.Running}}' "$CONTAINER_NAME" 2>/dev/null)" != "true" ]; then + local state + state="$(docker inspect --format 'status={{.State.Status}} exit_code={{.State.ExitCode}}' "$CONTAINER_NAME" 2>/dev/null || echo unavailable)" + echo "$CONTAINER_NAME exited before $desc became ready ($state)" >&2 + return 1 + fi if out="$("$@" 2>&1)"; then echo "$desc ready after ${i}s" return 0 @@ -55,36 +216,52 @@ wait_for() { cd "$PROJECT_DIR" || exit 1 -# Set up VSOCK bridges for enclave communication. -# Port 8000: env vars the enclave reads on startup; '|||'-delimited (APP_ENV only, rest empty). -socat VSOCK-LISTEN:8000,reuseaddr,fork SYSTEM:"printf '%s|||' '${APP_ENV}'" & -# Port 8001: enclave log output forwarded to host stdout -socat -u VSOCK-LISTEN:8001,reuseaddr,fork EXEC:"cat",stderr & - -# Start the enclave in debug mode so PCR values are all zeros (predictable for CI testing) -sudo nitro-cli run-enclave \ - --cpu-count "$ENCLAVE_CPU" \ - --memory "$ENCLAVE_MEMORY" \ - --enclave-cid "$ENCLAVE_CID" \ - --eif-path "$EIF_PATH" \ - --debug-mode - -ENCLAVE_ID="$(sudo nitro-cli describe-enclaves | jq -r '.[0].EnclaveID')" -if [ -z "$ENCLAVE_ID" ] || [ "$ENCLAVE_ID" = "null" ]; then - echo "::error::Failed to start enclave β€” describe-enclaves returned no ID" >&2 +verify_image_contents + +if ! remove_container 15 "$CONTAINER_NAME"; then + echo "failed to remove stale signer container" >&2 + exit 1 +fi +if ! terminate_enclaves; then + echo "failed to terminate stale Nitro enclaves" >&2 exit 1 fi -echo "Enclave started: $ENCLAVE_ID" - -# Wait for the enclave gRPC server on vsock port 10350 -wait_for "Enclave" 30 socat -u VSOCK-CONNECT:"$ENCLAVE_CID":10350,connect-timeout=1 /dev/null -# Start the host app with VSOCK transport enabled -AWS_REGION=us-east-1 \ - "${BIN_DIR}/amd64/app" --config "${PROJECT_DIR}/configs/app.yaml" run & -APP_PID=$! +# Debug mode permits per-PR EIFs but does not validate PCR-based identity policy. +docker run -d \ + --name "$CONTAINER_NAME" \ + --network host \ + --device=/dev/nitro_enclaves \ + --security-opt seccomp=unconfined \ + --entrypoint /bin/bash \ + -e AWS_ACCESS_KEY_ID \ + -e AWS_SECRET_ACCESS_KEY \ + -e AWS_SESSION_TOKEN \ + -e AWS_REGION=us-east-1 \ + -e ENABLE_ENCLAVE=false \ + -e APP_ENV=dev \ + -e APP_PROVIDER_ENCLAVE_NITROENCLAVE_ENABLED=false \ + -e APP_PROVIDER_SECRETS_LOCALSTACK_ENABLED=true \ + -e APP_PROVIDER_AWSKMS_LOCALSTACK_ENABLED=true \ + -e APP_PROVIDER_ENCLAVE_CLIENT_BASEURL=127.0.0.1:1 \ + -e ENCLAVE_CID=1 \ + -e ENCLAVE_CPU_COUNT="$ENCLAVE_CPU" \ + -e ENCLAVE_MEMORY_SIZE="$ENCLAVE_MEMORY" \ + -e APP_SERVICE_SIGNER_KEYID="${APP_SERVICE_SIGNER_KEYID}" \ + -e APP_PROVIDER_AWSKMS_ARNS="${APP_PROVIDER_AWSKMS_ARNS}" \ + "$IMAGE_REF" -ec ' +nitro-cli() { + if [ "${1:-}" = "run-enclave" ]; then + shift + /usr/bin/nitro-cli run-enclave --debug-mode "$@" + return + fi + /usr/bin/nitro-cli "$@" +} +source /usr/local/circle/run.sh +' -# Wait for the host app gRPC health endpoint on port 10340 -wait_for "Host app" 30 grpc_health_probe -addr=127.0.0.1:10340 +# Wait for the host app gRPC health endpoint exposed by the image on port 10340 +wait_for "Host app" 60 grpc_health_probe -addr=127.0.0.1:10340 -go test -tags=smoke -count=1 "${PROJECT_DIR}/..." +go test -tags=smoke,nitro_smoke -count=1 -timeout=5m "${PROJECT_DIR}/internal/smoke/..." diff --git a/scripts/test-build-enclave.sh b/scripts/test-build-enclave.sh index 8c75d12..a332dff 100755 --- a/scripts/test-build-enclave.sh +++ b/scripts/test-build-enclave.sh @@ -18,10 +18,27 @@ get_image_id() { docker inspect --format '{{.Id}}' "${TAG}" } +verify_image_contents() { + docker run --rm --entrypoint /bin/sh "${TAG}" -eu -c ' + test -x /usr/local/circle/enclave + test ! -e /usr/local/circle/app + test -f /usr/local/circle/configs/enclave.yaml + test ! -e /usr/local/circle/configs/app.yaml + test ! -e /usr/local/circle/configs/.env.dev + ! command -v socat >/dev/null 2>&1 + grep -q "/usr/local/circle/enclave" /usr/local/circle/run_enclave.sh + grep -q "exec >/dev/null 2>&1" /usr/local/circle/run_enclave.sh + ! grep -Eq "APP_ENV|DD_|OTEL|VSOCK-CONNECT|LOG_FIFO" /usr/local/circle/run_enclave.sh + ! grep -q "run-enclave" /usr/local/circle/run_enclave.sh + ' +} + echo "==> Build 1..." build_enclave ID1=$(get_image_id) echo " ID: ${ID1}" +echo "==> Verifying enclave-only image contents..." +verify_image_contents echo "==> Pruning BuildKit cache..." docker builder prune -af >/dev/null 2>&1 diff --git a/scripts/test-ci-shell.sh b/scripts/test-ci-shell.sh new file mode 100755 index 0000000..d44b533 --- /dev/null +++ b/scripts/test-ci-shell.sh @@ -0,0 +1,225 @@ +#!/usr/bin/env bash + +set -eu + +PROJECT_DIR="$(cd "$(dirname "$0")/.." && pwd)" +# shellcheck source=SCRIPTDIR/test-harness.sh +source "$PROJECT_DIR/scripts/test-harness.sh" + +extract_function() { + local file="$1" + local name="$2" + + awk -v start="${name}() {" ' + $0 == start { printing = 1 } + printing { print } + printing && /^}$/ { exit } + ' "$file" +} + +run_workflow_fixture() { + local script_name="$1" + local function_name="$2" + local workflow_name="$3" + local workflow_content="$4" + local fixture_dir + local function_source + local status + + fixture_dir="$(mktemp -d)" + mkdir -p "$fixture_dir/.github/workflows" + printf 'target "signer-dev" {}\n' > "$fixture_dir/docker-bake.hcl" + printf 'local-enclave-docker:\n\t@docker buildx bake signer-dev\ntest-all: test-it smoke\ntest-it: build up\nup: local-enclave-docker\n' \ + > "$fixture_dir/Makefile" + printf '%s\n' "$workflow_content" > "$fixture_dir/.github/workflows/$workflow_name" + function_source="$(extract_function "$PROJECT_DIR/scripts/$script_name" "$function_name")" + + if ( + PROJECT_DIR="$fixture_dir" + eval "$function_source" + "$function_name" + ); then + status=0 + else + status=$? + fi + + rm -rf "$fixture_dir" + return "$status" +} + +public_ci_uses_the_test_target() { + run_workflow_fixture test-dev-image.sh development_builds_select_signer_dev \ + pipeline-common-ci.yaml ' run: make test-all' || return 1 + ! run_workflow_fixture test-dev-image.sh development_builds_select_signer_dev \ + pipeline-common-ci.yaml ' run: time make test-all' || return 1 + ! run_workflow_fixture test-dev-image.sh development_builds_select_signer_dev \ + pipeline-common-ci.yaml '# run: make test-all' || return 1 + ! run_workflow_fixture test-dev-image.sh development_builds_select_signer_dev \ + pipeline-common-ci.yaml ' run: make test-all-disabled' +} + +bounded_runner_reaps_its_timer() { + local function_source + local started_at + + function_source="$(extract_function "$PROJECT_DIR/scripts/smoke-eif.sh" run_bounded)" + started_at="$(date +%s)" + ( + timeout() { + shift 3 + "$@" + } + eval "$function_source" + output="$(run_bounded 3 true)" + : "$output" + ) + + [ $(( $(date +%s) - started_at )) -lt 2 ] +} + +bounded_wait_reaps_its_timer() { + local function_source + local started_at + + function_source="$(extract_function "$PROJECT_DIR/docker/run.sh" wait_bounded)" + started_at="$(date +%s)" + ( + eval "$function_source" + output="$({ sleep 0.1 & child_pid=$!; wait_bounded "$child_pid" 3; })" + : "$output" + ) + + [ $(( $(date +%s) - started_at )) -lt 2 ] +} + +run_cleanup_scenario() { + local scenario="$1" + local marker="${2:-}" + local function_source + + function_source="$(extract_function "$PROJECT_DIR/scripts/smoke-eif.sh" cleanup)" + ( + eval "$function_source" + # The dynamically loaded cleanup function uses this variable. + # shellcheck disable=SC2034 + CONTAINER_NAME=smoke-test + run_bounded() { + shift + if [ "$scenario" = inspect_unknown ] && [ "$1" = docker ] && [ "$2" = inspect ]; then + return 2 + fi + "$@" + } + docker() { + if [ "$scenario" = stop_failure ] && [ "$1" = stop ]; then + return 1 + fi + if [ "$1" = inspect ] && [ "${2:-}" = --format ]; then + printf '0\n' + fi + return 0 + } + remove_container() { + if [ -n "$marker" ]; then + : > "$marker" + fi + [ "$scenario" != remove_failure ] + } + terminate_enclaves() { + return 0 + } + + true + cleanup + ) 2>/dev/null +} + +critical_cleanup_failure_changes_success() { + run_cleanup_scenario remove_failure + [ "$?" -eq 1 ] +} + +non_graceful_shutdown_changes_success() { + run_cleanup_scenario stop_failure + [ "$?" -eq 1 ] +} + +unknown_container_state_still_attempts_removal() { + local marker + local status + + marker="$(mktemp)" + rm -f "$marker" + run_cleanup_scenario inspect_unknown "$marker" + status=$? + [ "$status" -eq 1 ] && [ -e "$marker" ] + status=$? + rm -f "$marker" + return "$status" +} + +shared_harness_reports_all_results() { + local output + local stderr_file + local result=0 + + stderr_file="$(mktemp)" + output="$( + ( + failures=0 + success_status=0 + skip_status=0 + failure_status=0 + check 'The nested success case passes.' true || success_status=$? + skip_check 'The nested skip case does not run.' 'The test requests this skip.' || + skip_status=$? + check 'The nested failure case fails.' false || failure_status=$? + printf 'failures=%s success_status=%s skip_status=%s failure_status=%s\n' \ + "$failures" "$success_status" "$skip_status" "$failure_status" + ) 2>"$stderr_file" + )" + + grep -Fxq 'ok - The nested success case passes.' <<<"$output" && + grep -Fxq 'skip - The nested skip case does not run. The test requests this skip.' <<<"$output" && + grep -Fxq 'not ok - The nested failure case fails.' "$stderr_file" && + ! grep -Fq 'not ok -' <<<"$output" && + grep -Fxq 'failures=1 success_status=0 skip_status=0 failure_status=0' \ + <<<"$output" || result=1 + + rm -f "$stderr_file" + return "$result" +} + +errexit_harness_reports_all_results() { + local output + local status=0 + + output="$( + bash -eu -c ' + source "$1" + check "The first child case fails." false + skip_check "The child skip case does not run." "The test requests this skip." + check "The second child case still runs." true + printf "failures=%s\n" "$failures" + ' _ "$PROJECT_DIR/scripts/test-harness.sh" 2>&1 + )" || status=$? + + [ "$status" -eq 0 ] && + grep -Fxq 'not ok - The first child case fails.' <<<"$output" && + grep -Fxq 'skip - The child skip case does not run. The test requests this skip.' \ + <<<"$output" && + grep -Fxq 'ok - The second child case still runs.' <<<"$output" && + grep -Fxq 'failures=1' <<<"$output" +} + +check 'The run_bounded function returns without an orphan timer delay.' bounded_runner_reaps_its_timer +check 'The wait_bounded function returns without an orphan timer delay.' bounded_wait_reaps_its_timer +check 'A critical cleanup failure changes a successful result.' critical_cleanup_failure_changes_success +check 'A non-graceful signer shutdown changes a successful result.' non_graceful_shutdown_changes_success +check 'The cleanup starts container removal and gives a failure result for an unknown state.' unknown_container_state_still_attempts_removal +check 'The shared harness reports success, skip, and failure results.' shared_harness_reports_all_results +check 'The harness reports check and skip results when errexit is active.' errexit_harness_reports_all_results +check 'The public CI workflow uses the test target.' public_ci_uses_the_test_target + +exit "$failures" diff --git a/scripts/test-dev-image.sh b/scripts/test-dev-image.sh new file mode 100755 index 0000000..c3cba34 --- /dev/null +++ b/scripts/test-dev-image.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash + +set -eu + +PROJECT_DIR="$(cd "$(dirname "$0")/.." && pwd)" +# shellcheck source=SCRIPTDIR/test-harness.sh +source "$PROJECT_DIR/scripts/test-harness.sh" + +builder_builds_dedicated_enclave() { + grep -Fq 'export CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH}' "$PROJECT_DIR/docker/Dockerfile" && + grep -Fq 'go build -o /out/enclave ./cmd/enclave' "$PROJECT_DIR/docker/Dockerfile" +} + +production_signer_omits_enclave_runtime_assets() { + local signer_stage + + signer_stage="$(sed -n '/^FROM runtime-base AS production-runtime$/,/^FROM runtime-base AS signer-dev$/p' "$PROJECT_DIR/docker/Dockerfile")" + ! grep -Eq '/out/enclave|run_enclave(\.dev)?\.sh' <<<"$signer_stage" && + grep -Fq 'FROM production-runtime AS signer-with-enclave' "$PROJECT_DIR/docker/Dockerfile" +} + +development_signer_contains_enclave_assets() { + local signer_dev_stage + + signer_dev_stage="$(sed -n '/^FROM runtime-base AS signer-dev$/,/^FROM production-runtime AS signer-with-enclave$/p' "$PROJECT_DIR/docker/Dockerfile")" + grep -Fq '/out/enclave ./enclave' <<<"$signer_dev_stage" && + grep -Fq 'docker/run_enclave.dev.sh ./run_enclave.dev.sh' <<<"$signer_dev_stage" +} + +development_builds_select_signer_dev() { + local internal_workflow="$PROJECT_DIR/.github/workflows/pipeline-common-golang.yaml" + local public_workflow="$PROJECT_DIR/.github/workflows/pipeline-common-ci.yaml" + + grep -Fq 'target "signer-dev"' "$PROJECT_DIR/docker-bake.hcl" || return 1 + grep -Eq 'docker buildx bake .*signer-dev' "$PROJECT_DIR/Makefile" || return 1 + grep -Eq '^test-all:.*[[:space:]]test-it([[:space:]]|$)' "$PROJECT_DIR/Makefile" || return 1 + grep -Eq '^test-it:.*[[:space:]]up([[:space:]]|$)' "$PROJECT_DIR/Makefile" || return 1 + grep -Eq '^up:.*[[:space:]]local-enclave-docker([[:space:]]|$)' "$PROJECT_DIR/Makefile" || return 1 + + if [ -f "$internal_workflow" ]; then + grep -Fq 'bake_file_target: "signer-dev"' "$internal_workflow" || return 1 + return 0 + fi + + if [ ! -f "$public_workflow" ]; then + printf 'The public CI workflow is not present: %s.\n' "$public_workflow" >&2 + return 1 + fi + + grep -Fqx ' run: make test-all' "$public_workflow" +} + +development_entrypoint_uses_dedicated_enclave() { + grep -Fq '/usr/local/circle/enclave' "$PROJECT_DIR/docker/run_enclave.dev.sh" && + ! grep -Fq '/usr/local/circle/app' "$PROJECT_DIR/docker/run_enclave.dev.sh" && + ! grep -Fq 'run-enclave' "$PROJECT_DIR/docker/run_enclave.dev.sh" +} + +development_environment_allows_aws_kms() ( + export APP_PROVIDER_SECRETS_LOCALSTACK_ENABLED=false + export APP_PROVIDER_AWSKMS_LOCALSTACK_ENABLED=false + + # shellcheck disable=SC1091 + source "$PROJECT_DIR/configs/.env.dev" + + [ "$APP_PROVIDER_SECRETS_LOCALSTACK_ENABLED" = false ] && + [ "$APP_PROVIDER_AWSKMS_LOCALSTACK_ENABLED" = false ] +) + +check 'The app builder builds the dedicated enclave executable.' builder_builds_dedicated_enclave +check 'Production signer targets do not include enclave runtime assets.' production_signer_omits_enclave_runtime_assets +check 'The signer-dev target contains the dedicated enclave assets.' development_signer_contains_enclave_assets +check 'Local and CI development builds select signer-dev.' development_builds_select_signer_dev +check 'Development startup uses the dedicated enclave executable.' development_entrypoint_uses_dedicated_enclave +check 'The signer-dev target can use AWS KMS.' development_environment_allows_aws_kms + +exit "$failures" diff --git a/scripts/test-harness.sh b/scripts/test-harness.sh new file mode 100644 index 0000000..f070df3 --- /dev/null +++ b/scripts/test-harness.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +# This file contains the common shell test result functions. +failures=0 + +check() { + local description="$1" + shift + + if "$@"; then + printf 'ok - %s\n' "$description" + else + printf 'not ok - %s\n' "$description" >&2 + failures=$((failures + 1)) + fi +} + +skip_check() { + local description="$1" + local reason="$2" + + printf 'skip - %s %s\n' "$description" "$reason" +} diff --git a/scripts/test-production-coverage.sh b/scripts/test-production-coverage.sh new file mode 100755 index 0000000..cd13fd7 --- /dev/null +++ b/scripts/test-production-coverage.sh @@ -0,0 +1,371 @@ +#!/usr/bin/env bash + +set -eu + +PROJECT_DIR="$(cd "$(dirname "$0")/.." && pwd)" +# shellcheck source=SCRIPTDIR/test-harness.sh +source "$PROJECT_DIR/scripts/test-harness.sh" +work_dir="$(mktemp -d)" +trap 'rm -rf "$work_dir"' EXIT + +merge() { + bash "$PROJECT_DIR/scripts/merge-production-coverage.sh" "$@" +} + +fails_with() { + local expected="$1" + shift + local output + + if output="$("$@" 2>&1)"; then + return 1 + fi + grep -Fq "$expected" <<<"$output" +} + +missing_or_empty_arguments_are_rejected() { + local coverage="$work_dir/arguments-coverage.out" + local production="$work_dir/arguments-production.out" + + printf 'mode: set\nexample.com/signer/internal/app/app.go:1.1,2.2 1 1\n' >"$coverage" + printf 'mode: set\nexample.com/signer/internal/app/runtime_prod.go:1.1,2.2 1 1\n' >"$production" + + fails_with 'MISSING_ARGUMENT [BASE_PROFILE]' merge || return 1 + fails_with 'MISSING_ARGUMENT [PRODUCTION_PROFILE]' merge "$coverage" || return 1 + fails_with 'MISSING_ARGUMENT [PACKAGE_PREFIX]' \ + merge "$coverage" "$production" || return 1 + fails_with 'MISSING_ARGUMENT [BASE_PROFILE]' \ + merge '' "$production" 'internal/app/' || return 1 + fails_with 'MISSING_ARGUMENT [PRODUCTION_PROFILE]' \ + merge "$coverage" '' 'internal/app/' || return 1 + fails_with 'MISSING_ARGUMENT [PACKAGE_PREFIX]' merge "$coverage" "$production" '' +} + +new_production_files_are_merged() { + local coverage="$work_dir/merge-coverage.out" + local production="$work_dir/merge-production.out" + local expected="$work_dir/merge-expected.out" + + cat >"$coverage" <<'EOF' +mode: set +example.com/signer/internal/app/app.go:10.1,12.2 1 1 +example.com/signer/internal/common/config.go:5.1,7.2 1 1 +EOF + cat >"$production" <<'EOF' +mode: set +example.com/signer/internal/app/app.go:10.1,12.2 1 0 +example.com/signer/internal/app/runtime_prod.go:20.1,22.2 1 1 +example.com/signer/internal/app/runtime_prod_policy.go:30.1,32.2 1 0 +example.com/signer/internal/common/config.go:5.1,7.2 1 1 +EOF + cat >"$expected" <<'EOF' +mode: set +example.com/signer/internal/app/app.go:10.1,12.2 1 1 +example.com/signer/internal/common/config.go:5.1,7.2 1 1 +example.com/signer/internal/app/runtime_prod.go:20.1,22.2 1 1 +example.com/signer/internal/app/runtime_prod_policy.go:30.1,32.2 1 0 +EOF + + merge "$coverage" "$production" 'internal/app/' && + diff -u "$expected" "$coverage" +} + +coverage_mode_mismatch_is_rejected() { + local coverage="$work_dir/mismatch-coverage.out" + local production="$work_dir/mismatch-production.out" + local output + + printf 'mode: set\nexample.com/signer/internal/app/app.go:1.1,2.2 1 1\n' >"$coverage" + printf 'mode: atomic\nexample.com/signer/internal/app/runtime_prod.go:1.1,2.2 1 1\n' >"$production" + + if output="$(merge "$coverage" "$production" 'internal/app/' 2>&1)"; then + return 1 + fi + + grep -Fq 'COVERAGE_MODE_MISMATCH' <<<"$output" && + grep -Fq 'mode: set and mode: atomic' <<<"$output" +} + +missing_mode_header_is_rejected() { + local coverage="$work_dir/header-coverage.out" + local production="$work_dir/header-production.out" + + cat >"$coverage" <<'EOF' +example.com/signer/internal/app/a.go:1.1,2.2 1 1 +example.com/signer/internal/app/b.go:1.1,2.2 1 1 +EOF + cat >"$production" <<'EOF' +example.com/signer/internal/app/a.go:1.1,2.2 1 1 +example.com/signer/internal/app/a.go:3.1,4.2 1 0 +EOF + + fails_with "INVALID_MODE_HEADER [$coverage]" \ + merge "$coverage" "$production" 'internal/app/' +} + +production_missing_mode_header_is_rejected() { + local coverage="$work_dir/production-header-coverage.out" + local production="$work_dir/production-header-production.out" + + printf 'mode: set\nexample.com/signer/internal/app/app.go:1.1,2.2 1 1\n' >"$coverage" + printf 'example.com/signer/internal/app/runtime_prod.go:1.1,2.2 1 1\n' >"$production" + + fails_with "INVALID_MODE_HEADER [$production]" \ + merge "$coverage" "$production" 'internal/app/' +} + +missing_production_profile_is_rejected() { + local coverage="$work_dir/missing-coverage.out" + + printf 'mode: set\nexample.com/signer/internal/app/app.go:1.1,2.2 1 1\n' >"$coverage" + + fails_with 'PRODUCTION_PROFILE_MISSING_OR_EMPTY' \ + merge "$coverage" "$work_dir/does-not-exist.out" 'internal/app/' +} + +empty_base_profile_is_rejected() { + local coverage="$work_dir/empty-coverage.out" + local production="$work_dir/empty-production.out" + + : >"$coverage" + printf 'mode: set\nexample.com/signer/internal/app/runtime_prod.go:1.1,2.2 1 1\n' >"$production" + + fails_with 'BASE_PROFILE_MISSING_OR_EMPTY' \ + merge "$coverage" "$production" 'internal/app/' +} + +unreadable_profiles_are_rejected() { + local coverage="$work_dir/unreadable-coverage.out" + local production="$work_dir/unreadable-production.out" + + printf 'mode: set\nexample.com/signer/internal/app/app.go:1.1,2.2 1 1\n' >"$coverage" + printf 'mode: set\nexample.com/signer/internal/app/runtime_prod.go:1.1,2.2 1 1\n' >"$production" + + chmod 000 "$coverage" + fails_with "BASE_PROFILE_UNREADABLE [$coverage]" \ + merge "$coverage" "$production" 'internal/app/' || return 1 + + chmod 600 "$coverage" + chmod 000 "$production" + + fails_with "PRODUCTION_PROFILE_UNREADABLE [$production]" \ + merge "$coverage" "$production" 'internal/app/' +} + +prepare_awk_failure() { + local coverage="$1" + local production="$2" + local fake_bin="$3" + + printf 'mode: set\nexample.com/signer/internal/app/app.go:1.1,2.2 1 1\n' >"$coverage" + printf 'mode: set\nexample.com/signer/internal/app/runtime_prod.go:1.1,2.2 1 1\n' >"$production" + mkdir -p "$fake_bin" + printf '#!/usr/bin/env bash\nprintf "example.com/signer/internal/app/partial.go:1.1,2.2 1 1\\n"\nexit 2\n' >"$fake_bin/awk" + chmod 700 "$fake_bin/awk" +} + +merge_fails_after_awk() { + local coverage="$1" + local production="$2" + local fake_bin="$3" + + PATH="$fake_bin:$PATH" fails_with "COVERAGE_MERGE_FAILED [$production -> $coverage]" \ + merge "$coverage" "$production" 'internal/app/' +} + +awk_fatal_error_preserves_base_profile() { + local coverage="$work_dir/preserve-coverage.out" + local production="$work_dir/preserve-production.out" + local before="$work_dir/preserve-before.out" + local fake_bin="$work_dir/preserve-fake-bin" + + prepare_awk_failure "$coverage" "$production" "$fake_bin" + cp "$coverage" "$before" + + merge_fails_after_awk "$coverage" "$production" "$fake_bin" || return 1 + diff -u "$before" "$coverage" +} + +awk_fatal_error_removes_temp_profiles() { + local profile_dir="$work_dir/cleanup" + local coverage="$profile_dir/coverage.out" + local production="$profile_dir/production.out" + local expected="$work_dir/cleanup-expected.out" + local fake_bin="$work_dir/cleanup-fake-bin" + + mkdir -p "$profile_dir" + prepare_awk_failure "$coverage" "$production" "$fake_bin" + printf 'coverage.out\nproduction.out\n' >"$expected" + + merge_fails_after_awk "$coverage" "$production" "$fake_bin" || return 1 + LC_ALL=C ls -A1 "$profile_dir" | diff -u "$expected" - +} + +first_temp_profile_failure_uses_stable_diagnostic() { + local coverage="$work_dir/first-temp-coverage.out" + local production="$work_dir/first-temp-production.out" + local fake_bin="$work_dir/first-temp-fake-bin" + + printf 'mode: set\nexample.com/signer/internal/app/app.go:1.1,2.2 1 1\n' >"$coverage" + printf 'mode: set\nexample.com/signer/internal/app/runtime_prod.go:1.1,2.2 1 1\n' >"$production" + mkdir -p "$fake_bin" + printf '#!/usr/bin/env bash\nexit 1\n' >"$fake_bin/mktemp" + chmod 700 "$fake_bin/mktemp" + + PATH="$fake_bin:$PATH" fails_with \ + "TEMP_PROFILE_CREATE_FAILED [${coverage}.additions.XXXXXX]" \ + merge "$coverage" "$production" 'internal/app/' +} + +second_temp_profile_failure_removes_first_temp_profile() { + local profile_dir="$work_dir/second-temp" + local coverage="$profile_dir/coverage.out" + local production="$profile_dir/production.out" + local expected="$work_dir/second-temp-expected.out" + local before="$work_dir/second-temp-before.out" + local fake_bin="$work_dir/second-temp-fake-bin" + + mkdir -p "$profile_dir" "$fake_bin" + printf 'mode: set\nexample.com/signer/internal/app/app.go:1.1,2.2 1 1\n' >"$coverage" + printf 'mode: set\nexample.com/signer/internal/app/runtime_prod.go:1.1,2.2 1 1\n' >"$production" + cp "$coverage" "$before" + printf 'coverage.out\nproduction.out\n' >"$expected" + cat >"$fake_bin/mktemp" <<'EOF' +#!/usr/bin/env bash + +case "$1" in + *.additions.XXXXXX) + temp_profile="${1%XXXXXX}TEST" + : >"$temp_profile" + printf '%s\n' "$temp_profile" + ;; + *) exit 1 ;; +esac +EOF + chmod 700 "$fake_bin/mktemp" + + PATH="$fake_bin:$PATH" fails_with \ + "TEMP_PROFILE_CREATE_FAILED [${coverage}.merged.XXXXXX]" \ + merge "$coverage" "$production" 'internal/app/' || return 1 + diff -u "$before" "$coverage" || return 1 + LC_ALL=C ls -A1 "$profile_dir" | diff -u "$expected" - +} + +final_profile_command_failure_uses_stable_diagnostic() { + local command_name="$1" + local expected_identifier="$2" + local profile_dir="$work_dir/${command_name}-failure" + local coverage="$profile_dir/coverage.out" + local production="$profile_dir/production.out" + local expected="$work_dir/${command_name}-failure-expected.out" + local before="$work_dir/${command_name}-failure-before.out" + local fake_bin="$work_dir/${command_name}-failure-fake-bin" + + mkdir -p "$profile_dir" "$fake_bin" + printf 'mode: set\nexample.com/signer/internal/app/app.go:1.1,2.2 1 1\n' >"$coverage" + printf 'mode: set\nexample.com/signer/internal/app/runtime_prod.go:1.1,2.2 1 1\n' >"$production" + cp "$coverage" "$before" + printf 'coverage.out\nproduction.out\n' >"$expected" + printf '#!/usr/bin/env bash\nexit 1\n' >"$fake_bin/$command_name" + chmod 700 "$fake_bin/$command_name" + + PATH="$fake_bin:$PATH" fails_with "$expected_identifier" \ + merge "$coverage" "$production" 'internal/app/' || return 1 + diff -u "$before" "$coverage" || return 1 + LC_ALL=C ls -A1 "$profile_dir" | diff -u "$expected" - +} + +zero_additions_is_allowed() { + local coverage="$work_dir/noop-coverage.out" + local production="$work_dir/noop-production.out" + local expected="$work_dir/noop-expected.out" + + printf 'mode: set\nexample.com/signer/internal/app/app.go:1.1,2.2 1 1\n' >"$coverage" + printf 'mode: set\nexample.com/signer/internal/app/app.go:1.1,2.2 1 0\n' >"$production" + cp "$coverage" "$expected" + + merge "$coverage" "$production" 'internal/app/' && + diff -u "$expected" "$coverage" +} + +out_of_package_lines_are_not_merged() { + local coverage="$work_dir/filter-coverage.out" + local production="$work_dir/filter-production.out" + + printf 'mode: set\nexample.com/signer/internal/app/app.go:1.1,2.2 1 1\n' >"$coverage" + cat >"$production" <<'EOF' +mode: set +example.com/signer/internal/app/runtime_prod.go:1.1,2.2 1 1 +example.com/signer/internal/enclave/enclave.go:1.1,2.2 1 1 +EOF + + merge "$coverage" "$production" 'internal/app/' && + grep -Fq 'internal/app/runtime_prod.go' "$coverage" && + ! grep -Fq 'internal/enclave/enclave.go' "$coverage" +} + +package_prefix_without_trailing_slash_is_anchored() { + local coverage="$work_dir/prefix-coverage.out" + local production="$work_dir/prefix-production.out" + + printf 'mode: set\nexample.com/signer/internal/common/config.go:1.1,2.2 1 1\n' >"$coverage" + cat >"$production" <<'EOF' +mode: set +example.com/signer/internal/app/runtime_prod.go:1.1,2.2 1 1 +example.com/signer/internal/appconfig/config.go:1.1,2.2 1 1 +EOF + + merge "$coverage" "$production" 'internal/app' && + grep -Fq 'internal/app/runtime_prod.go' "$coverage" && + ! grep -Fq 'internal/appconfig/config.go' "$coverage" +} + +same_profile_path_is_rejected() { + local profile="$work_dir/same-profile.out" + + printf 'mode: set\nexample.com/signer/internal/app/app.go:1.1,2.2 1 1\n' >"$profile" + + fails_with 'PROFILE_PATHS_MATCH' \ + merge "$profile" "$profile" 'internal/app/' +} + +production_profile_without_package_is_rejected() { + local coverage="$work_dir/package-coverage.out" + local production="$work_dir/package-production.out" + + printf 'mode: set\nexample.com/signer/internal/app/app.go:1.1,2.2 1 1\n' >"$coverage" + printf 'mode: set\nexample.com/signer/internal/enclave/enclave.go:1.1,2.2 1 1\n' >"$production" + + fails_with 'PACKAGE_COVERAGE_MISSING' \ + merge "$coverage" "$production" 'internal/app/' +} + +check 'The merge identifies each missing or empty argument.' missing_or_empty_arguments_are_rejected +check 'The merge adds new production files.' new_production_files_are_merged +check 'The merge rejects different coverage modes.' coverage_mode_mismatch_is_rejected +check 'The merge identifies the base profile path for an invalid mode header.' missing_mode_header_is_rejected +check 'The merge identifies the production profile path for an invalid mode header.' production_missing_mode_header_is_rejected +check 'The merge rejects a missing production profile.' missing_production_profile_is_rejected +check 'The merge rejects an empty base profile.' empty_base_profile_is_rejected +unreadable_profile_description='The merge identifies each unreadable profile path.' +if [ "$(id -u)" -eq 0 ]; then + skip_check "$unreadable_profile_description" \ + 'Root can read files without read permission.' +else + check "$unreadable_profile_description" unreadable_profiles_are_rejected +fi +check 'The base profile does not change after an AWK error.' awk_fatal_error_preserves_base_profile +check 'The merge removes temporary profiles after an AWK error.' awk_fatal_error_removes_temp_profiles +check 'The first temporary profile error has a stable identifier.' first_temp_profile_failure_uses_stable_diagnostic +check 'The second temporary profile error removes the first temporary profile.' second_temp_profile_failure_removes_first_temp_profile +check 'The merge identifies a merged profile write error and preserves the base profile.' \ + final_profile_command_failure_uses_stable_diagnostic cat MERGED_PROFILE_WRITE_FAILED +check 'The merge identifies a base profile replace error and preserves the base profile.' \ + final_profile_command_failure_uses_stable_diagnostic mv BASE_PROFILE_REPLACE_FAILED +check 'The merge accepts a profile with no new files.' zero_additions_is_allowed +check 'The merge does not include files from other packages.' out_of_package_lines_are_not_merged +check 'The merge accepts a package prefix without a trailing slash.' package_prefix_without_trailing_slash_is_anchored +check 'The merge rejects the same path for both profiles.' same_profile_path_is_rejected +check 'The production profile must contain package coverage.' production_profile_without_package_is_rejected + +exit "$failures" diff --git a/scripts/test-production-image.sh b/scripts/test-production-image.sh new file mode 100755 index 0000000..71e391f --- /dev/null +++ b/scripts/test-production-image.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash + +set -eu + +PROJECT_DIR="$(cd "$(dirname "$0")/.." && pwd)" +failures=0 + +check() { + local description="$1" + shift + + if "$@"; then + printf 'ok - %s\n' "$description" + else + printf 'not ok - %s\n' "$description" >&2 + failures=$((failures + 1)) + fi +} + +production_builds_select_only_production_runtime_files() { + local app_files + local proxy_files + local enclave_files + local enclave_runtime_files + + app_files="$(GOOS=linux CGO_ENABLED=0 go list -tags=prod -f '{{join .GoFiles " "}}' "$PROJECT_DIR/internal/app")" + proxy_files="$(GOOS=linux CGO_ENABLED=0 go list -tags=prod -f '{{join .GoFiles " "}}' "$PROJECT_DIR/internal/vsockproxy")" + enclave_files="$(GOOS=linux CGO_ENABLED=1 go list -tags=prod -f '{{join .GoFiles " "}}' "$PROJECT_DIR/cmd/enclave")" + enclave_runtime_files="$(GOOS=linux CGO_ENABLED=1 go list -tags=prod -f '{{join .GoFiles " "}}' "$PROJECT_DIR/internal/enclave")" + + grep -Fq 'runtime_prod.go' <<<"$app_files" && + ! grep -Fq 'runtime_dev.go' <<<"$app_files" && + grep -Fq 'runtime_prod_linux.go' <<<"$proxy_files" && + ! grep -Fq 'runtime_dev_linux.go' <<<"$proxy_files" && + grep -Fq 'run_enclave_prod.go' <<<"$enclave_files" && + ! grep -Fq 'run_enclave_dev.go' <<<"$enclave_files" && + grep -Fq 'runtime_prod_linux_cgo.go' <<<"$enclave_runtime_files" && + ! grep -Fq 'runtime_dev.go' <<<"$enclave_runtime_files" +} + +docker_builds_production_executables() { + grep -Fq 'go build -tags=prod -o /out/app-production .' "$PROJECT_DIR/docker/Dockerfile" && + grep -Fq 'go build -tags=prod' "$PROJECT_DIR/docker/Dockerfile.enclave" +} + +docker_separates_production_and_development_stages() { + grep -Fq 'FROM runtime-base AS production-runtime' "$PROJECT_DIR/docker/Dockerfile" && + grep -Fq 'FROM production-runtime AS signer' "$PROJECT_DIR/docker/Dockerfile" && + grep -Fq 'FROM runtime-base AS signer-dev' "$PROJECT_DIR/docker/Dockerfile" && + grep -Fq 'FROM production-runtime AS signer-with-enclave' "$PROJECT_DIR/docker/Dockerfile" && + ! grep -Fq 'FROM signer AS signer-with-enclave' "$PROJECT_DIR/docker/Dockerfile" && + ! grep -Fq 'FROM signer AS signer-dev' "$PROJECT_DIR/docker/Dockerfile" +} + +production_stage_excludes_development_assets() { + local production_stage + + production_stage="$(sed -n '/^FROM runtime-base AS production-runtime$/,/^FROM production-runtime AS signer$/p' "$PROJECT_DIR/docker/Dockerfile")" + grep -Fq '/out/app-production ./app' <<<"$production_stage" && + grep -Fq 'configs/app.production.yaml ./config.yaml' <<<"$production_stage" && + ! grep -Eq 'deployments|/out/enclave|run_proxy|run_enclave\.dev|configs ./configs|\.env' <<<"$production_stage" +} + +production_dockerfile_excludes_socat() { + ! grep -Fqw 'socat' "$PROJECT_DIR/docker/Dockerfile" +} + +development_stage_contains_development_assets() { + local development_stage + + development_stage="$(sed -n '/^FROM runtime-base AS signer-dev$/,/^FROM production-runtime AS signer-with-enclave$/p' "$PROJECT_DIR/docker/Dockerfile")" + grep -Fq '/out/app ./app' <<<"$development_stage" && + grep -Fq '/out/enclave ./enclave' <<<"$development_stage" && + grep -Fq 'configs ./configs' <<<"$development_stage" && + grep -Fq 'deployments ./deployments' <<<"$development_stage" +} + +production_launcher_always_starts_nitro() { + local launcher="$PROJECT_DIR/docker/run.sh" + + ! grep -Eq 'ENABLE_ENCLAVE|APP_PROVIDER_ENCLAVE_NITROENCLAVE_ENABLED|enclave_enabled' "$launcher" && + ! grep -Fq 'APP_PROVIDER_ENCLAVE_NITROENCLAVE_CID' "$launcher" && + ! grep -Eq 'EXTRA_NITRO_CLI_FLAGS|--debug-mode' "$launcher" && + grep -Fq 'readonly ENCLAVE_CID=16' "$launcher" && + grep -Fq 'start_vsockproxy' "$launcher" && + grep -Fq 'start_enclave' "$launcher" && + grep -Fq 'start_app' "$launcher" +} + +production_config_excludes_development_modes() { + local config="$PROJECT_DIR/configs/app.production.yaml" + + test -f "$config" && + ! grep -Eqi 'localstack|baseURL|nitroEnclave|ENABLE_ENCLAVE' "$config" +} + +release_smoke_checks_image_contents() { + local smoke="$PROJECT_DIR/scripts/smoke-eif.sh" + + grep -Fq 'verify_image_contents' "$smoke" && + grep -Fq "docker inspect --format '{{json .Config.Entrypoint}}'" "$smoke" && + grep -Fq "test \"\$entrypoint\" = '[\"/usr/local/circle/run.sh\"]'" "$smoke" && + grep -Fq 'test ! -e /usr/local/circle/enclave' "$smoke" && + grep -Fq 'test ! -e /usr/local/circle/configs' "$smoke" && + grep -Fq 'test ! -e /usr/local/circle/run_proxy.sh' "$smoke" && + grep -Fq 'if command -v socat >/dev/null 2>&1; then exit 1; fi' "$smoke" && + grep -Fq 'if grep -aF "dev-session-token-placeholder"' "$smoke" && + grep -Fq 'if grep -aE "(localhost|localstack):4566"' "$smoke" && + grep -Fq 'nitro-cli()' "$smoke" && + grep -Fq '/usr/bin/nitro-cli run-enclave --debug-mode "$@"' "$smoke" && + grep -Fq 'source /usr/local/circle/run.sh' "$smoke" && + grep -Fq -- '-e ENABLE_ENCLAVE=false' "$smoke" && + grep -Fq -- '-e APP_ENV=dev' "$smoke" && + grep -Fq -- '-e APP_PROVIDER_ENCLAVE_NITROENCLAVE_ENABLED=false' "$smoke" && + grep -Fq -- '-e APP_PROVIDER_SECRETS_LOCALSTACK_ENABLED=true' "$smoke" && + grep -Fq -- '-e APP_PROVIDER_AWSKMS_LOCALSTACK_ENABLED=true' "$smoke" && + grep -Fq -- '-e APP_PROVIDER_ENCLAVE_CLIENT_BASEURL=127.0.0.1:1' "$smoke" && + grep -Fq -- '-e ENCLAVE_CID=1' "$smoke" +} + +check 'The production build selects only production runtime files.' production_builds_select_only_production_runtime_files +check 'Docker builds the production executables.' docker_builds_production_executables +check 'Docker separates the production and development stages.' docker_separates_production_and_development_stages +check 'The production stage excludes development assets.' production_stage_excludes_development_assets +check 'The production Dockerfile excludes socat.' production_dockerfile_excludes_socat +check 'The development stage contains development assets.' development_stage_contains_development_assets +check 'The production launcher always starts Nitro.' production_launcher_always_starts_nitro +check 'The production configuration excludes development modes.' production_config_excludes_development_modes +check 'The release smoke test checks the image contents.' release_smoke_checks_image_contents + +exit "$failures" diff --git a/scripts/test-secure-eif.sh b/scripts/test-secure-eif.sh new file mode 100755 index 0000000..9d06f4c --- /dev/null +++ b/scripts/test-secure-eif.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +set -eu + +PROJECT_DIR="$(cd "$(dirname "$0")/.." && pwd)" +# shellcheck source=SCRIPTDIR/test-harness.sh +source "$PROJECT_DIR/scripts/test-harness.sh" + +enclave_launcher_is_silent() { + grep -Fq 'exec >/dev/null 2>&1' "$PROJECT_DIR/docker/run_enclave.sh" && + grep -Fq 'exec env APP_PUBLIC_SERVER_PORT=10350' "$PROJECT_DIR/docker/run_enclave.sh" +} + +enclave_has_no_host_observability_bridge() { + ! grep -Eq 'APP_ENV|DD_(SERVICE|ENV|ENTITY_ID)|OTEL|socat|LOG_FIFO|8000|8001|4317|8125|8126' \ + "$PROJECT_DIR/docker/run_enclave.sh" "$PROJECT_DIR/docker/Dockerfile.enclave" +} + +host_has_no_enclave_observability_bridge() { + ! grep -Eq 'setup_vsock_bridges|VSOCK-LISTEN:(8000|8001|4317|8125|8126)' \ + "$PROJECT_DIR/docker/run.sh" +} + +development_enclave_has_no_host_config() { + awk ' + /^ enclave:$/ { in_enclave = 1; found_enclave = 1; next } + in_enclave && /^ [[:alnum:]_-]+:$/ { in_enclave = 0 } + in_enclave && /APP_ENV|AWS_PROFILE|AWS_ACCESS_KEY_ID|AWS_SECRET_ACCESS_KEY|APP_PROVIDER_AWSKMS_/ { + found_host_config = 1 + } + END { exit !found_enclave || found_host_config } + ' "$PROJECT_DIR/deployments/docker-compose.yaml" +} + +build_has_one_enclave_target() { + [ "$(grep -Ec '^target "enclave"' "$PROJECT_DIR/docker-bake.hcl")" -eq 1 ] && + ! grep -Rqi 'enclave-debug' \ + "$PROJECT_DIR/docker-bake.hcl" \ + "$PROJECT_DIR/.github/workflows" \ + "$PROJECT_DIR/docker" +} + +check 'The enclave launcher does not write to stdout or stderr.' enclave_launcher_is_silent +check 'The enclave image has no host observability bridge.' enclave_has_no_host_observability_bridge +check 'The host has no enclave observability bridge.' host_has_no_enclave_observability_bridge +check 'The host does not give configuration to the development enclave process.' development_enclave_has_no_host_config +check 'The build has one secure enclave target.' build_has_one_enclave_target + +exit "$failures" diff --git a/scripts/test.sh b/scripts/test.sh index fba411b..d2acc8f 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -7,6 +7,12 @@ BASE_DIR="$(dirname "$0")" cd "$PROJECT_DIR" || exit +bash "${BASE_DIR}/test-ci-shell.sh" +bash "${BASE_DIR}/test-dev-image.sh" +bash "${BASE_DIR}/test-production-image.sh" +bash "${BASE_DIR}/test-production-coverage.sh" +bash "${BASE_DIR}/test-secure-eif.sh" + # shellcheck disable=SC2039 case $OSTYPE in linux* ) @@ -58,6 +64,17 @@ TEST_RESULTS_DIR=test_results mkdir -p ${TEST_RESULTS_DIR} golangci-lint run --config "${PROJECT_DIR}/build/golangci.yaml" +# Run a separate lint pass for the production build tag. +if [ "$PLATFORM" = "linux" ]; then + golangci-lint run --config "${PROJECT_DIR}/build/golangci.yaml" \ + --build-tags=prod "${PROJECT_DIR}/..." +else + golangci-lint run --config "${PROJECT_DIR}/build/golangci.yaml" \ + --build-tags=prod \ + "${PROJECT_DIR}/internal/app/..." \ + "${PROJECT_DIR}/internal/vsockproxy/..." \ + "${PROJECT_DIR}/cmd/..." +fi if [ "${RUN_IT_TESTS:-false}" = true ]; then # count=1 ensures tests are not cached; helpful if making non src code changes that may affect the test. @@ -77,6 +94,21 @@ if [ $exit_code -ne 0 ]; then exit $exit_code fi +# Production app files use a mutually exclusive build tag. Add files that are +# present only in the production package to the default profile. +prod_exit=0 +go test -tags=prod -coverprofile="${COVERAGE_DIR}/prod_raw.out" "${PROJECT_DIR}/internal/app" || prod_exit=$? +go test -tags=prod "${PROJECT_DIR}/internal/vsockproxy" "${PROJECT_DIR}/internal/enclave" || prod_exit=$? +CGO_ENABLED=0 go test -tags=prod "${PROJECT_DIR}/cmd/enclave" || prod_exit=$? +if [ $prod_exit -ne 0 ]; then + echo "Production tests failed with exit code ${prod_exit}." + exit $prod_exit +fi +bash "${BASE_DIR}/merge-production-coverage.sh" \ + "${COVERAGE_DIR}/coverage.out" \ + "${COVERAGE_DIR}/prod_raw.out" \ + 'internal/app/' + # Run go-ignore-cov to process coverage ignore post processing # If no coverage to ignore, there are no changes to the coverage file. go-ignore-cov --file "${COVERAGE_DIR}/coverage.out" --output "${COVERAGE_DIR}/coverage.out" --root "${PROJECT_DIR}"