diff --git a/.github/workflows/scripts/split.sh b/.github/workflows/scripts/split.sh index 8f661461a9..ef77f9557e 100755 --- a/.github/workflows/scripts/split.sh +++ b/.github/workflows/scripts/split.sh @@ -15,7 +15,7 @@ fi declare -a job_set current_runner=1 for FILE in test/integration/suites/*; do - job_set[$current_runner]+="${FILE##test/integration/} " + job_set[current_runner]+="${FILE##test/integration/} " ((current_runner++)) if [ "$current_runner" -gt "$NUM_RUNNERS" ]; then diff --git a/.github/workflows/scripts/split_k8s.sh b/.github/workflows/scripts/split_k8s.sh index e3c8b28f50..bd801220a5 100755 --- a/.github/workflows/scripts/split_k8s.sh +++ b/.github/workflows/scripts/split_k8s.sh @@ -15,7 +15,7 @@ fi declare -a job_set current_runner=1 for FILE in test/integration/suites/k8s*; do - job_set[$current_runner]+="${FILE##test/integration/} " + job_set[current_runner]+="${FILE##test/integration/} " ((current_runner++)) if [ "$current_runner" -gt "$NUM_RUNNERS" ]; then diff --git a/cmd/spire-agent/cli/run/run.go b/cmd/spire-agent/cli/run/run.go index 823309fb97..a9ba4375d4 100644 --- a/cmd/spire-agent/cli/run/run.go +++ b/cmd/spire-agent/cli/run/run.go @@ -11,6 +11,7 @@ import ( "os" "os/signal" "path/filepath" + "slices" "sort" "strconv" "strings" @@ -24,7 +25,9 @@ import ( "github.com/hashicorp/hcl/hcl/token" "github.com/mitchellh/cli" "github.com/sirupsen/logrus" + "github.com/spiffe/go-spiffe/v2/spiffeid" "github.com/spiffe/spire/pkg/agent" + "github.com/spiffe/spire/pkg/agent/broker" "github.com/spiffe/spire/pkg/agent/client" "github.com/spiffe/spire/pkg/agent/trustbundlesources" "github.com/spiffe/spire/pkg/agent/workloadkey" @@ -108,6 +111,89 @@ type agentConfig struct { UnusedKeyPositions map[string][]token.Pos `hcl:",unusedKeyPositions"` } +// brokerHCLConfig is the HCL block for the SPIFFE Broker API endpoint: +// +// broker { +// # At least one of these MUST be set; both MAY be set to expose the +// # broker endpoint over both transports simultaneously (e.g., UDS +// # for a local sidecar broker AND TCP for a remote broker). +// socket_path = "/run/spire/agent/broker.sock" # POSIX-only UDS +// bind_address = "0.0.0.0:8443" # TCP, both platforms +// +// brokers = [ +// { id = "spiffe://example.org/some/broker" }, +// ] +// } +type brokerHCLConfig struct { + // SocketPath binds the broker endpoint to a Unix domain socket at the + // given path. POSIX-only. + SocketPath string `hcl:"socket_path"` + + // BindAddress binds the broker endpoint to a TCP address ("host:port"). + // Works on both POSIX and Windows. + BindAddress string `hcl:"bind_address"` + + // Brokers enumerates the brokers authorized to talk to this agent's + // broker endpoint. Each entry's `id` is any valid SPIFFE ID; cross- + // trust-domain broker identities are allowed. + Brokers []brokerHCLEntry `hcl:"brokers"` + + // AllowedReferenceTypesOverTCP restricts which WorkloadReference type + // URLs may be used by brokers connecting over the TCP listener. Empty + // (the default) denies every TCP request — fail-closed against remote + // use of local-only reference types such as PIDs. Each entry is a + // verbatim type URL; the wildcard `"*"` is intentionally NOT honoured + // here so operators must explicitly enumerate what is safe over the + // network. The list is global to the endpoint and applies on top of + // each broker's `allowed_reference_types`. Has no effect on UDS + // connections. + AllowedReferenceTypesOverTCP []string `hcl:"allowed_reference_types_over_tcp"` + + UnusedKeyPositions map[string][]token.Pos `hcl:",unusedKeyPositions"` +} + +type brokerHCLEntry struct { + ID string `hcl:"id"` + + // AllowedReferenceTypes restricts which WorkloadReference types this + // broker may use. Each entry is the verbatim protobuf type URL that + // the workload attestor plugin inspects, e.g. + // `type.googleapis.com/spiffe.broker.KubernetesObjectReference`. + // Use `"*"` to allow any reference type. Must list at least one entry. + AllowedReferenceTypes []string `hcl:"allowed_reference_types"` + + UnusedKeyPositions map[string][]token.Pos `hcl:",unusedKeyPositions"` +} + +// brokerBindAddrs resolves the broker endpoint's listener addresses from +// the HCL block. Returns one address per configured transport (TCP via +// `bind_address`, UDS via `socket_path`). Both MAY be set; at least one +// MUST be set. The TCP branch is platform-agnostic and resolved here; the +// UDS branch delegates to `brokerSocketAddr` which is defined per platform +// (POSIX has UDS support; Windows rejects with an error). +func (c *agentConfig) brokerBindAddrs() ([]net.Addr, error) { + sp, ba := c.Experimental.Broker.SocketPath, c.Experimental.Broker.BindAddress + if sp == "" && ba == "" { + return nil, errors.New("experimental.broker requires socket_path, bind_address, or both") + } + var addrs []net.Addr + if sp != "" { + uds, err := c.brokerSocketAddr() + if err != nil { + return nil, err + } + addrs = append(addrs, uds) + } + if ba != "" { + tcp, err := net.ResolveTCPAddr("tcp", ba) + if err != nil { + return nil, fmt.Errorf("invalid experimental.broker.bind_address %q: %w", ba, err) + } + addrs = append(addrs, tcp) + } + return addrs, nil +} + type sdsConfig struct { DefaultSVIDName string `hcl:"default_svid_name"` DefaultBundleName string `hcl:"default_bundle_name"` @@ -136,6 +222,12 @@ type experimentalConfig struct { RateLimit workloadAPIRateLimitConfig `hcl:"ratelimit"` + // Broker holds the configuration for the SPIFFE Broker API endpoint + // (distinct from the Delegated Identity API's authorized_delegates). + // Kept under `experimental` while the spec stabilizes — breaking + // changes may land before this moves to the top-level config. + Broker *brokerHCLConfig `hcl:"broker"` + Flags fflag.RawConfig `hcl:"feature_flags"` } @@ -526,6 +618,7 @@ func NewAgentConfig(c *Config, logOptions []log.Option, allowUnknownConfig bool) } ac.AdminBindAddress = adminAddr } + // Handle join token - read from file if specified if c.Agent.JoinTokenFile != "" { tokenBytes, err := os.ReadFile(c.Agent.JoinTokenFile) @@ -602,6 +695,59 @@ func NewAgentConfig(c *Config, logOptions []log.Option, allowUnknownConfig bool) ac.AuthorizedDelegates = c.Agent.AuthorizedDelegates + if c.Agent.Experimental.Broker != nil { + bindAddrs, err := c.Agent.brokerBindAddrs() + if err != nil { + return nil, err + } + // Pass 1: validate every entry has a non-empty, unique ID. Doing + // this first means subsequent errors can reference brokers by ID + // instead of slice index — which is more helpful to operators + // since they wrote the IDs themselves. + seen := make(map[string]struct{}, len(c.Agent.Experimental.Broker.Brokers)) + for i, b := range c.Agent.Experimental.Broker.Brokers { + if b.ID == "" { + return nil, fmt.Errorf("experimental.broker.brokers[%d].id: must be specified", i) + } + if _, dup := seen[b.ID]; dup { + return nil, fmt.Errorf("experimental.broker.brokers[%s].id: duplicate broker id", b.ID) + } + seen[b.ID] = struct{}{} + } + + // Pass 2: per-broker field validation. ID is referenced in error + // messages now that we know it's set and unique. + brokers := make([]broker.Broker, 0, len(c.Agent.Experimental.Broker.Brokers)) + for _, b := range c.Agent.Experimental.Broker.Brokers { + if _, err := spiffeid.FromString(b.ID); err != nil { + return nil, fmt.Errorf("experimental.broker.brokers[%s].id: %w", b.ID, err) + } + if len(b.AllowedReferenceTypes) == 0 { + return nil, fmt.Errorf("experimental.broker.brokers[%s].allowed_reference_types: must list at least one reference type URL", b.ID) + } + brokers = append(brokers, broker.Broker{ + ID: b.ID, + AllowedReferenceTypes: b.AllowedReferenceTypes, + }) + } + // allowed_reference_types_over_tcp is only meaningful when a TCP + // listener is configured; surface a clear error rather than + // silently ignoring the field. Wildcards are intentionally not + // honoured for the TCP gate — operators must list specific + // reference type URLs to opt them in. + if len(c.Agent.Experimental.Broker.AllowedReferenceTypesOverTCP) > 0 && c.Agent.Experimental.Broker.BindAddress == "" { + return nil, errors.New("experimental.broker.allowed_reference_types_over_tcp set but bind_address is not configured") + } + if slices.Contains(c.Agent.Experimental.Broker.AllowedReferenceTypesOverTCP, "*") { + return nil, errors.New("experimental.broker.allowed_reference_types_over_tcp: wildcard \"*\" is not supported; list specific reference type URLs") + } + ac.Broker = agent.BrokerConfig{ + BindAddresses: bindAddrs, + Brokers: brokers, + AllowedReferenceTypesOverTCP: c.Agent.Experimental.Broker.AllowedReferenceTypesOverTCP, + } + } + if c.Agent.AvailabilityTarget != "" { t, err := time.ParseDuration(c.Agent.AvailabilityTarget) if err != nil { @@ -694,6 +840,17 @@ func checkForUnknownConfig(c *Config, l logrus.FieldLogger) (err error) { detectedUnknown("agent", a.UnusedKeyPositions) } + if a := c.Agent; a != nil && a.Experimental.Broker != nil { + if len(a.Experimental.Broker.UnusedKeyPositions) != 0 { + detectedUnknown("experimental.broker", a.Experimental.Broker.UnusedKeyPositions) + } + for i, b := range a.Experimental.Broker.Brokers { + if len(b.UnusedKeyPositions) != 0 { + detectedUnknown(fmt.Sprintf("experimental.broker.brokers[%d]", i), b.UnusedKeyPositions) + } + } + } + // TODO: Re-enable unused key detection for telemetry. See // https://github.com/spiffe/spire/issues/1101 for more information // diff --git a/cmd/spire-agent/cli/run/run_posix.go b/cmd/spire-agent/cli/run/run_posix.go index 4c8ddd9d03..8a63a48e97 100644 --- a/cmd/spire-agent/cli/run/run_posix.go +++ b/cmd/spire-agent/cli/run/run_posix.go @@ -53,6 +53,28 @@ func (c *agentConfig) hasAdminAddr() bool { return c.AdminSocketPath != "" } +// brokerSocketAddr resolves the UDS branch of broker bind-address selection. +// Returns the platform-agnostic TCP branch is handled in brokerBindAddr. +func (c *agentConfig) brokerSocketAddr() (net.Addr, error) { + socketPathAbs, err := filepath.Abs(c.SocketPath) + if err != nil { + return nil, fmt.Errorf("failed to get absolute path for socket_path: %w", err) + } + brokerSocketPathAbs, err := filepath.Abs(c.Experimental.Broker.SocketPath) + if err != nil { + return nil, fmt.Errorf("failed to get absolute path for experimental.broker.socket_path: %w", err) + } + + if strings.HasPrefix(brokerSocketPathAbs, filepath.Dir(socketPathAbs)+"/") { + return nil, errors.New("broker socket cannot be in the same directory or a subdirectory as that containing the Workload API socket") + } + + return &net.UnixAddr{ + Name: brokerSocketPathAbs, + Net: "unix", + }, nil +} + // validateOS performs posix specific validations of the agent config func (c *agentConfig) validateOS() error { if c.Experimental.NamedPipeName != "" { @@ -88,5 +110,19 @@ func prepareEndpoints(c *agent.Config) error { } } + for _, addr := range c.Broker.BindAddresses { + if addr.Network() != "unix" { + continue + } + // Create uds dir and parents if not exists + brokerDir := filepath.Dir(addr.String()) + if _, statErr := os.Stat(brokerDir); os.IsNotExist(statErr) { + c.Log.WithField("dir", brokerDir).Infof("Creating broker UDS directory") + if err := os.MkdirAll(brokerDir, 0755); err != nil { + return err + } + } + } + return nil } diff --git a/cmd/spire-agent/cli/run/run_windows.go b/cmd/spire-agent/cli/run/run_windows.go index 015bbc3420..dd4a7d0a5a 100644 --- a/cmd/spire-agent/cli/run/run_windows.go +++ b/cmd/spire-agent/cli/run/run_windows.go @@ -32,6 +32,13 @@ func (c *agentConfig) hasAdminAddr() bool { return c.Experimental.AdminNamedPipeName != "" } +// brokerSocketAddr resolves the UDS branch of broker bind-address selection. +// Windows does not support UDS for the broker endpoint; configure +// `experimental.broker.bind_address` (TCP) instead. +func (c *agentConfig) brokerSocketAddr() (net.Addr, error) { + return nil, errors.New("experimental.broker.socket_path is not supported on this platform; use experimental.broker.bind_address (TCP) instead") +} + // validateOS performs windows specific validations of the agent config func (c *agentConfig) validateOS() error { if c.SocketPath != "" { diff --git a/conf/agent/agent_full.conf b/conf/agent/agent_full.conf index b814a057e8..82d638456e 100644 --- a/conf/agent/agent_full.conf +++ b/conf/agent/agent_full.conf @@ -150,6 +150,55 @@ agent { # # # fetch_secrets: Max calls/sec per selector set for SDS FetchSecrets (unary). Default: 0 (disabled). # # # fetch_secrets = 0 # # } + + # # broker: SPIFFE Broker API endpoint configuration. The broker endpoint + # # is opt-in; omitting the block disables it. At least one of socket_path + # # or bind_address MUST be set; both MAY be set to expose the endpoint + # # over both transports simultaneously. Kept under `experimental` while + # # the spec stabilizes — breaking changes may land before this moves to + # # the top-level config. + # # broker { + # # # socket_path: Bind the broker endpoint to a Unix domain socket + # # # at the given path. POSIX-only. Mutually compatible with + # # # bind_address (both can be set). + # # # socket_path = "/tmp/spire-agent/broker/api.sock" + # # + # # # bind_address: Bind the broker endpoint to a TCP address + # # # ("host:port"). Works on POSIX and Windows. The broker + # # # endpoint authenticates callers via mutual TLS using X.509-SVIDs + # # # so it is safe to expose over the network. + # # # bind_address = "0.0.0.0:8443" + # # + # # # allowed_reference_types_over_tcp: global allowlist of + # # # WorkloadReference type URLs permitted on the TCP listener. + # # # Empty (the default) denies every TCP request — fail-closed + # # # against remote use of local-only reference types such as + # # # PIDs. Entries are verbatim type URLs; no wildcard is + # # # honoured, so operators must enumerate what is safe over + # # # the network. Only meaningful when bind_address is set. + # # # UDS connections are unaffected. + # # # allowed_reference_types_over_tcp = [ + # # # "type.googleapis.com/spiffe.broker.KubernetesObjectReference", + # # # ] + # # + # # # brokers: brokers authorized to talk to this endpoint. At + # # # least one broker MUST be configured for the endpoint to + # # # start. Each entry's id is any valid SPIFFE ID; + # # # cross-trust-domain broker identities are allowed. + # # # allowed_reference_types restricts which WorkloadReference + # # # types the broker may use; each entry is the verbatim + # # # protobuf type URL the workload attestor plugin matches + # # # against. At least one entry is required; use ["*"] to allow + # # # any reference type the agent's attestor stack understands. + # # # brokers = [ + # # # { + # # # id = "spiffe://example.org/broker" + # # # allowed_reference_types = [ + # # # "type.googleapis.com/spiffe.broker.KubernetesObjectReference", + # # # ] + # # # }, + # # # ] + # # } # } } @@ -466,6 +515,52 @@ plugins { # Defaults to false. (Linux only) # verbose_container_locator_logs = false + # broker: Broker API-specific configuration used by AttestReference. + # Broker API uses AttestReference, so this block is required for + # all Broker API references handled by this plugin, including + # WorkloadPIDReference and KubernetesObjectReference. Each broker + # entry identifies a broker SPIFFE ID. IDs must be valid, unique, + # and non-empty. The broker SPIFFE ID is used as the + # SubjectAccessReview username; no groups are set. + # pod_reference_scope controls pod KubernetesObjectReference + # resolution for the broker. Valid values are "agent_node" + # (default) and "cluster". agent_node only accepts pods whose + # spec.nodeName matches this agent's node name. Non-pod object + # references always use the Kubernetes API server. + # + # Additional Kubernetes authorization requirements for Broker API + # object references: + # * The SPIRE agent ServiceAccount must be allowed to create + # subjectaccessreviews.authorization.k8s.io resources. + # * The broker SPIFFE ID must be allowed to use SPIRE's custom + # Kubernetes authorization verb "impersonate-via-spire" on + # every Kubernetes resource brokers may reference. Do not + # grant Kubernetes' built-in "impersonate" verb for this + # purpose: that verb can grant Kubernetes-native impersonation + # of users, groups, and ServiceAccounts. The custom verb is + # only a SPIRE broker authorization gate. + # + # Example broker-side Kubernetes RBAC for broker ID + # spiffe://example.org/broker: + # kind: ClusterRole + # rules: + # - apiGroups: [""] + # resources: ["pods"] + # verbs: ["impersonate-via-spire"] + # kind: ClusterRoleBinding + # subjects: + # - kind: User + # name: spiffe://example.org/broker + # + # broker { + # brokers = [ + # { + # id = "spiffe://example.org/broker" + # pod_reference_scope = "cluster" + # } + # ] + # } + # sigstore: sigstore options. Enables image cosign signatures checking. # sigstore { # allowed_identities: Maps OIDC issuer URIs to acceptable SANs in Fulcio certificates for validating signatures. diff --git a/doc/plugin_agent_workloadattestor_k8s.md b/doc/plugin_agent_workloadattestor_k8s.md index c42ff17614..da10a498ef 100644 --- a/doc/plugin_agent_workloadattestor_k8s.md +++ b/doc/plugin_agent_workloadattestor_k8s.md @@ -48,6 +48,7 @@ since [hostprocess](https://kubernetes.io/docs/tasks/configure-pod-container/cre | Configuration | Description | |----------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `api_server_cache.enabled` | If true, enables a controller-runtime Kubernetes API server cache for object-reference lookups. Defaults to false. | | `disable_container_selectors` | If true, container selectors are not produced. This can be used to produce pod selectors when the workload pod is known but the workload container is not ready at the time of attestation. | | `kubelet_read_only_port` | The kubelet read-only port. This is mutually exclusive with `kubelet_secure_port`. | | `kubelet_secure_port` | The kubelet secure port. It defaults to `10250` unless `kubelet_read_only_port` is set. | @@ -59,6 +60,7 @@ since [hostprocess](https://kubernetes.io/docs/tasks/configure-pod-container/cre | `use_anonymous_authentication` | If true, use anonymous authentication for kubelet communication | | `node_name_env` | The environment variable used to obtain the node name. Defaults to `MY_NODE_NAME`. | | `node_name` | The name of the node. Overrides the value obtained by the environment variable specified by `node_name_env`. | +| `broker` | Broker API options for `AttestReference`. Required when this plugin handles Broker API references. See [Broker API](#broker-api). | | `sigstore` | Sigstore options. Options described below. See [Sigstore options](#sigstore-options). When set, enables verification of container image signatures and attestations. | | `use_new_container_locator` | If true, enables the new container locator algorithm that has support for cgroups v2. Defaults to true. | | `verbose_container_locator_logs` | If true, enables verbose logging of mountinfo and cgroup information used to locate containers. Defaults to false. | @@ -125,6 +127,189 @@ If `ignore_tlog` is set to `true`, the selectors based on the Rekor bundle (`-lo > the pod, whereas `pod-image` and `pod-init-image` will match against ANY container or init container in the Pod, > respectively. +### Broker API + +When SPIRE Agent's [SPIFFE Broker API](spire_agent.md#spiffe-broker-api) is +enabled, the k8s workload attestor handles Broker API `AttestReference` +requests for `WorkloadPIDReference` and `KubernetesObjectReference`. +`AttestReference` requires a `broker` block in the plugin configuration. Each +`broker.brokers` entry identifies one broker SPIFFE ID that may use this +plugin. Broker IDs must be valid, unique, and non-empty. Each broker may set +`pod_reference_scope` to `agent_node` (default) or `cluster`; this only affects +pod `KubernetesObjectReference` resolution. + +Example: + +```hcl +WorkloadAttestor "k8s" { + plugin_data { + broker { + brokers = [ + { + id = "spiffe://example.org/broker" + pod_reference_scope = "cluster" + } + ] + } + } +} +``` + +`WorkloadPIDReference` follows the legacy PID-based k8s attestation path to +resolve the workload pod and selectors. When that Broker API reference +resolves to a pod, the plugin creates a `SubjectAccessReview` asking whether +the broker SPIFFE ID may use SPIRE's custom Kubernetes authorization verb +`impersonate-via-spire` on the resolved pod. The review uses the broker +SPIFFE ID as the SAR username and does not set groups. Legacy PID attestation +via the workload attestor's `Attest` RPC does not use the broker configuration +or run this review. + +For `KubernetesObjectReference`, the reference identifies the target object by +its resource (`.`, with `core` as the group string for core +resources) and either its namespaced name (`namespace` + `name`), its `uid`, +or both. Pod references try the local kubelet pod list first, then may fall +back to the Kubernetes API server. With the default +`pod_reference_scope = "agent_node"`, API server results are accepted only +when the resolved pod's `spec.nodeName` matches the agent node name from +`node_name` or `node_name_env`. With `pod_reference_scope = "cluster"`, pod +references may resolve to pods on any node. Non-pod object references are +resolved through the Kubernetes API server. The plugin then creates the same +`SubjectAccessReview` for the referenced object. The review uses the broker +SPIFFE ID as the SAR username, no groups, the reference's resource group and +plural, and the resolved namespace and name. If the authorizer denies the +review, attestation fails with `PermissionDenied`. Kubernetes API server +lookups require the agent ServiceAccount to have permission for the referenced +resource. + +**Pods (`pods/core`).** A `KubernetesObjectReference` to a pod is attested +through the same pod-resolution path as the legacy PID reference and emits +the **same** pod-shaped selectors documented in the table above +(`k8s:ns`, `k8s:sa`, `k8s:pod-name`, `k8s:container-name`, `k8s:pod-uid`, +`k8s:pod-label`, `k8s:pod-image`, `k8s:pod-owner`, ...). By default, broker +pod references are limited to pods scheduled to the agent's node +(`agent_node`), even when the pod is resolved through the Kubernetes API +server. Set `pod_reference_scope = "cluster"` for a broker that must +reference pods on other nodes. A registration entry written for the legacy +PID flow continues to match either reference type. + +**Other resources (any `.` for which the agent has permission).** +The agent fetches the object's `metadata` via the Kubernetes API server +(using a `PartialObjectMetadata` request) and emits a uniform vocabulary +that is independent of the resource's kind: + +| Selector | Value | +|-------------------------|-------------------------------------------------------------------------------------------------------------| +| k8s:uid | The object's UID. | +| k8s:resource | `.` for the resource (e.g. `deployments.apps`, `pods.core`). | +| k8s:plural | The resource plural (e.g. `deployments`). | +| k8s:group | The API group (e.g. `apps`); `core` for core resources. | +| k8s:version | The discovered version (e.g. `v1`, `v1beta1`). | +| k8s:apiVersion | The Kubernetes wire form: `v1` for core or `/` otherwise. | +| k8s:kind | The object kind (e.g. `Deployment`). | +| k8s:name | The object name. | +| k8s:namespace | The object namespace; omitted for cluster-scoped objects. | +| k8s:key | `/` for namespaced objects, or just `` for cluster-scoped ones. | +| k8s:label | A label on the object, formatted `:` (one selector per label entry). | +| k8s:owner-key | `//` for each entry in `metadata.ownerReferences`. | +| k8s:owner-uid | `//` for each entry in `metadata.ownerReferences`. | +| k8s:controller-key | Same as `k8s:owner-key`, but only emitted for owner references with `Controller: true`. | +| k8s:controller-uid | Same as `k8s:owner-uid`, but only emitted for owner references with `Controller: true`. | + +Annotations are intentionally **not** exposed as selectors. Kubernetes labels +are validated and indexed by the API server and are the appropriate identity +anchor; annotations are an unconstrained metadata grab bag (often multi-line +JSON) that does not fit equality-matched selectors. + +The agent's ServiceAccount needs `get` (and `list` when references identify +objects by `uid` alone) permission on every resource it is expected to +resolve via this path. It also needs `create` on +`subjectaccessreviews.authorization.k8s.io` to perform broker authorization +checks. For example: + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: spire-agent-k8s-attestor +rules: + # Object-reference resolution. Add every resource brokers may reference. + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list"] + - apiGroups: ["kustomize.toolkit.fluxcd.io"] + resources: ["kustomizations"] + verbs: ["get", "list"] + + # Required so the k8s workload attestor can create SubjectAccessReview + # objects while handling Broker API reference requests. + - apiGroups: ["authorization.k8s.io"] + resources: ["subjectaccessreviews"] + verbs: ["create"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: spire-agent-k8s-attestor +subjects: + - kind: ServiceAccount + name: spire-agent + namespace: spire +roleRef: + kind: ClusterRole + name: spire-agent-k8s-attestor + apiGroup: rbac.authorization.k8s.io +``` + +The Kubernetes authorizer must also allow the broker SPIFFE ID to use SPIRE's +custom **`impersonate-via-spire`** verb on the resources brokers may reference. +The `SubjectAccessReview` checks the broker SPIFFE ID as a Kubernetes username +and does not set groups. The broker pod's Kubernetes ServiceAccount is not +used as the reviewed subject. + +SPIRE intentionally does **not** use Kubernetes' built-in `impersonate` verb +for this check. The built-in verb has broader meaning in Kubernetes RBAC: it +can authorize native impersonation of users, groups, ServiceAccounts, and +other impersonation targets. Granting that built-in verb to broker-related +Kubernetes identities would create powerful RBAC subjects with permissions +outside SPIRE's broker authorization decision. The `impersonate-via-spire` +verb is a SPIRE-specific authorization gate checked only by this +`SubjectAccessReview`; granting it lets Kubernetes answer SPIRE's question +without granting Kubernetes-native impersonation power. + +For example, with broker ID `spiffe://example.org/broker`: + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: spire-broker-impersonation +rules: + # This is the broker authorization decision enforced by SubjectAccessReview. + # Use SPIRE's custom verb, not Kubernetes' built-in `impersonate` verb. + - apiGroups: [""] + resources: ["pods"] + verbs: ["impersonate-via-spire"] + - apiGroups: ["kustomize.toolkit.fluxcd.io"] + resources: ["kustomizations"] + verbs: ["impersonate-via-spire"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: spire-broker-impersonation +subjects: + - kind: User + name: spiffe://example.org/broker +roleRef: + kind: ClusterRole + name: spire-broker-impersonation + apiGroup: rbac.authorization.k8s.io +``` + +A `Role` and `RoleBinding` can be used instead when brokers should only be +allowed to authorize SPIRE broker references for namespaced resources in a +specific namespace. + ### Image selector limitations The `container-image`, `pod-image`, and `pod-init-image` selectors are derived from two fields in the Kubernetes [ContainerStatus](https://pkg.go.dev/k8s.io/api/core/v1#ContainerStatus): `Image` (typically the tag-based name, e.g. `myimage:v1.2.3`) and `ImageID` (typically the digest-based identifier, e.g. `myimage@sha256:abc...`). Both values are emitted as selectors for each container to support matching by either form. @@ -201,6 +386,18 @@ WorkloadAttestor "k8s" { } ``` +To enable the Kubernetes API server cache used by object-reference lookups: + +```hcl +WorkloadAttestor "k8s" { + plugin_data { + api_server_cache { + enabled = true + } + } +} +``` + ### Platform support This plugin is only supported on Unix systems. diff --git a/doc/spire_agent.md b/doc/spire_agent.md index 6dfa61f2d9..62c7b37033 100644 --- a/doc/spire_agent.md +++ b/doc/spire_agent.md @@ -86,6 +86,7 @@ This may be useful for templating configuration files, for example across differ | `require_pq_kem` | Require use of a post-quantum-safe key exchange method for TLS handshakes | false | | `jwt_svid_cache_hit_timeout` | Custom gRPC timeout (between 5 and 30s) when retrieving a NewJWTSVID when a valid JWT-SVID in cache | 30s | | `ratelimit` | Optional per-caller rate limiting for Workload API and SDS methods, enforced after workload attestation. See [Workload API Rate Limiting](#workload-api-rate-limiting) for details. | | +| `broker` | Optional SPIFFE Broker API endpoint configuration. See [SPIFFE Broker API](#spiffe-broker-api). | | ### Workload API Rate Limiting @@ -519,6 +520,139 @@ agent { } ``` +## SPIFFE Broker API + +The SPIFFE Broker API lets an authorized infrastructure component (a "broker") +obtain SVIDs and trust bundles on behalf of workloads that the broker +references, rather than the workload connecting to SPIRE Agent itself. It +implements the [SPIFFE Broker API](https://github.com/spiffe/spiffe/blob/main/standards/SPIFFE_Broker_API.md) +specification and is served from SPIRE Agent over a dedicated, mutually +authenticated endpoint that is distinct from the Workload API. + +> **Status:** experimental. The Broker API is being stabilized as the +> upstream SPIFFE specification matures; breaking changes may land before +> this configuration moves out of `experimental`. + +How it differs from the existing APIs: + +* The **Workload API** is invoked by a workload to obtain its own SVID, + and the workload is attested implicitly via its socket peer credentials. +* The **Delegated Identity API** lets an authorized delegate impersonate + any workload in the agent's scope; the delegate is responsible for + identifying the target workload (by selectors or PID). +* The **Broker API** is also delegate-style — a broker presents a + `WorkloadReference` identifying a workload, and the agent runs its + configured workload attestor stack against that reference to produce + selectors. Unlike the Delegated Identity API, the broker never + hand-supplies selectors; the agent's own attestor remains the source of + truth for what selectors apply to a reference. This makes the Broker + API safe to expose across the network (over mTLS) and not just over a + local UDS. + +The specification currently defines two reference types: + +* `WorkloadPIDReference` — a process ID on the agent's node. Per + [SPIFFE Broker API §3.1.2](https://github.com/spiffe/spiffe/blob/main/standards/SPIFFE_Broker_API.md#312-reference-scope), + this is a local reference and should not cross network boundaries. + SPIRE Agent enforces this server-side via `allowed_reference_types_over_tcp` + (see [Configuration](#configuration)), which defaults to empty — + fail-closed: every TCP request is denied unless the operator explicitly + opts the reference type in. +* `KubernetesObjectReference` — an arbitrary Kubernetes object identified + by `type` (resource and group) plus `key` (namespace and name) and/or + `uid`. SPIRE Agent's `k8s` workload attestor can support this reference + type when its `allowed_reference_types` includes + `type.googleapis.com/spiffe.broker.KubernetesObjectReference`, and emits + selectors describing the object (`k8s:pod-name`, etc. for pods; or the + generic `k8s:resource`, `k8s:kind`, `k8s:name`, ... for any other + resource). + +When the `k8s` workload attestor handles Broker API `AttestReference` calls, +its plugin-level `broker` block is required for every reference type. It uses +that broker configuration to run Kubernetes `SubjectAccessReview` checks +against the resolved pod for `WorkloadPIDReference`, and against the referenced +object for `KubernetesObjectReference`. + +### Transport and authentication + +The Broker API is gated by mutual TLS using X.509-SVIDs. A broker is +expected to obtain its own SVID via the Workload API first, then use that +SVID to dial the broker endpoint. The agent presents its own SVID and +verifies the broker's certificate against the configured per-broker +allowlist. + +Per the spec, every request must carry the `broker.spiffe.io: true` gRPC +metadata header as an SSRF mitigation. SPIRE Agent rejects requests that +omit it with `InvalidArgument`. This applies to _every_ method, including +gRPC server reflection (which is only served over UDS). Tools like +`grpcurl`/`grpcui` do not send the header by default, so reflection calls +must set it explicitly, e.g.: + +```sh +grpcurl -unix -rpc-header 'broker.spiffe.io: true' /path/to/broker.sock list +``` + +### Configuration + +The endpoint is enabled by adding a `broker` block under `experimental` +in the agent config. At least one of `socket_path` (POSIX-only UDS) or +`bind_address` (TCP, all platforms) must be set; both may be set +simultaneously. + +The `socket_path` must not live in the same directory as the Workload +API socket — SPIRE Agent will refuse to start otherwise. + +Each entry in `brokers` enumerates an authorized broker's SPIFFE ID +(cross-trust-domain identities are allowed) and the reference types it +may use. The `allowed_reference_types` list takes verbatim protobuf type +URLs and must contain at least one entry; use `"*"` to allow any type +the agent's attestor stack understands. + +When a TCP listener is configured, the global `allowed_reference_types_over_tcp` +list further restricts which reference types may be used over TCP. Empty +(the default) denies every TCP request. Entries are verbatim type URLs; +no wildcard is honoured, so operators must enumerate what is safe over +the network. The TCP gate applies on top of each broker's per-broker +`allowed_reference_types`. UDS connections are unaffected. + +```hcl +agent { + trust_domain = "example.org" + ... + experimental { + broker { + socket_path = "/run/spire/broker-sockets/broker.sock" # POSIX UDS + bind_address = "0.0.0.0:8443" # optional TCP + + # Empty by default (TCP denied). Enumerate the verbatim + # reference type URLs that may be used over TCP. No wildcard + # is honoured — operators must list each type explicitly. + allowed_reference_types_over_tcp = [ + "type.googleapis.com/spiffe.broker.KubernetesObjectReference", + ] + + brokers = [ + { + id = "spiffe://example.org/some/broker" + allowed_reference_types = ["*"] + }, + { + id = "spiffe://example.org/restricted/broker" + allowed_reference_types = [ + "type.googleapis.com/spiffe.broker.KubernetesObjectReference", + ] + }, + ] + } + } +} +``` + +Brokers whose ID isn't in this list are rejected at the TLS layer. +Brokers in the list but using a reference type outside their +`allowed_reference_types` are rejected with `PermissionDenied` at the +gRPC layer. + ## Envoy SDS Support SPIRE agent has support for the [Envoy](https://envoyproxy.io) [Secret Discovery Service](https://www.envoyproxy.io/docs/envoy/latest/configuration/security/secret) (SDS). diff --git a/go.mod b/go.mod index e3baedf06c..7a0e6197da 100644 --- a/go.mod +++ b/go.mod @@ -77,9 +77,9 @@ require ( github.com/sigstore/sigstore v1.10.8 github.com/sirupsen/logrus v1.9.4 github.com/smallstep/pkcs7 v0.2.1 - github.com/spiffe/go-spiffe/v2 v2.7.0 + github.com/spiffe/go-spiffe/v2 v2.8.0 github.com/spiffe/spire-api-sdk v1.2.5-0.20260428072036-00f73a61093a - github.com/spiffe/spire-plugin-sdk v1.4.4-0.20251120194148-791bbd274dc7 + github.com/spiffe/spire-plugin-sdk v1.4.4-0.20260617144146-5dcde407c4d1 github.com/stretchr/testify v1.11.1 github.com/uber-go/tally/v4 v4.1.17 github.com/valyala/fastjson v1.6.10 @@ -162,6 +162,7 @@ require ( github.com/evanphx/json-patch/v5 v5.9.11 // indirect github.com/fatih/color v1.19.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/fsnotify/fsnotify v1.10.1 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/go-chi/chi/v5 v5.3.0 // indirect github.com/go-jose/go-jose/v3 v3.0.5 // indirect @@ -315,11 +316,13 @@ require ( golang.org/x/oauth2 v0.36.0 // indirect golang.org/x/term v0.44.0 // indirect golang.org/x/text v0.38.0 // indirect + gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/genproto v0.0.0-20260406210006-6f92a3bedf2d // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/apiextensions-apiserver v0.36.0 // indirect k8s.io/klog/v2 v2.140.0 // indirect k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a // indirect k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 // indirect diff --git a/go.sum b/go.sum index c1fc0a122e..f133670a67 100644 --- a/go.sum +++ b/go.sum @@ -262,6 +262,8 @@ github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMD github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 h1:Yzb9+7DPaBjB8zlTR87/ElzFsnQfuHnVUVqpZZIcV5Y= github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= +github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8P3k= +github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -789,12 +791,12 @@ github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiT github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spiffe/go-spiffe/v2 v2.7.0 h1:uXe1MflJoHw58wAUvxVlcM7WpKtijWG7I1UidcGh6g4= -github.com/spiffe/go-spiffe/v2 v2.7.0/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U= +github.com/spiffe/go-spiffe/v2 v2.8.0 h1:vHCTEZYhpXZ9y6JkIouIdHLJobWGUFn2467/WsXHHjA= +github.com/spiffe/go-spiffe/v2 v2.8.0/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U= github.com/spiffe/spire-api-sdk v1.2.5-0.20260428072036-00f73a61093a h1:x+oSCqSwCrpSUMHR0cuuRkFQAM6yAjVa0c5yAUOZjVg= github.com/spiffe/spire-api-sdk v1.2.5-0.20260428072036-00f73a61093a/go.mod h1:9hXJcMzatM1KwAtBDO3s6HccDCic++/5c2yOc5Iln8Y= -github.com/spiffe/spire-plugin-sdk v1.4.4-0.20251120194148-791bbd274dc7 h1:OAvr7TNirmBpXnAp82cTosuB+JAus5cyFCRqXHE0WHs= -github.com/spiffe/spire-plugin-sdk v1.4.4-0.20251120194148-791bbd274dc7/go.mod h1:QvrRDiBlXiJ7kNd176ZHsF5eklxxeTRgJSu2CXe0MKw= +github.com/spiffe/spire-plugin-sdk v1.4.4-0.20260617144146-5dcde407c4d1 h1:gN8nWPdRSxYIACrgdPBuVwhNe6ABzpvUVza7cP8Yy38= +github.com/spiffe/spire-plugin-sdk v1.4.4-0.20260617144146-5dcde407c4d1/go.mod h1:QvrRDiBlXiJ7kNd176ZHsF5eklxxeTRgJSu2CXe0MKw= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= @@ -1070,6 +1072,8 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= +gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/api v0.284.0 h1:i+cKTgeQRcRySkP7QTl5PDO7/pAm8EcMFIUMlNbk4Vc= diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index 3b68f5a2aa..7ed47cab02 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -19,6 +19,7 @@ import ( admin_api "github.com/spiffe/spire/pkg/agent/api" node_attestor "github.com/spiffe/spire/pkg/agent/attestor/node" workload_attestor "github.com/spiffe/spire/pkg/agent/attestor/workload" + "github.com/spiffe/spire/pkg/agent/broker" "github.com/spiffe/spire/pkg/agent/catalog" "github.com/spiffe/spire/pkg/agent/endpoints" "github.com/spiffe/spire/pkg/agent/manager" @@ -292,6 +293,25 @@ func (a *Agent) Run(ctx context.Context) error { tasks = append(tasks, adminEndpoints.ListenAndServe) } + if len(a.c.Broker.BindAddresses) != 0 { + brokerEndpoints, err := broker.New(&broker.Config{ + BindAddrs: a.c.Broker.BindAddresses, + Manager: mgr, + Log: a.c.Log, + Metrics: metrics, + Attestor: workloadAttestor, + Brokers: a.c.Broker.Brokers, + AllowedReferenceTypesOverTCP: a.c.Broker.AllowedReferenceTypesOverTCP, + SVIDSource: liveAgentSVIDSource{m: mgr}, + BundleSource: mgr.GetX509Bundle(), + TLSPolicy: a.c.TLSPolicy, + }) + if err != nil { + return fmt.Errorf("failed to create broker endpoints: %w", err) + } + tasks = append(tasks, brokerEndpoints.ListenAndServe) + } + if a.c.LogReopener != nil { tasks = append(tasks, a.c.LogReopener) } @@ -558,6 +578,27 @@ type agentHealthDetails struct { WorkloadAPIErr string `json:"make_new_x509_err,omitempty"` } +// liveAgentSVIDSource adapts the agent manager into an x509svid.Source that +// always reflects the manager's currently rotated agent SVID. Without this, +// passing the bootstrap AttestationResult directly would freeze the listener +// on the bootstrap cert until it expires. +type liveAgentSVIDSource struct { + m manager.Manager +} + +func (s liveAgentSVIDSource) GetX509SVID() (*x509svid.SVID, error) { + state := s.m.GetCurrentCredentials() + id, err := x509svid.IDFromCert(state.SVID[0]) + if err != nil { + return nil, err + } + return &x509svid.SVID{ + ID: id, + Certificates: state.SVID, + PrivateKey: state.Key, + }, nil +} + func errString(suppress bool, err error) string { if suppress { return "" diff --git a/pkg/agent/api/delegatedidentity/v1/service_test.go b/pkg/agent/api/delegatedidentity/v1/service_test.go index bebea30049..abb4d479a5 100644 --- a/pkg/agent/api/delegatedidentity/v1/service_test.go +++ b/pkg/agent/api/delegatedidentity/v1/service_test.go @@ -36,6 +36,7 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials/insecure" + "google.golang.org/protobuf/types/known/anypb" ) var ( @@ -955,6 +956,10 @@ func (fa FakeWorkloadPIDAttestor) Attest(_ context.Context, _ int) ([]*common.Se return fa.selectors, fa.err } +func (fa FakeWorkloadPIDAttestor) AttestReference(_ context.Context, _ *anypb.Any) ([]*common.Selector, error) { + return fa.selectors, fa.err +} + type FakeManager struct { manager.Manager diff --git a/pkg/agent/attestor/node/node.go b/pkg/agent/attestor/node/node.go index de8d52096d..65597fd608 100644 --- a/pkg/agent/attestor/node/node.go +++ b/pkg/agent/attestor/node/node.go @@ -36,13 +36,6 @@ const ( roundRobinServiceConfig = `{ "loadBalancingConfig": [ { "round_robin": {} } ] }` ) -type AttestationResult struct { - SVID []*x509.Certificate - Key keymanager.Key - Bundle *spiffebundle.Bundle - Reattestable bool -} - type Attestor interface { Attest(ctx context.Context) (*AttestationResult, error) } diff --git a/pkg/agent/attestor/node/result.go b/pkg/agent/attestor/node/result.go new file mode 100644 index 0000000000..27417ca1ff --- /dev/null +++ b/pkg/agent/attestor/node/result.go @@ -0,0 +1,39 @@ +package attestor + +import ( + "crypto/x509" + "fmt" + + "github.com/spiffe/go-spiffe/v2/bundle/spiffebundle" + "github.com/spiffe/go-spiffe/v2/bundle/x509bundle" + "github.com/spiffe/go-spiffe/v2/spiffeid" + "github.com/spiffe/go-spiffe/v2/svid/x509svid" + "github.com/spiffe/spire/pkg/agent/plugin/keymanager" +) + +// Allow AttestationResult to be used as go-spiffe SVID and bundle sources. +var ( + _ x509svid.Source = (*AttestationResult)(nil) + _ x509bundle.Source = (*AttestationResult)(nil) +) + +type AttestationResult struct { + SVID []*x509.Certificate + Key keymanager.Key + Bundle *spiffebundle.Bundle + Reattestable bool +} + +func (ar *AttestationResult) GetX509SVID() (*x509svid.SVID, error) { + return &x509svid.SVID{ + Certificates: ar.SVID, + PrivateKey: ar.Key, + }, nil +} + +func (ar *AttestationResult) GetX509BundleForTrustDomain(trustDomain spiffeid.TrustDomain) (*x509bundle.Bundle, error) { + if ar.Bundle.TrustDomain() != trustDomain { + return nil, fmt.Errorf("bundle for trust domain %q not found", trustDomain) + } + return ar.Bundle.X509Bundle(), nil +} diff --git a/pkg/agent/attestor/workload/workload.go b/pkg/agent/attestor/workload/workload.go index 04e6535487..febc794fc2 100644 --- a/pkg/agent/attestor/workload/workload.go +++ b/pkg/agent/attestor/workload/workload.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "os" + "sync" "github.com/sirupsen/logrus" "github.com/spiffe/spire/pkg/agent/catalog" @@ -12,14 +13,20 @@ import ( "github.com/spiffe/spire/pkg/common/telemetry" telemetry_workload "github.com/spiffe/spire/pkg/common/telemetry/agent/workloadapi" "github.com/spiffe/spire/proto/spire/common" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/anypb" ) +var errReferenceUnsupported = errors.New("workload reference type unsupported by attestor") + type attestor struct { c *Config } type Attestor interface { Attest(ctx context.Context, pid int) ([]*common.Selector, error) + AttestReference(ctx context.Context, reference *anypb.Any) ([]*common.Selector, error) } func New(config *Config) Attestor { @@ -46,75 +53,126 @@ type Config struct { // Attest invokes all workload attestor plugins against the provided PID. If some // attestors fail, the errors are logged and selectors from the failing plugins // are discarded. If all attestors fail, the combined error is returned. -func (wla *attestor) Attest(ctx context.Context, pid int) (_ []*common.Selector, retErr error) { +func (wla *attestor) Attest(ctx context.Context, pid int) ([]*common.Selector, error) { + log := wla.c.Log.WithField(telemetry.PID, pid) + + selectors, err := wla.attest(ctx, func(a workloadattestor.WorkloadAttestor) ([]*common.Selector, error) { + var err error + counter := telemetry_workload.StartAttestorCall(wla.c.Metrics, a.Name()) + defer counter.Done(&err) + + selectors, err := a.Attest(ctx, pid) + if err != nil { + log.WithError(err).Errorf("workload attestor %q failed", a.Name()) + return nil, fmt.Errorf("workload attestor %q failed: %w", a.Name(), err) + } + return selectors, nil + }, nil, nil) + if err != nil { + return nil, err + } + + // The agent health check currently exercises the Workload API. Since this + // can happen with some frequency, it has a tendency to fill up logs with + // hard-to-filter details if we're not careful (e.g. issue #1537). Only log + // if it is not the agent itself. + if pid != os.Getpid() { + log.WithField(telemetry.Selectors, selectors).Debug("PID attested to have selectors") + } + + return selectors, nil +} + +func (wla *attestor) AttestReference(ctx context.Context, reference *anypb.Any) ([]*common.Selector, error) { + log := wla.c.Log.WithField(telemetry.ReferenceType, reference.GetTypeUrl()) + selectors, err := wla.attest(ctx, func(a workloadattestor.WorkloadAttestor) ([]*common.Selector, error) { + var err error + counter := telemetry_workload.StartAttestorCall(wla.c.Metrics, a.Name()) + defer counter.Done(&err) + + selectors, err := a.AttestReference(ctx, reference) + if err != nil { + if status.Code(err) == codes.Unimplemented { + log.WithError(err).Debugf("workload attestor %q does not support reference attestation", a.Name()) + err = nil + return nil, errReferenceUnsupported + } + log.WithError(err).Errorf("workload attestor %q failed", a.Name()) + return nil, fmt.Errorf("workload attestor %q failed: %w", a.Name(), err) + } + return selectors, nil + }, errReferenceUnsupported, status.Error(codes.Unimplemented, "no workload attestor handled reference")) + if err != nil { + return nil, err + } + log.WithField(telemetry.Selectors, selectors).Debug("Reference attested to have selectors") + return selectors, nil +} + +func (wla *attestor) attest(ctx context.Context, attestFunc func(attestor workloadattestor.WorkloadAttestor) ([]*common.Selector, error), skippableErr error, allSkippedErr error) (_ []*common.Selector, retErr error) { counter := telemetry_workload.StartAttestationCall(wla.c.Metrics) defer counter.Done(&retErr) - log := wla.c.Log.WithField(telemetry.PID, pid) - plugins := wla.c.Catalog.GetWorkloadAttestors() - sChan := make(chan []*common.Selector) - errChan := make(chan error) - + // Buffered so plugin goroutines never block sending if the outer loop + // returns early (e.g., on ctx cancellation). Combined with the deferred + // wg.Wait() below, this guarantees plugin-level logs are flushed before + // we return to the caller. + sChan := make(chan []*common.Selector, len(plugins)) + errChan := make(chan error, len(plugins)) + + var wg sync.WaitGroup for _, p := range plugins { - go func() { - if selectors, err := wla.invokeAttestor(ctx, p, pid); err == nil { + wg.Go(func() { + if selectors, err := attestFunc(p); err == nil { sChan <- selectors } else { errChan <- err } - }() + }) } + defer wg.Wait() // Collect the results - var selectors []*common.Selector + selectors := []*common.Selector{} + successes := 0 + skipped := 0 var errs []error for range plugins { select { case s := <-sChan: + successes++ selectors = append(selectors, s...) wla.c.selectorHook(selectors) case err := <-errChan: if ctx.Err() != nil { - log.WithError(ctx.Err()).Error("Timed out collecting selectors for PID") + wla.c.Log.WithError(ctx.Err()).Error("Timed out collecting selectors") return nil, ctx.Err() } + if skippableErr != nil && errors.Is(err, skippableErr) { + skipped++ + continue + } errs = append(errs, err) case <-ctx.Done(): - log.WithError(ctx.Err()).Error("Timed out collecting selectors for PID") + wla.c.Log.WithError(ctx.Err()).Error("Timed out collecting selectors") return nil, ctx.Err() } } - if len(plugins) > 0 && len(errs) == len(plugins) { - return nil, errors.Join(errs...) + if len(plugins) > 0 && successes == 0 { + if len(errs) > 0 { + return nil, errors.Join(errs...) + } + if skipped > 0 && allSkippedErr != nil { + return nil, allSkippedErr + } } if len(errs) > 0 { - log.WithError(errors.Join(errs...)).Error("Failed to collect all selectors for PID") + wla.c.Log.WithError(errors.Join(errs...)).Error("Failed to collect all selectors") } telemetry_workload.AddDiscoveredSelectorsSample(wla.c.Metrics, float32(len(selectors))) - - // The agent health check currently exercises the Workload API. Since this - // can happen with some frequency, it has a tendency to fill up logs with - // hard-to-filter details if we're not careful (e.g. issue #1537). Only log - // if it is not the agent itself. - if pid != os.Getpid() { - log.WithField(telemetry.Selectors, selectors).Debug("PID attested to have selectors") - } - - return selectors, nil -} - -// invokeAttestor invokes attestation against the supplied plugin. Should be called from a goroutine. -func (wla *attestor) invokeAttestor(ctx context.Context, a workloadattestor.WorkloadAttestor, pid int) (_ []*common.Selector, err error) { - counter := telemetry_workload.StartAttestorCall(wla.c.Metrics, a.Name()) - defer counter.Done(&err) - - selectors, err := a.Attest(ctx, pid) - if err != nil { - return nil, fmt.Errorf("workload attestor %q failed: %w", a.Name(), err) - } return selectors, nil } diff --git a/pkg/agent/attestor/workload/workload_test.go b/pkg/agent/attestor/workload/workload_test.go index 3ad153d780..2b0ab15a21 100644 --- a/pkg/agent/attestor/workload/workload_test.go +++ b/pkg/agent/attestor/workload/workload_test.go @@ -18,6 +18,9 @@ import ( "github.com/spiffe/spire/test/fakes/fakeworkloadattestor" "github.com/spiffe/spire/test/spiretest" "github.com/stretchr/testify/suite" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/anypb" ) var ( @@ -112,18 +115,54 @@ func (s *WorkloadAttestorTestSuite) TestAttestLogsOnPartialFailure() { selectors, err := s.attestor.Attest(ctx, 3) s.Nil(err) spiretest.AssertProtoListEqual(s.T(), selectors2, selectors) - spiretest.AssertLogs(s.T(), s.loggerHook.AllEntries(), []spiretest.LogEntry{ + spiretest.AssertLogsAnyOrder(s.T(), s.loggerHook.AllEntries(), []spiretest.LogEntry{ { Level: logrus.ErrorLevel, - Message: "Failed to collect all selectors for PID", + Message: `workload attestor "fake1" failed`, Data: logrus.Fields{ telemetry.PID: "3", + logrus.ErrorKey: "rpc error: code = Unknown desc = workloadattestor(fake1): cannot attest pid 3", + }, + }, + { + Level: logrus.ErrorLevel, + Message: "Failed to collect all selectors", + Data: logrus.Fields{ logrus.ErrorKey: `workload attestor "fake1" failed: rpc error: code = Unknown desc = workloadattestor(fake1): cannot attest pid 3`, }, }, }) } +func (s *WorkloadAttestorTestSuite) TestAttestReferenceSkipsUnsupportedAttestors() { + s.catalog.SetWorkloadAttestors( + &referenceWorkloadAttestor{name: "k8s", selectors: selectors1}, + &referenceWorkloadAttestor{name: "unix", err: status.Error(codes.Unimplemented, "AttestReference not implemented")}, + ) + + selectors, err := s.attestor.AttestReference(ctx, &anypb.Any{TypeUrl: "type.googleapis.com/example.Reference"}) + s.Require().NoError(err) + spiretest.AssertProtoListEqual(s.T(), selectors1, selectors) + for _, entry := range s.loggerHook.AllEntries() { + s.NotEqual(logrus.ErrorLevel, entry.Level) + } +} + +func (s *WorkloadAttestorTestSuite) TestAttestReferenceReturnsUnimplementedWhenNoAttestorHandlesReference() { + s.catalog.SetWorkloadAttestors( + &referenceWorkloadAttestor{name: "unix", err: status.Error(codes.Unimplemented, "AttestReference not implemented")}, + &referenceWorkloadAttestor{name: "docker", err: status.Error(codes.Unimplemented, "AttestReference not implemented")}, + ) + + selectors, err := s.attestor.AttestReference(ctx, &anypb.Any{TypeUrl: "type.googleapis.com/example.Reference"}) + s.Require().Error(err) + s.Equal(codes.Unimplemented, status.Code(err)) + s.Empty(selectors) + for _, entry := range s.loggerHook.AllEntries() { + s.NotEqual(logrus.ErrorLevel, entry.Level) + } +} + func (s *WorkloadAttestorTestSuite) TestAttestWorkloadMetrics() { // Add only one attestor s.catalog.SetWorkloadAttestors( @@ -205,16 +244,45 @@ func (s *WorkloadAttestorTestSuite) TestAttestLogsOnContextCancellation() { // Wait for attestation goroutine to complete execution <-attestCh - s.Nil(selectors) - s.Error(attestErr) - spiretest.AssertLogs(s.T(), s.loggerHook.AllEntries(), []spiretest.LogEntry{ + s.Assert().Nil(selectors) + s.Assert().Error(attestErr) + spiretest.AssertLogsAnyOrder(s.T(), s.loggerHook.AllEntries(), []spiretest.LogEntry{ { Level: logrus.ErrorLevel, - Message: "Timed out collecting selectors for PID", + Message: "Timed out collecting selectors", Data: logrus.Fields{ - telemetry.PID: fmt.Sprint(pid), logrus.ErrorKey: context.Canceled.Error(), }, }, + { + Level: logrus.ErrorLevel, + Message: `workload attestor "faketimeoutattestor" failed`, + Data: logrus.Fields{ + telemetry.PID: fmt.Sprint(pid), + logrus.ErrorKey: "rpc error: code = Canceled desc = workloadattestor(faketimeoutattestor): context canceled", + }, + }, }) } + +type referenceWorkloadAttestor struct { + name string + selectors []*common.Selector + err error +} + +func (a *referenceWorkloadAttestor) Name() string { + return a.name +} + +func (a *referenceWorkloadAttestor) Type() string { + return "WorkloadAttestor" +} + +func (a *referenceWorkloadAttestor) Attest(context.Context, int) ([]*common.Selector, error) { + return nil, status.Error(codes.Unimplemented, "Attest not implemented") +} + +func (a *referenceWorkloadAttestor) AttestReference(context.Context, *anypb.Any) ([]*common.Selector, error) { + return a.selectors, a.err +} diff --git a/pkg/agent/broker/api/service.go b/pkg/agent/broker/api/service.go new file mode 100644 index 0000000000..ac8e6ba5eb --- /dev/null +++ b/pkg/agent/broker/api/service.go @@ -0,0 +1,453 @@ +package api + +import ( + "context" + "crypto/x509" + "errors" + "fmt" + "net" + "time" + + "github.com/sirupsen/logrus" + "github.com/spiffe/go-spiffe/v2/exp/proto/spiffe/broker" + "github.com/spiffe/go-spiffe/v2/spiffeid" + "github.com/spiffe/spire/pkg/agent/api/rpccontext" + workloadattestor "github.com/spiffe/spire/pkg/agent/attestor/workload" + "github.com/spiffe/spire/pkg/agent/broker/brokercontext" + "github.com/spiffe/spire/pkg/agent/client" + "github.com/spiffe/spire/pkg/agent/common/hintsfilter" + "github.com/spiffe/spire/pkg/agent/manager" + "github.com/spiffe/spire/pkg/agent/manager/cache" + "github.com/spiffe/spire/pkg/common/bundleutil" + "github.com/spiffe/spire/pkg/common/telemetry" + "github.com/spiffe/spire/pkg/common/telemetry/agent/adminapi" + "github.com/spiffe/spire/pkg/common/x509util" + "github.com/spiffe/spire/proto/spire/common" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/anypb" +) + +// RegisterService registers the SPIFFE Broker API service on the provided server. +func RegisterService(s *grpc.Server, service *Service) { + broker.RegisterAPIServer(s, service) +} + +type Config struct { + Log logrus.FieldLogger + Metrics telemetry.Metrics + Manager manager.Manager + Attestor workloadattestor.Attestor + + // AllowedReferenceTypesByCaller restricts which WorkloadReference type + // URLs each authenticated caller (broker SPIFFE ID) may use. A caller + // missing from the map has no restriction (the broker config builder + // omits brokers that opted into "*"). The set values are full protobuf + // type URLs (e.g. + // `type.googleapis.com/spiffe.broker.KubernetesObjectReference`). + AllowedReferenceTypesByCaller map[spiffeid.ID]map[string]struct{} + + // AllowedReferenceTypesOverTCP is the global allowlist of WorkloadReference + // type URLs permitted on TCP connections. Empty denies every TCP request + // (fail-closed default). No wildcard is honoured — operators must + // enumerate the type URLs explicitly. UDS connections are unaffected. + AllowedReferenceTypesOverTCP map[string]struct{} +} + +func New(config Config) *Service { + return &Service{ + manager: config.Manager, + peerAttestor: config.Attestor, + metrics: config.Metrics, + allowedReferenceTypesByCaller: config.AllowedReferenceTypesByCaller, + allowedReferenceTypesOverTCP: config.AllowedReferenceTypesOverTCP, + } +} + +// Service implements the SPIFFE Broker API server. +type Service struct { + broker.UnimplementedAPIServer + + manager manager.Manager + peerAttestor workloadattestor.Attestor + metrics telemetry.Metrics + allowedReferenceTypesByCaller map[spiffeid.ID]map[string]struct{} + allowedReferenceTypesOverTCP map[string]struct{} +} + +// authorizeReferenceType applies two gates: (1) the per-broker allowlist, +// (2) the global TCP allowlist when the connection is a TCP peer. Either +// gate failing yields PermissionDenied. A caller missing from the per-broker +// map has no per-broker restriction (the broker opted into "*"). The TCP +// gate is explicit-or-nothing — it only admits requests whose reference +// type URL is in the configured set, defaulting to deny-all when the set +// is empty. No wildcard is honoured for the TCP gate by design. +func (s *Service) authorizeReferenceType(ctx context.Context, caller spiffeid.ID, ref *anypb.Any) error { + // Reject malformed requests before the allowlist gates so a missing or + // empty reference yields InvalidArgument rather than PermissionDenied. + if ref.GetTypeUrl() == "" { + return status.Error(codes.InvalidArgument, "workload reference must be provided") + } + + allowed, ok := s.allowedReferenceTypesByCaller[caller] + if ok { + if _, ok := allowed[ref.GetTypeUrl()]; !ok { + return status.Errorf(codes.PermissionDenied, "broker %q is not allowed to use reference type %q", caller, ref.GetTypeUrl()) + } + } + if isTCPCaller(ctx) { + if _, ok := s.allowedReferenceTypesOverTCP[ref.GetTypeUrl()]; !ok { + return status.Errorf(codes.PermissionDenied, "reference type %q is not allowed over TCP", ref.GetTypeUrl()) + } + } + return nil +} + +// isTCPCaller reports whether the incoming gRPC connection is over TCP +// (as opposed to a Unix domain socket). +func isTCPCaller(ctx context.Context) bool { + p, ok := peer.FromContext(ctx) + if !ok { + return false + } + _, tcp := p.Addr.(*net.TCPAddr) + return tcp +} + +func (s *Service) getCallerContext(ctx context.Context) (spiffeid.ID, error) { + // The broker endpoint configures the gRPC server with plain + // credentials.NewTLS (to keep SessionTicketsDisabled and TLS policy + // customizations on the *tls.Config). That credentials wrapper exposes + // the peer SPIFFE ID via credentials.TLSInfo rather than go-spiffe's + // grpccredentials authInfo, so we extract it ourselves here. + p, ok := peer.FromContext(ctx) + if !ok || p.AuthInfo == nil { + return spiffeid.ID{}, status.Error(codes.Unauthenticated, "unable to determine caller identity") + } + tlsInfo, ok := p.AuthInfo.(credentials.TLSInfo) + if !ok || tlsInfo.SPIFFEID == nil { + return spiffeid.ID{}, status.Error(codes.Unauthenticated, "unable to determine caller identity") + } + id, err := spiffeid.FromString(tlsInfo.SPIFFEID.String()) + if err != nil { + return spiffeid.ID{}, status.Errorf(codes.Unauthenticated, "invalid caller SPIFFE ID: %v", err) + } + return id, nil +} + +func (s *Service) SubscribeToX509SVID(req *broker.SubscribeToX509SVIDRequest, stream broker.API_SubscribeToX509SVIDServer) error { + latency := adminapi.StartFirstX509SVIDUpdateLatency(s.metrics) + ctx := stream.Context() + log := rpccontext.Logger(ctx) + var receivedFirstUpdate bool + + peer, err := s.getCallerContext(ctx) + if err != nil { + return err + } + log = log.WithField("broker_peer", peer.String()) + + if err := s.authorizeReferenceType(ctx, peer, req.GetReference().GetReference()); err != nil { + return err + } + + selectors, err := s.constructValidSelectorsFromReference(brokercontext.WithCallerID(ctx, peer), log, req.Reference) + if err != nil { + return err + } + + log.WithField(telemetry.Selectors, selectors).Debug("Subscribing to cache changes") + + subscriber, err := s.manager.SubscribeToCacheChanges(ctx, selectors) + if err != nil { + log.WithError(err).Error("Subscribe to cache changes failed") + return err + } + defer subscriber.Finish() + + for { + select { + case update := <-subscriber.Updates(): + update.Identities = hintsfilter.FilterIdentities(update.Identities, log) + if len(update.Identities) > 0 && !receivedFirstUpdate { + // emit latency metric for first update containing an SVID. + latency.Measure() + receivedFirstUpdate = true + } + + if err := sendX509SVIDResponse(update, stream, log); err != nil { + return err + } + case <-ctx.Done(): + return nil + } + } +} + +func (s *Service) SubscribeToX509Bundles(req *broker.SubscribeToX509BundlesRequest, stream broker.API_SubscribeToX509BundlesServer) error { + ctx := stream.Context() + log := rpccontext.Logger(ctx) + + peer, err := s.getCallerContext(ctx) + if err != nil { + return err + } + log = log.WithField("broker_peer", peer.String()) + + if err := s.authorizeReferenceType(ctx, peer, req.GetReference().GetReference()); err != nil { + return err + } + + // The bundle response is workload-independent, but per the SPIFFE Broker + // API spec the request still identifies a workload. Validate the reference + // resolves so a caller can't pull bundles for workloads it can't attest. + if _, err := s.constructValidSelectorsFromReference(brokercontext.WithCallerID(ctx, peer), log, req.Reference); err != nil { + return err + } + + subscriber := s.manager.SubscribeToBundleChanges() + + send := func(bundles map[spiffeid.TrustDomain]*cache.Bundle) error { + // Rebuild the map each tick — Next()/Value() return the full set of + // trust domains, so reusing the previous map would leave entries for + // trust domains that have since been removed. + caCerts := make(map[string][]byte, len(bundles)) + for td, bundle := range bundles { + caCerts[td.IDString()] = marshalBundle(bundle.X509Authorities()) + } + return stream.Send(&broker.SubscribeToX509BundlesResponse{Bundles: caCerts}) + } + + if err := send(subscriber.Value()); err != nil { + return err + } + + for { + select { + case <-subscriber.Changes(): + if err := send(subscriber.Next()); err != nil { + return err + } + case <-ctx.Done(): + return nil + } + } +} + +func (s *Service) FetchJWTSVID(ctx context.Context, req *broker.FetchJWTSVIDRequest) (*broker.FetchJWTSVIDResponse, error) { + log := rpccontext.Logger(ctx) + if len(req.Audience) == 0 { + log.Error("Missing required audience parameter") + return nil, status.Error(codes.InvalidArgument, "audience must be specified") + } + + peer, err := s.getCallerContext(ctx) + if err != nil { + return nil, err + } + log = log.WithField("broker_peer", peer.String()) + + if err := s.authorizeReferenceType(ctx, peer, req.GetReference().GetReference()); err != nil { + return nil, err + } + + selectors, err := s.constructValidSelectorsFromReference(brokercontext.WithCallerID(ctx, peer), log, req.Reference) + if err != nil { + return nil, err + } + + resp := new(broker.FetchJWTSVIDResponse) + entries := s.manager.MatchingRegistrationEntries(selectors) + entries = hintsfilter.FilterRegistrations(entries, log) + for _, entry := range entries { + spiffeID, err := spiffeid.FromString(entry.SpiffeId) + if err != nil { + log.WithField(telemetry.SPIFFEID, entry.SpiffeId).WithError(err).Error("Invalid requested SPIFFE ID") + return nil, status.Errorf(codes.InvalidArgument, "invalid requested SPIFFE ID: %v", err) + } + + loopLog := log.WithField(telemetry.SPIFFEID, spiffeID.String()) + + var svid *client.JWTSVID + svid, err = s.manager.FetchJWTSVID(ctx, entry, req.Audience) + if err != nil { + loopLog.WithError(err).Error("Could not fetch JWT-SVID") + return nil, status.Errorf(codes.Unavailable, "could not fetch JWT-SVID: %v", err) + } + resp.Svids = append(resp.Svids, &broker.JWTSVID{ + SpiffeId: spiffeID.String(), + Hint: entry.Hint, + Svid: svid.Token, + }) + + ttl := time.Until(svid.ExpiresAt) + loopLog.WithField(telemetry.TTL, ttl.Seconds()).Debug("Fetched JWT SVID") + } + + if len(resp.Svids) == 0 { + log.Error("No identity issued") + return nil, status.Error(codes.PermissionDenied, "no identity issued") + } + + return resp, nil +} + +func (s *Service) SubscribeToJWTBundles(req *broker.SubscribeToJWTBundlesRequest, stream broker.API_SubscribeToJWTBundlesServer) error { + ctx := stream.Context() + log := rpccontext.Logger(ctx) + + peer, err := s.getCallerContext(ctx) + if err != nil { + return err + } + log = log.WithField("broker_peer", peer.String()) + + if err := s.authorizeReferenceType(ctx, peer, req.GetReference().GetReference()); err != nil { + return err + } + + // The bundle response is workload-independent, but per the SPIFFE Broker + // API spec the request still identifies a workload. Validate the reference + // resolves so a caller can't pull bundles for workloads it can't attest. + if _, err := s.constructValidSelectorsFromReference(brokercontext.WithCallerID(ctx, peer), log, req.Reference); err != nil { + return err + } + + subscriber := s.manager.SubscribeToBundleChanges() + + send := func(bundles map[spiffeid.TrustDomain]*cache.Bundle) error { + // Rebuild the map each tick — Next()/Value() return the full set of + // trust domains, so reusing the previous map would leave entries for + // trust domains that have since been removed. + jwtbundles := make(map[string][]byte, len(bundles)) + for td, bundle := range bundles { + jwksBytes, err := bundleutil.Marshal(bundle, bundleutil.NoX509SVIDKeys(), bundleutil.StandardJWKS()) + if err != nil { + return err + } + jwtbundles[td.IDString()] = jwksBytes + } + return stream.Send(&broker.SubscribeToJWTBundlesResponse{Bundles: jwtbundles}) + } + + if err := send(subscriber.Value()); err != nil { + return err + } + + for { + select { + case <-subscriber.Changes(): + if err := send(subscriber.Next()); err != nil { + return err + } + case <-ctx.Done(): + return nil + } + } +} + +func (s *Service) constructValidSelectorsFromReference(ctx context.Context, log logrus.FieldLogger, ref *broker.WorkloadReference) ([]*common.Selector, error) { + if ref == nil { + log.Error("No workload reference provided") + return nil, status.Error(codes.InvalidArgument, "workload reference must be provided") + } + + selectors, err := s.peerAttestor.AttestReference(ctx, ref.Reference) + if err != nil { + log.WithError(err).Error("Workload attestation failed") + // Preserve the plugin's status (InvalidArgument, NotFound, + // PermissionDenied, etc.); only opaque errors are wrapped as + // Unauthenticated. + if _, ok := status.FromError(err); ok { + return nil, err + } + return nil, status.Errorf(codes.Unauthenticated, "workload attestation failed: %v", err) + } + + return selectors, nil +} + +func sendX509SVIDResponse(update *cache.WorkloadUpdate, stream broker.API_SubscribeToX509SVIDServer, log logrus.FieldLogger) (err error) { + resp, notAfters, err := composeX509SVIDBySelectors(update) + if err != nil { + log.WithError(err).Error("Could not serialize X.509 SVID response") + return status.Error(codes.Internal, "could not serialize response") + } + + if err := stream.Send(resp); err != nil { + log.WithError(err).Error("Failed to send X.509 SVID response") + return err + } + + log = log.WithField(telemetry.Count, len(resp.Svids)) + + // log details on each SVID + // a response has already been sent so nothing is + // blocked on this logic + for i, svid := range resp.Svids { + ttl := time.Until(notAfters[i]) + log.WithFields(logrus.Fields{ + telemetry.SPIFFEID: svid.SpiffeId, + telemetry.TTL: ttl.Seconds(), + }).Debug("Fetched X.509 SVID for broker") + } + + return nil +} + +func composeX509SVIDBySelectors(update *cache.WorkloadUpdate) (*broker.SubscribeToX509SVIDResponse, []time.Time, error) { + resp := new(broker.SubscribeToX509SVIDResponse) + resp.Svids = make([]*broker.X509SVID, 0, len(update.Identities)) + resp.FederatedBundles = make(map[string][]byte, len(update.FederatedBundles)) + notAfters := make([]time.Time, 0, len(update.Identities)) + + x509Bundle := marshalBundle(update.Bundle.X509Authorities()) + for _, identity := range update.Identities { + // Do not send admin nor downstream SVIDs to the caller + if identity.Entry.Admin || identity.Entry.Downstream { + continue + } + + // check if SVIDs exist for the identity + if len(identity.SVID) == 0 { + return nil, nil, errors.New("unable to get SVID from identity") + } + + id, err := spiffeid.FromString(identity.Entry.SpiffeId) + if err != nil { + return nil, nil, fmt.Errorf("error during SPIFFE ID parsing: %w", err) + } + + keyData, err := x509.MarshalPKCS8PrivateKey(identity.PrivateKey) + if err != nil { + return nil, nil, fmt.Errorf("marshal key for %v: %w", id, err) + } + + svid := &broker.X509SVID{ + SpiffeId: id.String(), + X509Svid: x509util.DERFromCertificates(identity.SVID), + Bundle: x509Bundle, + Hint: identity.Entry.Hint, + X509SvidKey: keyData, + } + resp.Svids = append(resp.Svids, svid) + notAfters = append(notAfters, identity.SVID[0].NotAfter) + } + + for td, bundle := range update.FederatedBundles { + resp.FederatedBundles[td.IDString()] = marshalBundle(bundle.X509Authorities()) + } + + return resp, notAfters, nil +} + +func marshalBundle(certs []*x509.Certificate) []byte { + bundle := []byte{} + for _, c := range certs { + bundle = append(bundle, c.Raw...) + } + return bundle +} diff --git a/pkg/agent/broker/api/service_test.go b/pkg/agent/broker/api/service_test.go new file mode 100644 index 0000000000..f65680dc9e --- /dev/null +++ b/pkg/agent/broker/api/service_test.go @@ -0,0 +1,143 @@ +package api + +import ( + "context" + "net" + "testing" + + "github.com/spiffe/go-spiffe/v2/spiffeid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/anypb" +) + +const ( + k8sType = "type.googleapis.com/spiffe.broker.KubernetesObjectReference" + pidType = "type.googleapis.com/spiffe.broker.PIDReference" +) + +func udsContext() context.Context { + return peer.NewContext(context.Background(), &peer.Peer{ + Addr: &net.UnixAddr{Name: "/tmp/broker.sock", Net: "unix"}, + }) +} + +func tcpContext() context.Context { + return peer.NewContext(context.Background(), &peer.Peer{ + Addr: &net.TCPAddr{IP: net.ParseIP("10.0.0.1"), Port: 12345}, + }) +} + +func ref(typeURL string) *anypb.Any { + if typeURL == "" { + return nil + } + return &anypb.Any{TypeUrl: typeURL} +} + +func TestIsTCPCaller(t *testing.T) { + assert.True(t, isTCPCaller(tcpContext())) + assert.False(t, isTCPCaller(udsContext())) + assert.False(t, isTCPCaller(context.Background())) +} + +func TestAuthorizeReferenceType(t *testing.T) { + caller := spiffeid.RequireFromString("spiffe://example.org/broker") + other := spiffeid.RequireFromString("spiffe://example.org/other") + + for _, tt := range []struct { + name string + byCaller map[spiffeid.ID]map[string]struct{} + overTCP map[string]struct{} + ctx context.Context + caller spiffeid.ID + ref *anypb.Any + expectCode codes.Code + }{ + { + name: "nil reference is rejected before any gate", + ctx: udsContext(), + caller: caller, + ref: ref(""), + expectCode: codes.InvalidArgument, + }, + { + name: "empty type url is rejected", + ctx: udsContext(), + caller: caller, + ref: &anypb.Any{}, + expectCode: codes.InvalidArgument, + }, + { + name: "caller present and type allowed over UDS", + byCaller: map[spiffeid.ID]map[string]struct{}{caller: {k8sType: {}}}, + ctx: udsContext(), + caller: caller, + ref: ref(k8sType), + expectCode: codes.OK, + }, + { + name: "caller present and type not allowed", + byCaller: map[spiffeid.ID]map[string]struct{}{caller: {k8sType: {}}}, + ctx: udsContext(), + caller: caller, + ref: ref(pidType), + expectCode: codes.PermissionDenied, + }, + { + name: "caller absent from map behaves as wildcard over UDS", + byCaller: map[spiffeid.ID]map[string]struct{}{other: {k8sType: {}}}, + ctx: udsContext(), + caller: caller, + ref: ref(pidType), + expectCode: codes.OK, + }, + { + name: "TCP caller allowed when type in TCP allowlist", + overTCP: map[string]struct{}{k8sType: {}}, + ctx: tcpContext(), + caller: caller, + ref: ref(k8sType), + expectCode: codes.OK, + }, + { + name: "TCP caller denied when type not in TCP allowlist", + overTCP: map[string]struct{}{k8sType: {}}, + ctx: tcpContext(), + caller: caller, + ref: ref(pidType), + expectCode: codes.PermissionDenied, + }, + { + name: "TCP caller denied by default when TCP allowlist empty", + ctx: tcpContext(), + caller: caller, + ref: ref(k8sType), + expectCode: codes.PermissionDenied, + }, + { + name: "UDS caller unaffected by empty TCP allowlist", + ctx: udsContext(), + caller: caller, + ref: ref(k8sType), + expectCode: codes.OK, + }, + } { + t.Run(tt.name, func(t *testing.T) { + s := New(Config{ + AllowedReferenceTypesByCaller: tt.byCaller, + AllowedReferenceTypesOverTCP: tt.overTCP, + }) + err := s.authorizeReferenceType(tt.ctx, tt.caller, tt.ref) + if tt.expectCode == codes.OK { + require.NoError(t, err) + return + } + require.Error(t, err) + assert.Equal(t, tt.expectCode, status.Code(err)) + }) + } +} diff --git a/pkg/agent/broker/brokercontext/context.go b/pkg/agent/broker/brokercontext/context.go new file mode 100644 index 0000000000..776d3d3698 --- /dev/null +++ b/pkg/agent/broker/brokercontext/context.go @@ -0,0 +1,50 @@ +package brokercontext + +import ( + "context" + "errors" + "fmt" + + "github.com/spiffe/go-spiffe/v2/spiffeid" + "google.golang.org/grpc/metadata" +) + +const callerIDMetadataKey = "spire-agent-broker-caller-id" + +type callerIDKey struct{} + +func WithCallerID(ctx context.Context, id spiffeid.ID) context.Context { + return context.WithValue(ctx, callerIDKey{}, id) +} + +func CallerIDFromContext(ctx context.Context) (spiffeid.ID, bool, error) { + if id, ok := ctx.Value(callerIDKey{}).(spiffeid.ID); ok { + return id, true, nil + } + + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return spiffeid.ID{}, false, nil + } + values := md.Get(callerIDMetadataKey) + switch len(values) { + case 0: + return spiffeid.ID{}, false, nil + case 1: + id, err := spiffeid.FromString(values[0]) + if err != nil { + return spiffeid.ID{}, false, fmt.Errorf("invalid broker caller SPIFFE ID: %w", err) + } + return id, true, nil + default: + return spiffeid.ID{}, false, errors.New("multiple broker caller SPIFFE IDs provided") + } +} + +func AppendCallerIDToOutgoingContext(ctx context.Context) context.Context { + id, ok := ctx.Value(callerIDKey{}).(spiffeid.ID) + if !ok { + return ctx + } + return metadata.AppendToOutgoingContext(ctx, callerIDMetadataKey, id.String()) +} diff --git a/pkg/agent/broker/endpoints.go b/pkg/agent/broker/endpoints.go new file mode 100644 index 0000000000..e29a79429e --- /dev/null +++ b/pkg/agent/broker/endpoints.go @@ -0,0 +1,306 @@ +package broker + +import ( + "context" + "errors" + "fmt" + "net" + "slices" + "strings" + + "github.com/sirupsen/logrus" + "github.com/spiffe/go-spiffe/v2/spiffeid" + "github.com/spiffe/go-spiffe/v2/spiffetls/tlsconfig" + attestor "github.com/spiffe/spire/pkg/agent/attestor/workload" + brokerapi "github.com/spiffe/spire/pkg/agent/broker/api" + "github.com/spiffe/spire/pkg/agent/endpoints" + "github.com/spiffe/spire/pkg/agent/manager" + "github.com/spiffe/spire/pkg/common/api/middleware" + "github.com/spiffe/spire/pkg/common/telemetry" + "github.com/spiffe/spire/pkg/common/tlspolicy" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/reflection" + "google.golang.org/grpc/status" + + "github.com/spiffe/go-spiffe/v2/bundle/x509bundle" + "github.com/spiffe/go-spiffe/v2/svid/x509svid" + "golang.org/x/sync/errgroup" +) + +type Config struct { + // BindAddrs are the addresses the endpoint listens on. Each may be a + // `*net.UnixAddr` or a `*net.TCPAddr`; the same gRPC server (and same + // mTLS configuration) is fanned out across all of them. + BindAddrs []net.Addr + + Manager manager.Manager + + Log logrus.FieldLogger + + Metrics telemetry.Metrics + + Attestor attestor.Attestor + + // Brokers enumerates the brokers authorized to talk to this endpoint. + // mTLS at the listener gates the set; brokers absent from it are + // rejected at the TLS layer. + Brokers []Broker + + // AllowedReferenceTypesOverTCP is the global allowlist of WorkloadReference + // type URLs permitted on the TCP listener. Empty denies every TCP + // request with PermissionDenied. Wildcards are not honoured for this + // gate — entries are verbatim type URLs. UDS connections are not + // gated by this list. + AllowedReferenceTypesOverTCP []string + + SVIDSource x509svid.Source + BundleSource x509bundle.Source + + // TLSPolicy controls the post-quantum-safe TLS policy applied to the + // inbound mTLS listener. + TLSPolicy tlspolicy.Policy +} + +// Broker identifies a broker authorized to talk to the SPIFFE Broker API +// endpoint, and carries any per-broker configuration. +type Broker struct { + // ID is the SPIFFE ID of the broker. Cross-trust-domain broker + // identities are allowed. + ID string + + // AllowedReferenceTypes restricts which WorkloadReference types this + // broker is permitted to use. Each entry is the verbatim protobuf type + // URL the workload attestor plugin matches against (e.g. + // `type.googleapis.com/spiffe.broker.KubernetesObjectReference`). + // Use `"*"` to allow any reference type this agent's workload attestor + // stack understands. Must list at least one entry. + AllowedReferenceTypes []string +} + +type Endpoints struct { + c *Config +} + +func New(c *Config) (*Endpoints, error) { + switch { + case len(c.BindAddrs) == 0: + return nil, errors.New("at least one bind address is required") + case c.Manager == nil: + return nil, errors.New("manager is required") + case c.Log == nil: + return nil, errors.New("log is required") + case c.Metrics == nil: + return nil, errors.New("metrics is required") + case c.Attestor == nil: + return nil, errors.New("attestor is required") + case c.SVIDSource == nil: + return nil, errors.New("SVID source is required") + case c.BundleSource == nil: + return nil, errors.New("bundle source is required") + } + return &Endpoints{ + c: c, + }, nil +} + +func (e *Endpoints) ListenAndServe(ctx context.Context) error { + unaryInterceptor, streamInterceptor := middleware.Interceptors( + middleware.Chain( + endpoints.Middleware(e.c.Log, e.c.Metrics), + middleware.Preprocess(restrictReflectionToUDS), + middleware.Preprocess(verifyBrokerSecurityHeader), + ), + ) + + // TODO(arndt): Delegated Identity API allows to be served without any authorized peer. + // I think it's better to fail as it's a misconfiguration and having that socket up + // without any authorized peer is just a potential security risk. + if len(e.c.Brokers) == 0 { + return errors.New("at least one broker is required") + } + + brokerIDs, err := brokerIDsAsSPIFFEIDs(e.c.Brokers) + if err != nil { + return fmt.Errorf("failed to parse broker IDs: %w", err) + } + + // In comparison to the admin endpoints, this API is secured by mutual TLS using X.509 SVIDs. + // Clients of this API are expected to use the Workload API to obtain their SVIDs first. + // This is to accommodate environments where this API is served over network. + tlsConfig := tlsconfig.MTLSServerConfig(e.c.SVIDSource, e.c.BundleSource, tlsconfig.AuthorizeOneOf(brokerIDs...)) + // Disable session ticket resumption so the peer-authorization callback + // runs on every connection — same rationale as the SPIRE server endpoint. + tlsConfig.SessionTicketsDisabled = true + if err := tlspolicy.ApplyPolicy(tlsConfig, e.c.TLSPolicy); err != nil { + return fmt.Errorf("failed to apply TLS policy: %w", err) + } + server := grpc.NewServer( + grpc.Creds(credentials.NewTLS(tlsConfig)), + grpc.UnaryInterceptor(unaryInterceptor), + grpc.StreamInterceptor(streamInterceptor), + ) + + e.registerBrokerAPI(server) + reflection.Register(server) + + listeners := make([]net.Listener, 0, len(e.c.BindAddrs)) + defer func() { + for _, l := range listeners { + _ = l.Close() + } + }() + for _, addr := range e.c.BindAddrs { + var l net.Listener + switch addr.Network() { + case "unix": + l, err = createUDSListener(addr) + case "tcp": + // TCP is permitted because the SPIFFE Broker API is secured by + // mutual TLS using X.509-SVIDs, so the listener is safe to + // expose over the network. + l, err = net.Listen("tcp", addr.String()) + default: + err = fmt.Errorf("unsupported network type %q for broker endpoint", addr.Network()) + } + if err != nil { + return fmt.Errorf("failed to listen on broker address %q: %w", addr.String(), err) + } + listeners = append(listeners, l) + e.c.Log.WithFields(logrus.Fields{ + telemetry.Network: l.Addr().Network(), + telemetry.Address: l.Addr().String(), + }).Info("Starting SPIFFE Broker Endpoint") + } + + // Fan one gRPC server out across every listener with an errgroup. The + // first goroutine to error (or context cancellation) cancels the + // errgroup's context, which the watcher goroutine uses to call + // server.Stop(), causing every blocked Serve to return. + g, gCtx := errgroup.WithContext(ctx) + for _, l := range listeners { + g.Go(func() error { + if err := server.Serve(l); err != nil && !errors.Is(err, grpc.ErrServerStopped) { + return err + } + return nil + }) + } + g.Go(func() error { + <-gCtx.Done() + e.c.Log.Info("Stopping SPIFFE Broker Endpoint") + server.Stop() + return nil + }) + + if err := g.Wait(); err != nil { + e.c.Log.WithError(err).Error("SPIFFE Broker Endpoint stopped prematurely") + return err + } + e.c.Log.Info("SPIFFE Broker Endpoint has stopped") + return nil +} + +func (e *Endpoints) registerBrokerAPI(server *grpc.Server) { + service := brokerapi.New(brokerapi.Config{ + Manager: e.c.Manager, + Attestor: e.c.Attestor, + Metrics: e.c.Metrics, + Log: e.c.Log.WithField(telemetry.SubsystemName, telemetry.BrokerAPI), + AllowedReferenceTypesByCaller: buildAllowedReferenceTypeMap(e.c.Brokers), + AllowedReferenceTypesOverTCP: buildAllowedReferenceTypesOverTCPSet(e.c.AllowedReferenceTypesOverTCP), + }) + + brokerapi.RegisterService(server, service) +} + +// buildAllowedReferenceTypesOverTCPSet materializes the global TCP allowlist +// into a set for O(1) membership checks. An empty input set means every +// TCP request will be rejected at the api service. +func buildAllowedReferenceTypesOverTCPSet(types []string) map[string]struct{} { + set := make(map[string]struct{}, len(types)) + for _, t := range types { + set[t] = struct{}{} + } + return set +} + +// buildAllowedReferenceTypeMap pre-computes the per-caller allowlist used +// by the api service to gate WorkloadReference type usage. Brokers whose +// allowed list contains "*" are absent from the map; the api service +// treats absence as "no restriction." Brokers whose ID fails to parse +// are skipped — the mTLS authorizer will already have rejected them at +// the listener level since the same parse runs in brokerIDsAsSPIFFEIDs. +func buildAllowedReferenceTypeMap(brokers []Broker) map[spiffeid.ID]map[string]struct{} { + if len(brokers) == 0 { + return nil + } + out := make(map[spiffeid.ID]map[string]struct{}) + for _, b := range brokers { + if slices.Contains(b.AllowedReferenceTypes, "*") { + continue + } + id, err := spiffeid.FromString(b.ID) + if err != nil { + continue + } + set := make(map[string]struct{}, len(b.AllowedReferenceTypes)) + for _, t := range b.AllowedReferenceTypes { + set[t] = struct{}{} + } + out[id] = set + } + return out +} + +func restrictReflectionToUDS(ctx context.Context, fullMethod string, _ any) (context.Context, error) { + if !isReflectionMethod(fullMethod) { + return ctx, nil + } + + p, ok := peer.FromContext(ctx) + if !ok { + return nil, status.Error(codes.Internal, "no peer information available") + } + if p.Addr.Network() != "unix" { + return nil, status.Error(codes.PermissionDenied, "server reflection is only available over Unix sockets") + } + return ctx, nil +} + +func isReflectionMethod(fullMethod string) bool { + return strings.HasPrefix(fullMethod, "/"+middleware.ServerReflectionServiceName+"/") || + strings.HasPrefix(fullMethod, "/"+middleware.ServerReflectionV1AlphaServiceName+"/") +} + +// verifyBrokerSecurityHeader enforces the SPIFFE Broker Endpoint spec +// requirement that every request carries the `broker.spiffe.io: true` gRPC +// metadata header — an SSRF mitigation analogous to the Workload API's +// `workload.spiffe.io` header. Requests missing or mismatching it are +// rejected with InvalidArgument. +func verifyBrokerSecurityHeader(ctx context.Context, _ string, _ any) (context.Context, error) { + md, _ := metadata.FromIncomingContext(ctx) + values := md["broker.spiffe.io"] + if len(values) != 1 || values[0] != "true" { + return nil, status.Error(codes.InvalidArgument, "security header missing from request") + } + return ctx, nil +} + +// brokerIDsAsSPIFFEIDs parses each broker's ID into a spiffeid.ID. Brokers +// from any trust domain are allowed; mTLS at the listener still pins +// authorization to the configured set of identities. +func brokerIDsAsSPIFFEIDs(brokers []Broker) ([]spiffeid.ID, error) { + ids := make([]spiffeid.ID, 0, len(brokers)) + for _, b := range brokers { + id, err := spiffeid.FromString(b.ID) + if err != nil { + return nil, err + } + ids = append(ids, id) + } + return ids, nil +} diff --git a/pkg/agent/broker/endpoints_fallback.go b/pkg/agent/broker/endpoints_fallback.go new file mode 100644 index 0000000000..9d46b700cb --- /dev/null +++ b/pkg/agent/broker/endpoints_fallback.go @@ -0,0 +1,12 @@ +//go:build windows + +package broker + +import ( + "errors" + "net" +) + +func createUDSListener(_ net.Addr) (net.Listener, error) { + return nil, errors.New("unsupported platform for broker API") +} diff --git a/pkg/agent/broker/endpoints_posix.go b/pkg/agent/broker/endpoints_posix.go new file mode 100644 index 0000000000..ef0f359c00 --- /dev/null +++ b/pkg/agent/broker/endpoints_posix.go @@ -0,0 +1,29 @@ +//go:build !windows + +package broker + +import ( + "fmt" + "net" + "os" + + "github.com/spiffe/spire/pkg/common/util" +) + +func createUDSListener(bindAddr net.Addr) (net.Listener, error) { + if bindAddr.Network() != "unix" { + return nil, fmt.Errorf("unsupported network type %q for UDS listener", bindAddr.Network()) + } + + // Remove uds if already exists + os.Remove(bindAddr.String()) + + l, err := net.ListenUnix(bindAddr.Network(), util.GetUnixAddr(bindAddr.String())) + if err != nil { + return nil, fmt.Errorf("error creating UDS listener: %w", err) + } + if err := os.Chmod(bindAddr.String(), 0770); err != nil { + return nil, fmt.Errorf("unable to change UDS permissions: %w", err) + } + return l, nil +} diff --git a/pkg/agent/broker/endpoints_test.go b/pkg/agent/broker/endpoints_test.go new file mode 100644 index 0000000000..79c74c1116 --- /dev/null +++ b/pkg/agent/broker/endpoints_test.go @@ -0,0 +1,138 @@ +package broker + +import ( + "context" + "net" + "testing" + + "github.com/spiffe/spire/pkg/common/api/middleware" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/status" +) + +func TestRestrictReflectionToUDS(t *testing.T) { + for _, tt := range []struct { + name string + fullMethod string + addr net.Addr + expectCode codes.Code + }{ + { + name: "non-reflection method without peer is allowed", + fullMethod: "/spiffe.broker.API/FetchX509SVID", + expectCode: codes.OK, + }, + { + name: "reflection over UDS is allowed", + fullMethod: "/" + middleware.ServerReflectionServiceName + "/ServerReflectionInfo", + addr: &net.UnixAddr{Name: "/tmp/broker.sock", Net: "unix"}, + expectCode: codes.OK, + }, + { + name: "reflection v1alpha over UDS is allowed", + fullMethod: "/" + middleware.ServerReflectionV1AlphaServiceName + "/ServerReflectionInfo", + addr: &net.UnixAddr{Name: "/tmp/broker.sock", Net: "unix"}, + expectCode: codes.OK, + }, + { + name: "reflection over TCP is denied", + fullMethod: "/" + middleware.ServerReflectionServiceName + "/ServerReflectionInfo", + addr: &net.TCPAddr{IP: net.ParseIP("127.0.0.1"), Port: 8081}, + expectCode: codes.PermissionDenied, + }, + { + name: "reflection without peer is internal", + fullMethod: "/" + middleware.ServerReflectionServiceName + "/ServerReflectionInfo", + expectCode: codes.Internal, + }, + } { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + if tt.addr != nil { + ctx = peer.NewContext(ctx, &peer.Peer{Addr: tt.addr}) + } + + _, err := restrictReflectionToUDS(ctx, tt.fullMethod, nil) + if tt.expectCode == codes.OK { + require.NoError(t, err) + return + } + require.Equal(t, tt.expectCode, status.Code(err)) + }) + } +} + +// TestPreprocessorChain exercises restrictReflectionToUDS and +// verifyBrokerSecurityHeader composed in the same order as the gRPC +// interceptor chain, since the security header is enforced for every +// method including reflection over UDS. +func TestPreprocessorChain(t *testing.T) { + reflectionMethod := "/" + middleware.ServerReflectionServiceName + "/ServerReflectionInfo" + udsAddr := &net.UnixAddr{Name: "/tmp/broker.sock", Net: "unix"} + tcpAddr := &net.TCPAddr{IP: net.ParseIP("127.0.0.1"), Port: 8081} + + for _, tt := range []struct { + name string + fullMethod string + addr net.Addr + header bool + expectCode codes.Code + }{ + { + name: "reflection over UDS without header is denied", + fullMethod: reflectionMethod, + addr: udsAddr, + header: false, + expectCode: codes.InvalidArgument, + }, + { + name: "reflection over UDS with header is allowed", + fullMethod: reflectionMethod, + addr: udsAddr, + header: true, + expectCode: codes.OK, + }, + { + name: "reflection over TCP is denied before header check", + fullMethod: reflectionMethod, + addr: tcpAddr, + header: true, + expectCode: codes.PermissionDenied, + }, + { + name: "non-reflection method without header is denied", + fullMethod: "/spiffe.broker.API/FetchX509SVID", + addr: udsAddr, + header: false, + expectCode: codes.InvalidArgument, + }, + { + name: "non-reflection method with header is allowed", + fullMethod: "/spiffe.broker.API/FetchX509SVID", + addr: udsAddr, + header: true, + expectCode: codes.OK, + }, + } { + t.Run(tt.name, func(t *testing.T) { + ctx := peer.NewContext(context.Background(), &peer.Peer{Addr: tt.addr}) + if tt.header { + ctx = metadata.NewIncomingContext(ctx, metadata.Pairs("broker.spiffe.io", "true")) + } + + ctx, err := restrictReflectionToUDS(ctx, tt.fullMethod, nil) + if err == nil { + _, err = verifyBrokerSecurityHeader(ctx, tt.fullMethod, nil) + } + + if tt.expectCode == codes.OK { + require.NoError(t, err) + return + } + require.Equal(t, tt.expectCode, status.Code(err)) + }) + } +} diff --git a/pkg/agent/common/hintsfilter/hintsfilter.go b/pkg/agent/common/hintsfilter/hintsfilter.go new file mode 100644 index 0000000000..50dd8fac79 --- /dev/null +++ b/pkg/agent/common/hintsfilter/hintsfilter.go @@ -0,0 +1,91 @@ +// Package hintsfilter deduplicates registration entries (and the identities +// derived from them) so that callers of the agent's local APIs see at most +// one entry per hint. Used by both the Workload API and the SPIFFE Broker +// API so that hint semantics are consistent across them. +package hintsfilter + +import ( + "github.com/sirupsen/logrus" + "github.com/spiffe/spire/pkg/agent/manager/cache" + "github.com/spiffe/spire/pkg/common/telemetry" + "github.com/spiffe/spire/proto/spire/common" +) + +// FilterRegistrations returns entries with duplicate hints removed, +// preferring the older entry (by CreatedAt) with EntryId as the +// deterministic tie-breaker. +func FilterRegistrations(entries []*common.RegistrationEntry, log logrus.FieldLogger) []*common.RegistrationEntry { + entriesToRemove := getEntriesToRemove(entries, log) + + var filteredEntries []*common.RegistrationEntry + for _, entry := range entries { + if _, ok := entriesToRemove[entry.EntryId]; !ok { + filteredEntries = append(filteredEntries, entry) + } + } + return filteredEntries +} + +// FilterIdentities returns identities whose underlying registration entries +// survive hint deduplication. Same tie-breaking rules as FilterRegistrations. +func FilterIdentities(identities []cache.Identity, log logrus.FieldLogger) []cache.Identity { + entries := make([]*common.RegistrationEntry, 0, len(identities)) + for _, identity := range identities { + entries = append(entries, identity.Entry) + } + entriesToRemove := getEntriesToRemove(entries, log) + + var filteredIdentities []cache.Identity + for _, identity := range identities { + if _, ok := entriesToRemove[identity.Entry.EntryId]; !ok { + filteredIdentities = append(filteredIdentities, identity) + } + } + return filteredIdentities +} + +func getEntriesToRemove(entries []*common.RegistrationEntry, log logrus.FieldLogger) map[string]struct{} { + entriesToRemove := make(map[string]struct{}) + hintsMap := make(map[string]*common.RegistrationEntry) + + for _, entry := range entries { + if entry.Hint == "" { + continue + } + if entryWithNonUniqueHint, ok := hintsMap[entry.Hint]; ok { + entryToMaintain, entryToRemove := hintTieBreaking(entry, entryWithNonUniqueHint) + + hintsMap[entry.Hint] = entryToMaintain + entriesToRemove[entryToRemove.EntryId] = struct{}{} + + log.WithFields(logrus.Fields{ + telemetry.Hint: entryToRemove.Hint, + telemetry.RegistrationID: entryToRemove.EntryId, + }).Warn("Ignoring entry with duplicate hint") + } else { + hintsMap[entry.Hint] = entry + } + } + + return entriesToRemove +} + +func hintTieBreaking(entryA, entryB *common.RegistrationEntry) (maintain, remove *common.RegistrationEntry) { + switch { + case entryA.CreatedAt < entryB.CreatedAt: + maintain = entryA + remove = entryB + case entryA.CreatedAt > entryB.CreatedAt: + maintain = entryB + remove = entryA + default: + if entryA.EntryId < entryB.EntryId { + maintain = entryA + remove = entryB + } else { + maintain = entryB + remove = entryA + } + } + return +} diff --git a/pkg/agent/config.go b/pkg/agent/config.go index 4ce0736e46..25da61bb21 100644 --- a/pkg/agent/config.go +++ b/pkg/agent/config.go @@ -7,6 +7,7 @@ import ( "github.com/sirupsen/logrus" "github.com/spiffe/go-spiffe/v2/spiffeid" + "github.com/spiffe/spire/pkg/agent/broker" "github.com/spiffe/spire/pkg/agent/endpoints" "github.com/spiffe/spire/pkg/agent/trustbundlesources" "github.com/spiffe/spire/pkg/agent/workloadkey" @@ -117,6 +118,12 @@ type Config struct { AuthorizedDelegates []string + // Broker holds the SPIFFE Broker API endpoint configuration. Distinct + // from AuthorizedDelegates, which gates the Delegated Identity API. + // Modeled as a struct (rather than a bare slice) so future top-level + // broker configuration can be added without breaking this field's API. + Broker BrokerConfig + // AvailabilityTarget controls how frequently rotate SVIDs AvailabilityTarget time.Duration @@ -127,6 +134,25 @@ type Config struct { WorkloadAPIRateLimit WorkloadAPIRateLimitConfig } +// BrokerConfig mirrors the agent's `experimental.broker {}` HCL block. +type BrokerConfig struct { + // BindAddresses are the addresses the broker endpoint listens on. Each + // element is a `*net.UnixAddr` (UDS, POSIX only) or a `*net.TCPAddr`. + // Multiple entries are served simultaneously by a single gRPC server. + // Empty disables the broker endpoint. + BindAddresses []net.Addr + + // Brokers enumerates the brokers authorized to talk to this agent's + // broker endpoint. + Brokers []broker.Broker + + // AllowedReferenceTypesOverTCP is the global allowlist of WorkloadReference + // type URLs permitted on the TCP listener. Empty denies every TCP + // request. Entries are verbatim type URLs (no wildcards). UDS + // connections are unaffected. + AllowedReferenceTypesOverTCP []string +} + func New(c *Config) *Agent { return &Agent{ c: c, diff --git a/pkg/agent/endpoints/peertracker_test.go b/pkg/agent/endpoints/peertracker_test.go index 3f559c2ccc..95e2d97a42 100644 --- a/pkg/agent/endpoints/peertracker_test.go +++ b/pkg/agent/endpoints/peertracker_test.go @@ -12,6 +12,8 @@ import ( "github.com/stretchr/testify/assert" "google.golang.org/grpc/codes" "google.golang.org/grpc/peer" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/anypb" ) func TestPeerTrackerAttestor(t *testing.T) { @@ -44,6 +46,10 @@ func (a FakeAttestor) Attest(_ context.Context, pid int) ([]*common.Selector, er return nil, nil } +func (a FakeAttestor) AttestReference(_ context.Context, _ *anypb.Any) ([]*common.Selector, error) { + return nil, status.Error(codes.Unimplemented, "AttestReference not implemented") +} + func WithFakeWatcher(alive bool) context.Context { return peer.NewContext(context.Background(), &peer.Peer{ AuthInfo: peertracker.AuthInfo{ diff --git a/pkg/agent/endpoints/workload/handler.go b/pkg/agent/endpoints/workload/handler.go index cf9e0be02c..d67c15e01d 100644 --- a/pkg/agent/endpoints/workload/handler.go +++ b/pkg/agent/endpoints/workload/handler.go @@ -18,6 +18,7 @@ import ( "github.com/spiffe/go-spiffe/v2/spiffeid" "github.com/spiffe/spire/pkg/agent/api/rpccontext" "github.com/spiffe/spire/pkg/agent/client" + "github.com/spiffe/spire/pkg/agent/common/hintsfilter" "github.com/spiffe/spire/pkg/agent/manager/cache" "github.com/spiffe/spire/pkg/common/bundleutil" "github.com/spiffe/spire/pkg/common/jwtsvid" @@ -119,7 +120,7 @@ func (h *Handler) FetchJWTSVID(ctx context.Context, req *workload.JWTSVIDRequest log = log.WithField(telemetry.Registered, true) entries := h.c.Manager.MatchingRegistrationEntries(selectors) - entries = filterRegistrations(entries, log) + entries = hintsfilter.FilterRegistrations(entries, log) resp = new(workload.JWTSVIDResponse) @@ -273,7 +274,7 @@ func (h *Handler) FetchX509SVID(_ *workload.X509SVIDRequest, stream workload.Spi for { select { case update := <-subscriber.Updates(): - update.Identities = filterIdentities(update.Identities, log) + update.Identities = hintsfilter.FilterIdentities(update.Identities, log) if err := h.sendX509SVIDResponse(update, stream, selectors, log, start); err != nil { return err } @@ -678,80 +679,3 @@ func isClaimAllowed(claim string, allowedClaims map[string]struct{}) bool { return ok } } - -func filterIdentities(identities []cache.Identity, log logrus.FieldLogger) []cache.Identity { - var filteredIdentities []cache.Identity - var entries []*common.RegistrationEntry - for _, identity := range identities { - entries = append(entries, identity.Entry) - } - - entriesToRemove := getEntriesToRemove(entries, log) - - for _, identity := range identities { - if _, ok := entriesToRemove[identity.Entry.EntryId]; !ok { - filteredIdentities = append(filteredIdentities, identity) - } - } - - return filteredIdentities -} - -func filterRegistrations(entries []*common.RegistrationEntry, log logrus.FieldLogger) []*common.RegistrationEntry { - var filteredEntries []*common.RegistrationEntry - entriesToRemove := getEntriesToRemove(entries, log) - - for _, entry := range entries { - if _, ok := entriesToRemove[entry.EntryId]; !ok { - filteredEntries = append(filteredEntries, entry) - } - } - - return filteredEntries -} - -func getEntriesToRemove(entries []*common.RegistrationEntry, log logrus.FieldLogger) map[string]struct{} { - entriesToRemove := make(map[string]struct{}) - hintsMap := make(map[string]*common.RegistrationEntry) - - for _, entry := range entries { - if entry.Hint == "" { - continue - } - if entryWithNonUniqueHint, ok := hintsMap[entry.Hint]; ok { - entryToMaintain, entryToRemove := hintTieBreaking(entry, entryWithNonUniqueHint) - - hintsMap[entry.Hint] = entryToMaintain - entriesToRemove[entryToRemove.EntryId] = struct{}{} - - log.WithFields(logrus.Fields{ - telemetry.Hint: entryToRemove.Hint, - telemetry.RegistrationID: entryToRemove.EntryId, - }).Warn("Ignoring entry with duplicate hint") - } else { - hintsMap[entry.Hint] = entry - } - } - - return entriesToRemove -} - -func hintTieBreaking(entryA *common.RegistrationEntry, entryB *common.RegistrationEntry) (maintain *common.RegistrationEntry, remove *common.RegistrationEntry) { - switch { - case entryA.CreatedAt < entryB.CreatedAt: - maintain = entryA - remove = entryB - case entryA.CreatedAt > entryB.CreatedAt: - maintain = entryB - remove = entryA - default: - if entryA.EntryId < entryB.EntryId { - maintain = entryA - remove = entryB - } else { - maintain = entryB - remove = entryA - } - } - return -} diff --git a/pkg/agent/manager/cache/bundle_cache.go b/pkg/agent/manager/cache/bundle_cache.go index 1f275ac630..5ffffc1d2a 100644 --- a/pkg/agent/manager/cache/bundle_cache.go +++ b/pkg/agent/manager/cache/bundle_cache.go @@ -1,15 +1,19 @@ package cache import ( + "fmt" "maps" "github.com/imkira/go-observer" "github.com/spiffe/go-spiffe/v2/bundle/spiffebundle" + "github.com/spiffe/go-spiffe/v2/bundle/x509bundle" "github.com/spiffe/go-spiffe/v2/spiffeid" ) type Bundle = spiffebundle.Bundle +var _ x509bundle.Source = (*BundleCache)(nil) + type BundleCache struct { trustDomain spiffeid.TrustDomain bundles observer.Property @@ -43,6 +47,15 @@ func (c *BundleCache) SubscribeToBundleChanges() *BundleStream { return NewBundleStream(c.bundles.Observe()) } +func (c *BundleCache) GetX509BundleForTrustDomain(trustDomain spiffeid.TrustDomain) (*x509bundle.Bundle, error) { + bundles := c.Bundles() + bundle, ok := bundles[trustDomain] + if !ok { + return nil, fmt.Errorf("bundle for trust domain %q not found", trustDomain) + } + return bundle.X509Bundle(), nil +} + // Wraps an observer stream to provide a type safe interface type BundleStream struct { stream observer.Stream diff --git a/pkg/agent/manager/cache/lru_cache.go b/pkg/agent/manager/cache/lru_cache.go index b2f879922f..d42f4f27b9 100644 --- a/pkg/agent/manager/cache/lru_cache.go +++ b/pkg/agent/manager/cache/lru_cache.go @@ -11,6 +11,7 @@ import ( "github.com/andres-erbsen/clock" "github.com/sirupsen/logrus" "github.com/spiffe/go-spiffe/v2/bundle/spiffebundle" + "github.com/spiffe/go-spiffe/v2/bundle/x509bundle" "github.com/spiffe/go-spiffe/v2/spiffeid" "github.com/spiffe/spire/pkg/common/backoff" "github.com/spiffe/spire/pkg/common/telemetry" @@ -585,6 +586,13 @@ func (c *LRUCache) SyncSVIDsWithSubscribers() { c.syncSVIDsWithSubscribers() } +func (c *LRUCache) X509Bundle() x509bundle.Source { + c.mu.RLock() + defer c.mu.RUnlock() + + return c.BundleCache +} + // scheduleRotation processes SVID entries in batches, removing those tainted by X.509 authorities. // The process continues at regular intervals until all entries have been processed or the context is cancelled. func (c *LRUCache) scheduleRotation(ctx context.Context, entryIDs []string, taintedX509Authorities []*x509.Certificate) { diff --git a/pkg/agent/manager/manager.go b/pkg/agent/manager/manager.go index 2bd1d0dea4..25f81c1088 100644 --- a/pkg/agent/manager/manager.go +++ b/pkg/agent/manager/manager.go @@ -11,6 +11,7 @@ import ( "github.com/andres-erbsen/clock" observer "github.com/imkira/go-observer" "github.com/spiffe/go-spiffe/v2/bundle/spiffebundle" + "github.com/spiffe/go-spiffe/v2/bundle/x509bundle" "github.com/spiffe/go-spiffe/v2/spiffeid" "github.com/spiffe/spire/pkg/agent/client" "github.com/spiffe/spire/pkg/agent/manager/cache" @@ -97,6 +98,9 @@ type Manager interface { // GetBundles gets the latest cached bundles for all trust domains. GetBundles() map[spiffeid.TrustDomain]*cache.Bundle + + // GetX509Bundle returns an X509 bundle source + GetX509Bundle() x509bundle.Source } // Cache stores each registration entry, signed X509-SVIDs for those entries, @@ -142,6 +146,8 @@ type Cache interface { // Identities get all identities in cache Identities() []cache.Identity + + X509Bundle() x509bundle.Source } type manager struct { @@ -453,6 +459,13 @@ func (m *manager) GetBundles() map[spiffeid.TrustDomain]*cache.Bundle { return m.cache.Bundles() } +func (m *manager) GetX509Bundle() x509bundle.Source { + m.mtx.RLock() + defer m.mtx.RUnlock() + + return m.cache.X509Bundle() +} + func (m *manager) runSVIDObserver(ctx context.Context) error { svidStream := m.SubscribeToSVIDChanges() for { diff --git a/pkg/agent/plugin/workloadattestor/docker/docker.go b/pkg/agent/plugin/workloadattestor/docker/docker.go index b8dcaddc8b..df5f861c9a 100644 --- a/pkg/agent/plugin/workloadattestor/docker/docker.go +++ b/pkg/agent/plugin/workloadattestor/docker/docker.go @@ -211,6 +211,13 @@ func (p *Plugin) Attest(ctx context.Context, req *workloadattestorv1.AttestReque }, nil } +// AttestReference returns Unimplemented. This plugin does not handle +// reference-based workload attestation; the host falls back to PID-based +// Attest when the reference is a WorkloadPIDReference. +func (p *Plugin) AttestReference(_ context.Context, _ *workloadattestorv1.AttestReferenceRequest) (*workloadattestorv1.AttestReferenceResponse, error) { + return nil, status.Error(codes.Unimplemented, "AttestReference not implemented") +} + func getSelectorValuesFromConfig(cfg *container.Config) []string { var selectorValues []string for label, value := range cfg.Labels { diff --git a/pkg/agent/plugin/workloadattestor/k8s/k8s.go b/pkg/agent/plugin/workloadattestor/k8s/k8s.go index c7b2ac653b..ce0864281d 100644 --- a/pkg/agent/plugin/workloadattestor/k8s/k8s.go +++ b/pkg/agent/plugin/workloadattestor/k8s/k8s.go @@ -21,8 +21,11 @@ import ( "github.com/hashicorp/go-hclog" "github.com/hashicorp/hcl" hcltoken "github.com/hashicorp/hcl/hcl/token" + "github.com/spiffe/go-spiffe/v2/exp/proto/spiffe/broker" + "github.com/spiffe/go-spiffe/v2/spiffeid" workloadattestorv1 "github.com/spiffe/spire-plugin-sdk/proto/spire/plugin/agent/workloadattestor/v1" configv1 "github.com/spiffe/spire-plugin-sdk/proto/spire/service/common/config/v1" + "github.com/spiffe/spire/pkg/agent/broker/brokercontext" "github.com/spiffe/spire/pkg/agent/common/sigstore" "github.com/spiffe/spire/pkg/common/catalog" "github.com/spiffe/spire/pkg/common/pemutil" @@ -32,19 +35,60 @@ import ( "golang.org/x/sync/singleflight" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/anypb" + authv1 "k8s.io/api/authorization/v1" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + crcache "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" ) +var k8sScheme = runtime.NewScheme() + +func init() { + if err := metav1.AddMetaToScheme(k8sScheme); err != nil { + panic(fmt.Sprintf("failed to register metav1 scheme: %v", err)) + } + if err := corev1.AddToScheme(k8sScheme); err != nil { + panic(fmt.Sprintf("failed to register corev1 scheme: %v", err)) + } + if err := authv1.AddToScheme(k8sScheme); err != nil { + panic(fmt.Sprintf("failed to register authv1 scheme: %v", err)) + } +} + +const ( + pluginName = "k8s" + brokerImpersonationReviewVerb = "impersonate-via-spire" + defaultMaxPollAttempts = 60 + defaultPollRetryInterval = time.Millisecond * 500 + defaultSecureKubeletPort = 10250 + defaultKubeletCAPath = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + defaultTokenPath = "/var/run/secrets/kubernetes.io/serviceaccount/token" //nolint: gosec // false positive + defaultNodeNameEnv = "MY_NODE_NAME" + defaultReloadInterval = time.Minute + + workloadPIDReferenceTypeURL = "type.googleapis.com/spiffe.broker.WorkloadPIDReference" + kubernetesObjectReferenceTypeURL = "type.googleapis.com/spiffe.broker.KubernetesObjectReference" +) + +type podReferenceScope string + const ( - pluginName = "k8s" - defaultMaxPollAttempts = 60 - defaultPollRetryInterval = time.Millisecond * 500 - defaultSecureKubeletPort = 10250 - defaultKubeletCAPath = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" - defaultTokenPath = "/var/run/secrets/kubernetes.io/serviceaccount/token" //nolint: gosec // false positive - defaultNodeNameEnv = "MY_NODE_NAME" - defaultReloadInterval = time.Minute + podReferenceScopeAgentNode podReferenceScope = "agent_node" + podReferenceScopeCluster podReferenceScope = "cluster" +) + +var ( + ErrNamespaceRequired = status.Error(codes.InvalidArgument, "namespace is required when name is set for a namespaced resource") ) func BuiltIn() catalog.BuiltIn { @@ -137,9 +181,38 @@ type HCLConfig struct { // Sigstore contains sigstore specific configs. Sigstore *sigstore.HCLConfig `hcl:"sigstore,omitempty"` + // APIServerCache contains Kubernetes API server cache-specific configs. + APIServerCache *k8sAPIServerCacheHCLConfig `hcl:"api_server_cache"` + + // Broker contains SPIFFE Broker API-specific configuration. + Broker *k8sBrokerHCLConfig `hcl:"broker"` + UnusedKeyPositions map[string][]hcltoken.Pos `hcl:",unusedKeyPositions"` } +type k8sBrokerHCLConfig struct { + Brokers []k8sBrokerHCLEntry `hcl:"brokers"` + + UnusedKeyPositions map[string][]hcltoken.Pos `hcl:",unusedKeyPositions"` +} + +type k8sBrokerHCLEntry struct { + ID string `hcl:"id"` + PodReferenceScope string `hcl:"pod_reference_scope"` + + UnusedKeyPositions map[string][]hcltoken.Pos `hcl:",unusedKeyPositions"` +} + +type k8sAPIServerCacheHCLConfig struct { + Enabled bool `hcl:"enabled"` + + UnusedKeyPositions map[string][]hcltoken.Pos `hcl:",unusedKeyPositions"` +} + +type k8sAPIServerCacheConfig struct { + Enabled bool +} + // k8sConfig holds the configuration distilled from HCL type k8sConfig struct { Secure bool @@ -157,6 +230,8 @@ type k8sConfig struct { DisableContainerSelectors bool ContainerHelper ContainerHelper sigstoreConfig *sigstore.Config + APIServerCache k8sAPIServerCacheConfig + Broker *k8sBrokerConfig Client *kubeletClient LastReload time.Time @@ -171,6 +246,7 @@ func (p *Plugin) buildConfig(coreConfig catalog.CoreConfig, hclText string, stat } pluginconf.ReportUnusedKeys(status, newConfig.UnusedKeyPositions) + brokerConfig := buildBrokerConfig(newConfig.Broker, status) // Determine max poll attempts with default maxPollAttempts := newConfig.MaxPollAttempts @@ -235,6 +311,8 @@ func (p *Plugin) buildConfig(coreConfig catalog.CoreConfig, hclText string, stat sigstoreConfig = sigstore.NewConfigFromHCL(newConfig.Sigstore, p.log) } + apiServerCacheConfig := buildAPIServerCacheConfig(newConfig.APIServerCache, status) + // return the kubelet client return &k8sConfig{ Secure: secure, @@ -252,6 +330,85 @@ func (p *Plugin) buildConfig(coreConfig catalog.CoreConfig, hclText string, stat DisableContainerSelectors: newConfig.DisableContainerSelectors, ContainerHelper: containerHelper, sigstoreConfig: sigstoreConfig, + APIServerCache: apiServerCacheConfig, + Broker: brokerConfig, + } +} + +func buildAPIServerCacheConfig(hclConfig *k8sAPIServerCacheHCLConfig, status *pluginconf.Status) k8sAPIServerCacheConfig { + if hclConfig == nil { + return k8sAPIServerCacheConfig{} + } + pluginconf.ReportUnusedKeys(status, hclConfig.UnusedKeyPositions) + return k8sAPIServerCacheConfig{ + Enabled: hclConfig.Enabled, + } +} + +type k8sBrokerConfig struct { + Brokers map[string]k8sBrokerEntry +} + +type k8sBrokerEntry struct { + ID spiffeid.ID + PodReferenceScope podReferenceScope +} + +func buildBrokerConfig(brokerConfig *k8sBrokerHCLConfig, status *pluginconf.Status) *k8sBrokerConfig { + if brokerConfig == nil { + return nil + } + + pluginconf.ReportUnusedKeys(status, brokerConfig.UnusedKeyPositions) + if len(brokerConfig.Brokers) == 0 { + status.ReportError("broker.brokers: at least one broker is required") + return &k8sBrokerConfig{Brokers: map[string]k8sBrokerEntry{}} + } + + brokers := make(map[string]k8sBrokerEntry, len(brokerConfig.Brokers)) + seen := make(map[string]struct{}, len(brokerConfig.Brokers)) + for i, b := range brokerConfig.Brokers { + pluginconf.ReportUnusedKeys(status, b.UnusedKeyPositions) + if b.ID == "" { + status.ReportErrorf("broker.brokers[%d].id: must be specified", i) + continue + } + if _, dup := seen[b.ID]; dup { + status.ReportErrorf("broker.brokers[%s].id: duplicate broker id", b.ID) + continue + } + seen[b.ID] = struct{}{} + + id, err := spiffeid.FromString(b.ID) + if err != nil { + status.ReportErrorf("broker.brokers[%s].id: %v", b.ID, err) + continue + } + + podRefScope, ok := buildPodReferenceScope(b.ID, b.PodReferenceScope, status) + if !ok { + continue + } + + brokers[b.ID] = k8sBrokerEntry{ + ID: id, + PodReferenceScope: podRefScope, + } + } + return &k8sBrokerConfig{Brokers: brokers} +} + +func buildPodReferenceScope(brokerID string, hclValue string, status *pluginconf.Status) (podReferenceScope, bool) { + switch hclValue { + case "": + return podReferenceScopeAgentNode, true + case string(podReferenceScopeAgentNode): + return podReferenceScopeAgentNode, true + case string(podReferenceScopeCluster): + return podReferenceScopeCluster, true + default: + status.ReportErrorf("broker.brokers[%s].pod_reference_scope: unsupported value %q; must be one of [agent_node, cluster]", brokerID, hclValue) + return "", false } } @@ -274,6 +431,16 @@ type Plugin struct { containerHelper ContainerHelper sigstoreVerifier sigstore.Verifier + // kubeClient is write-once-read-many. It carries its own RESTMapper + // internally (accessible via kubeClient.RESTMapper()), so we don't keep + // a separate field for it. Guarded by a dedicated mutex so the apiserver + // discovery handshake on first use does not block readers of unrelated + // plugin state. + kubeMu sync.RWMutex + kubeClient client.Client + kubeCacheCancel context.CancelFunc + kubeCacheDone chan struct{} + cachedPodList map[string]*fastjson.Value cachedPodListValidUntil time.Time singleflight singleflight.Group @@ -290,21 +457,188 @@ func (p *Plugin) SetLogger(log hclog.Logger) { p.log = log } +// Attest implements the legacy PID-only RPC for callers that haven't moved +// to AttestReference. PID handling is delegated through AttestReference so the +// two RPCs share behavior, including broker checks when broker metadata is +// present on the context. func (p *Plugin) Attest(ctx context.Context, req *workloadattestorv1.AttestRequest) (*workloadattestorv1.AttestResponse, error) { - config, containerHelper, sigstoreVerifier, err := p.getConfig() + ref, err := anypb.New(&broker.WorkloadPIDReference{Pid: req.Pid}) if err != nil { return nil, err } + resp, err := p.AttestReference(ctx, &workloadattestorv1.AttestReferenceRequest{Reference: ref}) + if err != nil { + return nil, err + } + return &workloadattestorv1.AttestResponse{SelectorValues: resp.GetSelectorValues()}, nil +} - podUID, containerID, err := containerHelper.GetPodUIDAndContainerID(req.Pid, p.log) +func (p *Plugin) AttestReference(ctx context.Context, req *workloadattestorv1.AttestReferenceRequest) (*workloadattestorv1.AttestReferenceResponse, error) { + config, _, _, err := p.getConfig() if err != nil { return nil, err } + + brokerEntry, result, err := p.attestReference(ctx, config, req) + if err != nil { + return nil, err + } + if err := p.checkBrokerImpersonationForReference(ctx, brokerEntry, result); err != nil { + return nil, err + } + return result.Response, nil +} + +func (p *Plugin) attestReference(ctx context.Context, config *k8sConfig, req *workloadattestorv1.AttestReferenceRequest) (*k8sBrokerEntry, *attestReferenceResult, error) { + reference := req.GetReference() + if reference == nil { + return nil, nil, status.Error(codes.InvalidArgument, "workload reference must be provided") + } + + brokerEntry, err := p.getBrokerEntryIfPresent(ctx, config) + if err != nil { + return nil, nil, err + } + + switch reference.TypeUrl { + case workloadPIDReferenceTypeURL: + var pidRef broker.WorkloadPIDReference + if err := reference.UnmarshalTo(&pidRef); err != nil { + return nil, nil, status.Errorf(codes.InvalidArgument, "unable to unmarshal PID reference: %v", err) + } + result, err := p.attestByPIDReference(ctx, pidRef.Pid) + return brokerEntry, result, err + case kubernetesObjectReferenceTypeURL: + var objRef broker.KubernetesObjectReference + if err := reference.UnmarshalTo(&objRef); err != nil { + return nil, nil, status.Errorf(codes.InvalidArgument, "unable to unmarshal object reference: %v", err) + } + if err := validateKubernetesObjectReference(&objRef); err != nil { + return nil, nil, err + } + result, err := p.attestByKubernetesObjectReference(ctx, brokerEntry, &objRef) + return brokerEntry, result, err + default: + return nil, nil, status.Errorf(codes.InvalidArgument, "unsupported reference type: %s", reference.TypeUrl) + } +} + +func validateKubernetesObjectReference(objRef *broker.KubernetesObjectReference) error { + objType := objRef.GetType() + if objType == nil { + return status.Error(codes.InvalidArgument, "object reference is missing type") + } + if objType.Plural == "" { + return status.Error(codes.InvalidArgument, "object reference type is missing plural") + } + if objType.Group == "" { + return status.Error(codes.InvalidArgument, "object reference type is missing group") + } + objKey := objRef.GetKey() + if objKey == nil && objRef.GetUid() == "" { + return status.Error(codes.InvalidArgument, "object reference is missing key and UID") + } + if objKey != nil { + name := objKey.GetName() + if name == "" { + return status.Error(codes.InvalidArgument, "object reference key is missing name") + } + } + return nil +} + +type attestReferenceResult struct { + Response *workloadattestorv1.AttestReferenceResponse + ObjectReference *broker.KubernetesObjectReference + Namespace string + Name string +} + +func (p *Plugin) attestByPIDReference(ctx context.Context, pid int32) (*attestReferenceResult, error) { + resp, pod, err := p.attestByPID(ctx, pid) + if err != nil { + return nil, err + } + + result := &attestReferenceResult{Response: resp} + if pod != nil { + result.ObjectReference = &broker.KubernetesObjectReference{ + Type: &broker.KubernetesObjectType{Plural: "pods", Group: "core"}, + } + result.Namespace = pod.Namespace + result.Name = pod.Name + } + return result, nil +} + +func (p *Plugin) attestByKubernetesObjectReference(ctx context.Context, brokerEntry *k8sBrokerEntry, objRef *broker.KubernetesObjectReference) (*attestReferenceResult, error) { + objType := objRef.GetType() + switch { + case objType.Plural == "pods" && objType.Group == "core": + return p.attestByPodReference(ctx, brokerEntry, objRef) + default: + return p.attestByObjectReference(ctx, objRef) + } +} + +func (p *Plugin) checkBrokerImpersonationForReference(ctx context.Context, brokerEntry *k8sBrokerEntry, result *attestReferenceResult) error { + if brokerEntry == nil { + return nil + } + if result.ObjectReference == nil { + return nil + } + kubeClient, err := p.getOrCreateKubeClient(ctx) + if err != nil { + return status.Errorf(codes.Internal, "unable to set up Kubernetes client: %v", err) + } + return p.checkBrokerImpersonation(ctx, kubeClient, brokerEntry, result.ObjectReference, result.Namespace, result.Name) +} + +func (p *Plugin) getBrokerEntryIfPresent(ctx context.Context, config *k8sConfig) (*k8sBrokerEntry, error) { + _, ok, err := brokercontext.CallerIDFromContext(ctx) + if err != nil { + return nil, status.Errorf(codes.Internal, "unable to determine broker caller identity: %v", err) + } + if !ok { + return nil, nil + } + return p.getBrokerEntry(ctx, config) +} + +func (p *Plugin) getBrokerEntry(ctx context.Context, config *k8sConfig) (*k8sBrokerEntry, error) { + if config.Broker == nil { + return nil, status.Error(codes.Internal, "broker configuration missing") + } + callerID, ok, err := brokercontext.CallerIDFromContext(ctx) + if err != nil { + return nil, status.Errorf(codes.Internal, "unable to determine broker caller identity: %v", err) + } + if !ok { + return nil, status.Error(codes.Internal, "broker caller identity missing") + } + brokerEntry, ok := config.Broker.Brokers[callerID.String()] + if !ok { + return nil, status.Errorf(codes.PermissionDenied, "broker %q is not configured", callerID.String()) + } + return &brokerEntry, nil +} + +func (p *Plugin) attestByPID(ctx context.Context, pid int32) (*workloadattestorv1.AttestReferenceResponse, *corev1.Pod, error) { + config, containerHelper, sigstoreVerifier, err := p.getConfig() + if err != nil { + return nil, nil, err + } + + podUID, containerID, err := containerHelper.GetPodUIDAndContainerID(pid, p.log) + if err != nil { + return nil, nil, err + } podKnown := podUID != "" // Not a Kubernetes pod if containerID == "" { - return &workloadattestorv1.AttestResponse{}, nil + return &workloadattestorv1.AttestReferenceResponse{}, nil, nil } log := p.log.With( @@ -320,10 +654,11 @@ func (p *Plugin) Attest(ctx context.Context, req *workloadattestorv1.AttestReque podList, err := p.getPodList(ctx, config.Client, config.PollRetryInterval/2) if err != nil { - return nil, err + return nil, nil, err } - var attestResponse *workloadattestorv1.AttestResponse + var attestResponse *workloadattestorv1.AttestReferenceResponse + var attestedPod *corev1.Pod for podKey, podValue := range podList { if podKnown { if podKey != string(podUID) { @@ -338,7 +673,7 @@ func (p *Plugin) Attest(ctx context.Context, req *workloadattestorv1.AttestReque pod := new(corev1.Pod) if err := json.Unmarshal(scratch, &pod); err != nil { - return nil, status.Errorf(codes.Internal, "unable to decode pod info from kubelet response: %v", err) + return nil, nil, status.Errorf(codes.Internal, "unable to decode pod info from kubelet response: %v", err) } var selectorValues []string @@ -358,7 +693,7 @@ func (p *Plugin) Attest(ctx context.Context, req *workloadattestorv1.AttestReque log.Debug("Attempting to verify sigstore image signature", "image", containerStatus.Image) sigstoreSelectors, err := p.sigstoreVerifier.Verify(ctx, containerStatus.ImageID) if err != nil { - return nil, status.Errorf(codes.Internal, "error verifying sigstore image signature for imageID %s: %v", containerStatus.ImageID, err) + return nil, nil, status.Errorf(codes.Internal, "error verifying sigstore image signature for imageID %s: %v", containerStatus.ImageID, err) } selectorValues = append(selectorValues, sigstoreSelectors...) } @@ -373,20 +708,21 @@ func (p *Plugin) Attest(ctx context.Context, req *workloadattestorv1.AttestReque if len(selectorValues) > 0 { if attestResponse != nil { log.Warn("Two pods found with same container Id") - return nil, status.Error(codes.Internal, "two pods found with same container Id") + return nil, nil, status.Error(codes.Internal, "two pods found with same container Id") } - attestResponse = &workloadattestorv1.AttestResponse{SelectorValues: selectorValues} + attestResponse = &workloadattestorv1.AttestReferenceResponse{SelectorValues: selectorValues} + attestedPod = pod } } if attestResponse != nil { - return attestResponse, nil + return attestResponse, attestedPod, nil } // if the container was not located after the maximum number of attempts then the search is over. if attempt >= config.MaxPollAttempts { log.Warn("Container id not found; giving up") - return nil, status.Error(codes.DeadlineExceeded, "no selectors found after max poll attempts") + return nil, nil, status.Error(codes.DeadlineExceeded, "no selectors found after max poll attempts") } // wait a bit for containers to initialize before trying again. @@ -395,9 +731,499 @@ func (p *Plugin) Attest(ctx context.Context, req *workloadattestorv1.AttestReque select { case <-p.clock.After(config.PollRetryInterval): case <-ctx.Done(): - return nil, status.Errorf(codes.Canceled, "no selectors found: %v", ctx.Err()) + return nil, nil, status.Errorf(codes.Canceled, "no selectors found: %v", ctx.Err()) + } + } +} + +// attestByPodReference handles the `pods/core` path: a Kubernetes object +// reference whose resource is the core Pod type. Reference shape is already +// validated by the AttestReference dispatcher (at least one of key/uid; name +// required when key is set); this function adds the pod-specific namespace +// requirement (pods are always namespaced) and enforces the spec cross-check +// ("if both key and uid are supplied, the resolved pod's UID MUST match the +// supplied uid"). Resolution tries the kubelet pod list first (cheap, +// node-local, indexed by UID — same path the legacy PID flow uses), then +// falls back to the API server when needed. Under agent_node scope, API +// server results must still match the agent node name. Selector emission uses pod-shaped selectors +// (sa, ns, pod-uid, pod-name, pod-image, pod-label, pod-owner, ...), distinct +// from the generic-object vocabulary so registration entries +// can match pod-specific fields like container images and service accounts +// that aren't present on a PartialObjectMetadata. +func (p *Plugin) attestByPodReference(ctx context.Context, brokerEntry *k8sBrokerEntry, objRef *broker.KubernetesObjectReference) (*attestReferenceResult, error) { + key := objRef.GetKey() + namespace := key.GetNamespace() + name := key.GetName() + uid := types.UID(objRef.GetUid()) + + config, _, _, err := p.getConfig() + if err != nil { + return nil, err + } + + var pod *corev1.Pod + switch { + case name != "": + if namespace == "" { + return nil, ErrNamespaceRequired } + pod, err = p.findPodByName(ctx, config, namespace, name, brokerPodReferenceScope(brokerEntry)) + default: + pod, err = p.findPodByUID(ctx, config, uid, brokerPodReferenceScope(brokerEntry)) + } + if err != nil { + return nil, err + } + + // Per spec: when name (and namespace) and uid are both supplied, the + // resolved object's UID MUST match the supplied uid. + if uid != "" && pod.UID != uid { + return nil, status.Errorf(codes.NotFound, "pod %s/%s has UID %s, expected %s", pod.Namespace, pod.Name, pod.UID, uid) + } + + return &attestReferenceResult{ + Response: &workloadattestorv1.AttestReferenceResponse{SelectorValues: getSelectorValuesFromPodInfo(pod)}, + ObjectReference: objRef, + Namespace: pod.Namespace, + Name: pod.Name, + }, nil +} + +func brokerPodReferenceScope(brokerEntry *k8sBrokerEntry) podReferenceScope { + if brokerEntry == nil { + return podReferenceScopeAgentNode + } + return brokerEntry.PodReferenceScope +} + +// findPodByName resolves a single pod by its namespaced name. The kubelet +// pod list is iterated first; this is O(n) over the node's pods (the list +// is indexed by UID, not name) but n is small in practice and saves an API +// server round-trip when the pod is local. If the pod is not in the kubelet +// list, the apiserver answers a precise Get directly — no list, no +// client-side filter. Under agent_node scope, the resolved pod must still be +// scheduled to the configured agent node. +func (p *Plugin) findPodByName(ctx context.Context, config *k8sConfig, namespace, name string, scope podReferenceScope) (*corev1.Pod, error) { + // Try kubelet pod list first; iterate to find a match by namespace+name. + podList, err := p.getPodList(ctx, config.Client, config.PollRetryInterval/2) + if err != nil { + return nil, err + } + for _, podValue := range podList { + if string(podValue.GetStringBytes("metadata", "namespace")) != namespace { + continue + } + if string(podValue.GetStringBytes("metadata", "name")) != name { + continue + } + return decodePodFromKubelet(podValue) + } + + // Fallback: direct Get from API server. + kubeClient, err := p.getOrCreateKubeClient(ctx) + if err != nil { + return nil, status.Errorf(codes.Internal, "unable to create Kubernetes client: %v", err) + } + pod := &corev1.Pod{} + if err := kubeClient.Get(ctx, client.ObjectKey{Namespace: namespace, Name: name}, pod); err != nil { + if apierrors.IsNotFound(err) { + return nil, status.Errorf(codes.NotFound, "pod %s/%s not found", namespace, name) + } + return nil, status.Errorf(codes.Internal, "unable to get pod from Kubernetes API: %v", err) + } + if err := checkPodReferenceScope(scope, config.NodeName, pod); err != nil { + return nil, err + } + return pod, nil +} + +// findPodByUID resolves a single pod by its Kubernetes UID. The kubelet pod +// list is checked first because it's already keyed by UID and only contains +// pods scheduled to this node — both common-case wins. If the pod is not in +// the kubelet list, it falls back to a cluster-wide List from the API server. +// Kubernetes does not support `metadata.uid` as a field selector, so we list +// and filter client-side regardless. Under agent_node scope, the resolved pod +// must still be scheduled to the configured agent node. +func (p *Plugin) findPodByUID(ctx context.Context, config *k8sConfig, uid types.UID, scope podReferenceScope) (*corev1.Pod, error) { + // Try kubelet pod list first (already indexed by UID). + podList, err := p.getPodList(ctx, config.Client, config.PollRetryInterval/2) + if err != nil { + return nil, err + } + if podValue, ok := podList[string(uid)]; ok { + return decodePodFromKubelet(podValue) + } + + // Fallback: list all pods via API server. k8s doesn't support + // metadata.uid as a field selector, so we list and filter client-side. + kubeClient, err := p.getOrCreateKubeClient(ctx) + if err != nil { + return nil, status.Errorf(codes.Internal, "unable to create Kubernetes client: %v", err) + } + pods := &corev1.PodList{} + if err := kubeClient.List(ctx, pods); err != nil { + return nil, status.Errorf(codes.Internal, "unable to list pods from Kubernetes API: %v", err) + } + for i := range pods.Items { + if pods.Items[i].UID != uid { + continue + } + if err := checkPodReferenceScope(scope, config.NodeName, &pods.Items[i]); err != nil { + return nil, err + } + return &pods.Items[i], nil + } + return nil, status.Errorf(codes.NotFound, "pod with UID %s not found", uid) +} + +func checkPodReferenceScope(scope podReferenceScope, agentNodeName string, pod *corev1.Pod) error { + if scope != podReferenceScopeAgentNode { + return nil + } + if agentNodeName == "" { + return status.Error(codes.Internal, "agent node name is not configured") + } + if pod.Spec.NodeName != agentNodeName { + return status.Error(codes.PermissionDenied, "pod is not on the agent node") + } + return nil +} + +// decodePodFromKubelet rehydrates a `corev1.Pod` from the partially-parsed +// JSON the kubelet pod-list path keeps in fastjson form (the cache stores +// pods as `*fastjson.Value` to avoid per-request unmarshalling when no pod +// is needed). The two-stage marshal-then-unmarshal is unavoidable here: +// fastjson is read-only, so the only way to project into a typed struct is +// to serialise back to bytes and let `encoding/json` parse it. +func decodePodFromKubelet(podValue *fastjson.Value) (*corev1.Pod, error) { + var scratch []byte + scratch = podValue.MarshalTo(scratch) + pod := new(corev1.Pod) + if err := json.Unmarshal(scratch, pod); err != nil { + return nil, status.Errorf(codes.Internal, "unable to decode pod info from kubelet response: %v", err) + } + return pod, nil +} + +// attestByObjectReference handles the generic-Kubernetes-object path: any +// resource other than `pods/core`. It resolves the resource's GVK and scope +// via the discovery-backed REST mapper, fetches the object's metadata via +// PartialObjectMetadata, and emits a uniform set of selectors derived from +// `ObjectMeta` (resource, namespace, name, uid, labels, owner references). +func (p *Plugin) attestByObjectReference(ctx context.Context, objRef *broker.KubernetesObjectReference) (*attestReferenceResult, error) { + r := objRef.GetType() + key := objRef.GetKey() + namespace := key.GetNamespace() + name := key.GetName() + uid := types.UID(objRef.GetUid()) + + kubeClient, err := p.getOrCreateKubeClient(ctx) + if err != nil { + return nil, status.Errorf(codes.Internal, "unable to set up Kubernetes client: %v", err) + } + mapper := kubeClient.RESTMapper() + + // Per the SPIFFE Broker API spec, `core` is the canonical group string + // for the Kubernetes core API group, but Kubernetes itself uses the + // empty string on the wire — translate before mapping. + group := kubernetesAPIGroup(r.GetGroup()) + gvr := schema.GroupVersionResource{Group: group, Resource: r.GetPlural()} + gvk, err := mapper.KindFor(gvr) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "unknown Kubernetes resource %s.%s: %v", r.GetPlural(), r.GetGroup(), err) + } + mapping, err := mapper.RESTMapping(gvk.GroupKind(), gvk.Version) + if err != nil { + return nil, status.Errorf(codes.Internal, "no REST mapping for %s.%s: %v", r.GetPlural(), r.GetGroup(), err) + } + + namespaced := mapping.Scope.Name() == meta.RESTScopeNameNamespace + switch { + case namespaced && name != "" && namespace == "": + return nil, ErrNamespaceRequired + case !namespaced && namespace != "": + return nil, status.Error(codes.InvalidArgument, "namespace must be empty for cluster-scoped resource") + } + + obj, err := p.findObject(ctx, kubeClient, gvk, namespace, name, uid) + if err != nil { + return nil, err } + + if uid != "" && obj.UID != uid { + return nil, status.Errorf(codes.NotFound, "%s.%s %s/%s has UID %s, expected %s", + r.GetPlural(), r.GetGroup(), obj.Namespace, obj.Name, obj.UID, uid) + } + return &attestReferenceResult{ + Response: &workloadattestorv1.AttestReferenceResponse{ + SelectorValues: getSelectorValuesFromObjectMeta(r, gvk, obj), + }, + ObjectReference: objRef, + Namespace: obj.Namespace, + Name: obj.Name, + }, nil +} + +// checkBrokerImpersonation asks the Kubernetes authorizer whether the broker +// SPIFFE ID may use SPIRE's custom verb on the referenced object. +func (p *Plugin) checkBrokerImpersonation(ctx context.Context, kubeClient client.Client, brokerEntry *k8sBrokerEntry, objRef *broker.KubernetesObjectReference, namespace, name string) error { + review := &authv1.SubjectAccessReview{ + Spec: authv1.SubjectAccessReviewSpec{ + User: brokerEntry.ID.String(), + ResourceAttributes: &authv1.ResourceAttributes{ + Group: kubernetesAPIGroup(objRef.GetType().GetGroup()), + Resource: objRef.GetType().GetPlural(), + Namespace: namespace, + Name: name, + Verb: brokerImpersonationReviewVerb, + }, + }, + } + if err := kubeClient.Create(ctx, review); err != nil { + return status.Errorf(codes.Internal, "unable to check Kubernetes authorization for broker: %v", err) + } + if !review.Status.Allowed { + return status.Error(codes.PermissionDenied, "Kubernetes authorizer does not allow the broker to use impersonate-via-spire for the referenced object") + } + return nil +} + +func kubernetesAPIGroup(group string) string { + if group == "core" { + return "" + } + return group +} + +// findObject resolves a single Kubernetes object's metadata. When `name` is +// supplied, a direct Get is used (precise; returns NotFound cleanly). When +// only `uid` is supplied, the API server is listed and filtered client-side +// (the apiserver does not support metadata.uid as a field selector). +func (p *Plugin) findObject(ctx context.Context, kubeClient client.Client, gvk schema.GroupVersionKind, namespace, name string, uid types.UID) (*metav1.PartialObjectMetadata, error) { + if name != "" { + obj := &metav1.PartialObjectMetadata{} + obj.SetGroupVersionKind(gvk) + if err := kubeClient.Get(ctx, client.ObjectKey{Namespace: namespace, Name: name}, obj); err != nil { + if apierrors.IsNotFound(err) { + return nil, status.Errorf(codes.NotFound, "%s %s/%s not found", gvk.Kind, namespace, name) + } + return nil, status.Errorf(codes.Internal, "unable to get %s: %v", gvk.Kind, err) + } + return obj, nil + } + + list := &metav1.PartialObjectMetadataList{} + list.SetGroupVersionKind(schema.GroupVersionKind{Group: gvk.Group, Version: gvk.Version, Kind: gvk.Kind + "List"}) + if err := kubeClient.List(ctx, list); err != nil { + return nil, status.Errorf(codes.Internal, "unable to list %s: %v", gvk.Kind, err) + } + for i := range list.Items { + if list.Items[i].UID == uid { + return &list.Items[i], nil + } + } + return nil, status.Errorf(codes.NotFound, "%s with UID %s not found", gvk.Kind, uid) +} + +// getSelectorValuesFromObjectMeta produces the broker-object selector set +// from a resolved Kubernetes object's metadata. The vocabulary is uniform +// across resource types so registration entries can be authored without +// caring whether the workload is a Pod, a Deployment, or a CRD instance. +// +// Selector values (the leading `k8s:` selector type is added by the V1 +// wrapper, not here): +// +// uid: always +// resource:. always +// plural: always +// group: always; "core" for core resources +// version: always; e.g. "v1", "v1beta1" +// apiVersion: Kubernetes wire form: "v1" (core) or "/" +// kind: always; e.g. "Pod", "Deployment" +// name: always +// namespace: omitted for cluster-scoped objects +// key:/ `/` for namespaced objects; `` for cluster-scoped +// label:: one per .metadata.labels entry +// owner-key:// atomic identity of every owner reference by key +// owner-uid:// atomic identity of every owner reference by UID +// controller-key:// atomic identity of every controller owner reference by key +// controller-uid:// atomic identity of every controller owner reference by UID +func getSelectorValuesFromObjectMeta(r *broker.KubernetesObjectType, gvk schema.GroupVersionKind, obj *metav1.PartialObjectMetadata) []string { + objType := r.GetPlural() + "." + r.GetGroup() + values := []string{ + "uid:" + string(obj.UID), + "resource:" + objType, + "plural:" + r.GetPlural(), + "group:" + r.GetGroup(), + "version:" + gvk.Version, + "apiVersion:" + gvk.GroupVersion().String(), + "kind:" + gvk.Kind, + "name:" + obj.Name, + } + + // Namespace and key. + objKey := obj.Name + if obj.Namespace != "" { + objKey = obj.Namespace + "/" + obj.Name + values = append(values, "namespace:"+obj.Namespace) + } + values = append(values, "key:"+objKey) + + // Labels. + for k, v := range obj.Labels { + values = append(values, "label:"+k+":"+v) + } + + // Owners and controllers. + for _, owner := range obj.OwnerReferences { + ownerGV, _ := schema.ParseGroupVersion(owner.APIVersion) + if ownerGV.Group == "" { + ownerGV.Group = "core" + } + + // We omit ownerGV.Version here on purpose, the version is + // not part of the object identity. + ownerGK := ownerGV.Group + "/" + owner.Kind + ownerKey := ownerGK + "/" + owner.Name + ownerUID := ownerGK + "/" + string(owner.UID) + values = append(values, + "owner-key:"+ownerKey, + "owner-uid:"+ownerUID, + ) + + // Also a controller? + if owner.Controller != nil && *owner.Controller { + values = append(values, + "controller-key:"+ownerKey, + "controller-uid:"+ownerUID, + ) + } + } + return values +} + +// getOrCreateKubeClient lazily builds the controller-runtime client. The +// client is paired with a discovery-backed REST mapper (accessible via +// `client.RESTMapper()`) used by the arbitrary-object path to resolve +// plural+group → GVK and determine namespace scoping at runtime. The +// client, mapper, and cache share an HTTP client to avoid duplicate +// connection pools and duplicate discovery caches. Guarded by a dedicated +// mutex so the apiserver +// discovery handshake on first use does not block readers of unrelated +// plugin state. +func (p *Plugin) getOrCreateKubeClient(ctx context.Context) (client.Client, error) { + p.kubeMu.RLock() + if p.kubeClient != nil { + c := p.kubeClient + p.kubeMu.RUnlock() + return c, nil + } + p.kubeMu.RUnlock() + + p.kubeMu.Lock() + defer p.kubeMu.Unlock() + if p.kubeClient != nil { + return p.kubeClient, nil + } + + config, _, _, err := p.getConfig() + if err != nil { + return nil, err + } + + restConfig, err := ctrl.GetConfig() + if err != nil { + return nil, fmt.Errorf("unable to load Kubernetes client config: %w", err) + } + httpClient, err := rest.HTTPClientFor(restConfig) + if err != nil { + return nil, fmt.Errorf("unable to build Kubernetes HTTP client: %w", err) + } + mapper, err := apiutil.NewDynamicRESTMapper(restConfig, httpClient) + if err != nil { + return nil, fmt.Errorf("unable to build Kubernetes REST mapper: %w", err) + } + + clientOptions := client.Options{ + Scheme: k8sScheme, + Mapper: mapper, + HTTPClient: httpClient, + } + if !config.APIServerCache.Enabled { + c, err := client.New(restConfig, clientOptions) + if err != nil { + return nil, fmt.Errorf("unable to create Kubernetes client: %w", err) + } + p.kubeClient = c + return c, nil + } + + // This cache will hold only corev1.Pod and metav1.PartialObjectMetadata, + // which is quite fine in terms of memory consumption. We use PartialObjectMetadata + // specifically to avoid caching large objects, and some clusters have thousands of + // them (e.g. large Helm release Secrets). + // Pod is a special exception as we emit specific selectors for pods that + // require non-metadata fields, e.g. spec.serviceAccountName. + kubeCache, err := crcache.New(restConfig, crcache.Options{ + Scheme: k8sScheme, + Mapper: mapper, + HTTPClient: httpClient, + DefaultTransform: crcache.TransformStripManagedFields(), + }) + if err != nil { + return nil, fmt.Errorf("unable to create Kubernetes cache: %w", err) + } + cacheCtx, cacheCancel := context.WithCancel(context.Background()) + cacheDone := make(chan struct{}) + go func() { + defer close(cacheDone) + if err := kubeCache.Start(cacheCtx); err != nil && cacheCtx.Err() == nil { + if p.log != nil { + p.log.Warn("Kubernetes cache stopped unexpectedly", telemetry.Error, err) + } + } + }() + if !kubeCache.WaitForCacheSync(ctx) { + cacheCancel() + <-cacheDone + return nil, errors.New("timed out waiting for Kubernetes cache to start") + } + + clientOptions.Cache = &client.CacheOptions{ + Reader: kubeCache, + Unstructured: false, // Avoid caching unwanted large objects. + DisableFor: nil, // No point in disabling for specific types. + // We want caching disabled for everything except corev1.Pod and + // metav1.PartialObjectMetadata, but the client.CacheOptions API + // doesn't support an allow-list, only a deny-list. + } + c, err := client.New(restConfig, clientOptions) + if err != nil { + cacheCancel() + <-cacheDone + return nil, fmt.Errorf("unable to create Kubernetes client: %w", err) + } + + p.kubeClient = c + p.kubeCacheCancel = cacheCancel + p.kubeCacheDone = cacheDone + return c, nil +} + +func (p *Plugin) Close() error { + p.kubeMu.Lock() + cacheCancel := p.kubeCacheCancel + cacheDone := p.kubeCacheDone + p.kubeClient = nil + p.kubeCacheCancel = nil + p.kubeCacheDone = nil + p.kubeMu.Unlock() + + if cacheCancel != nil { + cacheCancel() + <-cacheDone + } + return nil } func (p *Plugin) Configure(ctx context.Context, req *configv1.ConfigureRequest) (resp *configv1.ConfigureResponse, err error) { diff --git a/pkg/agent/plugin/workloadattestor/k8s/k8s_test.go b/pkg/agent/plugin/workloadattestor/k8s/k8s_test.go index 2517aa0192..62eedb301b 100644 --- a/pkg/agent/plugin/workloadattestor/k8s/k8s_test.go +++ b/pkg/agent/plugin/workloadattestor/k8s/k8s_test.go @@ -19,7 +19,10 @@ import ( "testing" "time" + "github.com/spiffe/go-spiffe/v2/exp/proto/spiffe/broker" "github.com/spiffe/go-spiffe/v2/spiffeid" + workloadattestorv1 "github.com/spiffe/spire-plugin-sdk/proto/spire/plugin/agent/workloadattestor/v1" + "github.com/spiffe/spire/pkg/agent/broker/brokercontext" "github.com/spiffe/spire/pkg/agent/common/sigstore" "github.com/spiffe/spire/pkg/agent/plugin/workloadattestor" "github.com/spiffe/spire/pkg/common/catalog" @@ -32,11 +35,23 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "google.golang.org/grpc/codes" + "google.golang.org/protobuf/types/known/anypb" + authv1 "k8s.io/api/authorization/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" ) const ( pid = 123 + testBrokerID = "spiffe://example.org/broker" + testPollRetryInterval = time.Second podListFilePath = "testdata/pod_list.json" @@ -47,6 +62,8 @@ const ( ) var ( + testBrokerSPIFFEID = spiffeid.RequireFromString(testBrokerID) + clientKey, _ = pemutil.ParseECPrivateKey([]byte(`-----BEGIN PRIVATE KEY----- MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgNRa/6HIy0uwQe8iG Kz24zEvwGiIsTDPHzrLUaml1hQ6hRANCAATz6vtJYIvPM0KOqKpdDPlsOw09hZ8P @@ -142,6 +159,45 @@ func (s *Suite) TestAttestWithPidInPod() { s.requireAttestSuccessWithPod(p) } +func (s *Suite) TestAttestWithPidDoesNotRunBrokerRBAC() { + s.startInsecureKubelet() + var reviews []authv1.SubjectAccessReview + p := s.loadInsecurePluginWithBrokerAndKubeClient(fakeKubeClientWithSubjectAccessReview(false, &reviews)) + + s.addPodListResponse(podListFilePath) + s.addGetContainerResponsePidInPod() + + selectors, err := p.Attest(context.Background(), pid) + s.Require().NoError(err) + s.requireSelectorsEqual(testPodAndContainerSelectors, selectors) + s.Require().Empty(reviews) +} + +func (s *Suite) TestAttestWithBrokerCallerRunsBrokerRBAC() { + s.startInsecureKubelet() + var reviews []authv1.SubjectAccessReview + p := s.loadRawInsecurePluginWithBrokerAndKubeClient(fakeKubeClientWithSubjectAccessReview(true, &reviews)) + + s.addPodListResponse(podListFilePath) + s.addGetContainerResponsePidInPod() + + resp, err := p.Attest(testBrokerContext(), &workloadattestorv1.AttestRequest{Pid: int32(pid)}) + s.Require().NoError(err) + s.requireSelectorValuesEqual(testPodAndContainerSelectors, resp.GetSelectorValues()) + s.Require().Len(reviews, 1) + + review := reviews[0] + assert.Equal(s.T(), testBrokerID, review.Spec.User) + assert.Empty(s.T(), review.Spec.Groups) + if assert.NotNil(s.T(), review.Spec.ResourceAttributes) { + assert.Equal(s.T(), "", review.Spec.ResourceAttributes.Group) + assert.Equal(s.T(), "pods", review.Spec.ResourceAttributes.Resource) + assert.Equal(s.T(), "default", review.Spec.ResourceAttributes.Namespace) + assert.Equal(s.T(), "blog-24ck7", review.Spec.ResourceAttributes.Name) + assert.Equal(s.T(), brokerImpersonationReviewVerb, review.Spec.ResourceAttributes.Verb) + } +} + func (s *Suite) TestAttestWithPidInPodAfterRetry() { s.startInsecureKubelet() p := s.loadInsecurePlugin() @@ -343,6 +399,7 @@ func (s *Suite) TestConfigure() { PollRetryInterval time.Duration ReloadInterval time.Duration SigstoreConfig *sigstore.Config + APIServerCache bool } testCases := []struct { @@ -447,6 +504,24 @@ func (s *Suite) TestConfigure() { ReloadInterval: defaultReloadInterval, }, }, + { + name: "API server cache enabled", + trustDomain: "example.org", + hcl: ` + kubelet_read_only_port = 12345 + api_server_cache { + enabled = true + } + `, + config: &config{ + Insecure: true, + KubeletURL: "http://127.0.0.1:12345", + MaxPollAttempts: defaultMaxPollAttempts, + PollRetryInterval: defaultPollRetryInterval, + ReloadInterval: defaultReloadInterval, + APIServerCache: true, + }, + }, { name: "invalid hcl", @@ -624,6 +699,119 @@ func (s *Suite) TestConfigure() { assert.Equal(t, testCase.config.MaxPollAttempts, c.MaxPollAttempts) assert.Equal(t, testCase.config.PollRetryInterval, c.PollRetryInterval) assert.Equal(t, testCase.config.ReloadInterval, c.ReloadInterval) + assert.Equal(t, testCase.config.APIServerCache, c.APIServerCache.Enabled) + }) + } +} + +func (s *Suite) TestConfigureBroker() { + testCases := []struct { + name string + hcl string + expectedErr string + }{ + { + name: "valid broker", + hcl: fmt.Sprintf(` + kubelet_read_only_port = 12345 + %s + `, testBrokerConfig()), + }, + { + name: "valid pod reference scope", + hcl: ` + kubelet_read_only_port = 12345 + broker { + brokers = [ + { + id = "spiffe://example.org/broker" + pod_reference_scope = "cluster" + } + ] + } + `, + }, + { + name: "empty brokers", + hcl: ` + kubelet_read_only_port = 12345 + broker { + brokers = [] + } + `, + expectedErr: "broker.brokers: at least one broker is required", + }, + { + name: "missing id", + hcl: ` + kubelet_read_only_port = 12345 + broker { + brokers = [ + {} + ] + } + `, + expectedErr: "broker.brokers[0].id: must be specified", + }, + { + name: "duplicate id", + hcl: ` + kubelet_read_only_port = 12345 + broker { + brokers = [ + { id = "spiffe://example.org/broker" }, + { id = "spiffe://example.org/broker" }, + ] + } + `, + expectedErr: "broker.brokers[spiffe://example.org/broker].id: duplicate broker id", + }, + { + name: "invalid id", + hcl: ` + kubelet_read_only_port = 12345 + broker { + brokers = [ + { id = "not-a-spiffe-id" } + ] + } + `, + expectedErr: "broker.brokers[not-a-spiffe-id].id", + }, + { + name: "invalid pod reference scope", + hcl: ` + kubelet_read_only_port = 12345 + broker { + brokers = [ + { + id = "spiffe://example.org/broker" + pod_reference_scope = "Cluster" + } + ] + } + `, + expectedErr: `broker.brokers[spiffe://example.org/broker].pod_reference_scope: unsupported value "Cluster"; must be one of [agent_node, cluster]`, + }, + } + + for _, tc := range testCases { + s.T().Run(tc.name, func(t *testing.T) { + p := s.newPlugin() + + var err error + plugintest.Load(t, builtin(p), nil, + plugintest.CoreConfig(catalog.CoreConfig{ + TrustDomain: spiffeid.RequireTrustDomainFromString("example.org"), + }), + plugintest.Configure(tc.hcl), + plugintest.CaptureConfigureError(&err)) + + if tc.expectedErr != "" { + s.RequireGRPCStatusContains(err, codes.InvalidArgument, tc.expectedErr) + return + } + require.NoError(t, err) }) } } @@ -791,6 +979,41 @@ func (s *Suite) loadPlugin(configuration string) workloadattestor.WorkloadAttest return v1 } +func (s *Suite) loadPluginWithKubeClient(configuration string, kubeClient client.Client) workloadattestor.WorkloadAttestor { + v1 := new(workloadattestor.V1) + p := s.newPlugin() + p.kubeClient = kubeClient + + plugintest.Load(s.T(), builtin(p), v1, + plugintest.CoreConfig(catalog.CoreConfig{ + TrustDomain: spiffeid.RequireTrustDomainFromString("example.org"), + }), + plugintest.Configure(configuration), + ) + + if cHelper := s.oc.getContainerHelper(p); cHelper != nil { + p.setContainerHelper(cHelper) + } + return v1 +} + +func (s *Suite) loadRawPluginWithKubeClient(configuration string, kubeClient client.Client) *Plugin { + p := s.newPlugin() + p.kubeClient = kubeClient + + plugintest.Load(s.T(), builtin(p), nil, + plugintest.CoreConfig(catalog.CoreConfig{ + TrustDomain: spiffeid.RequireTrustDomainFromString("example.org"), + }), + plugintest.Configure(configuration), + ) + + if cHelper := s.oc.getContainerHelper(p); cHelper != nil { + p.setContainerHelper(cHelper) + } + return p +} + func (s *Suite) loadInsecurePlugin() workloadattestor.WorkloadAttestor { return s.loadPlugin(fmt.Sprintf(` kubelet_read_only_port = %d @@ -799,6 +1022,28 @@ func (s *Suite) loadInsecurePlugin() workloadattestor.WorkloadAttestor { `, s.kubeletPort())) } +func (s *Suite) loadInsecurePluginWithBroker() workloadattestor.WorkloadAttestor { + return s.loadInsecurePluginWithExtra(testBrokerConfig()) +} + +func (s *Suite) loadInsecurePluginWithBrokerAndKubeClient(kubeClient client.Client) workloadattestor.WorkloadAttestor { + return s.loadPluginWithKubeClient(fmt.Sprintf(` + kubelet_read_only_port = %d + max_poll_attempts = 5 + poll_retry_interval = "1s" + %s +`, s.kubeletPort(), testBrokerConfig()), kubeClient) +} + +func (s *Suite) loadRawInsecurePluginWithBrokerAndKubeClient(kubeClient client.Client) *Plugin { + return s.loadRawPluginWithKubeClient(fmt.Sprintf(` + kubelet_read_only_port = %d + max_poll_attempts = 5 + poll_retry_interval = "1s" + %s +`, s.kubeletPort(), testBrokerConfig()), kubeClient) +} + func (s *Suite) loadInsecurePluginWithExtra(extraConfig string) workloadattestor.WorkloadAttestor { return s.loadPlugin(fmt.Sprintf(` kubelet_read_only_port = %d @@ -818,6 +1063,57 @@ func (s *Suite) loadInsecurePluginWithSigstore() workloadattestor.WorkloadAttest `, s.kubeletPort())) } +func testBrokerConfig() string { + return testBrokerConfigWithPodReferenceScope("") +} + +func testBrokerConfigWithPodReferenceScope(scope string) string { + var scopeConfig string + if scope != "" { + scopeConfig = fmt.Sprintf("\n\t\t\t\t\tpod_reference_scope = %q", scope) + } + return fmt.Sprintf(` + broker { + brokers = [ + { + id = %q%s + } + ] + } +`, testBrokerID, scopeConfig) +} + +func testBrokerContext() context.Context { + return brokercontext.WithCallerID(context.Background(), testBrokerSPIFFEID) +} + +func fakeKubeClientWithSubjectAccessReview(allowed bool, reviews *[]authv1.SubjectAccessReview, objects ...client.Object) client.Client { + return fakeKubeClientWithSubjectAccessReviewAndRESTMapper(allowed, reviews, nil, objects...) +} + +func fakeKubeClientWithSubjectAccessReviewAndRESTMapper(allowed bool, reviews *[]authv1.SubjectAccessReview, mapper meta.RESTMapper, objects ...client.Object) client.Client { + builder := fake.NewClientBuilder(). + WithScheme(k8sScheme). + WithObjects(objects...). + WithInterceptorFuncs(interceptor.Funcs{ + Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + review, ok := obj.(*authv1.SubjectAccessReview) + if !ok { + return c.Create(ctx, obj, opts...) + } + if reviews != nil { + *reviews = append(*reviews, *review.DeepCopy()) + } + review.Status.Allowed = allowed + return nil + }, + }) + if mapper != nil { + builder = builder.WithRESTMapper(mapper) + } + return builder.Build() +} + func (s *Suite) startInsecureKubelet() { s.setServer(httptest.NewServer(http.HandlerFunc(s.serveHTTP))) } @@ -983,6 +1279,14 @@ func (s *Suite) requireSelectorsEqual(expected, actual []*common.Selector) { s.RequireProtoListEqual(expected, actual) } +func (s *Suite) requireSelectorValuesEqual(expected []*common.Selector, actual []string) { + selectors := make([]*common.Selector, 0, len(actual)) + for _, value := range actual { + selectors = append(selectors, &common.Selector{Type: pluginName, Value: value}) + } + s.requireSelectorsEqual(expected, selectors) +} + func (s *Suite) goAttest(p workloadattestor.WorkloadAttestor) <-chan attestResult { resultCh := make(chan attestResult, 1) go func() { @@ -1021,6 +1325,445 @@ func (s *Suite) podListResponseCount() int { return len(s.podList) } +// testPodUID is the UID of the blog pod in testdata/pod_list.json. +const testPodUID = "2c48913c-b29f-11e7-9350-020968147796" + +func testAPIServerBlogPod() *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "blog-24ck7", + Namespace: "default", + UID: types.UID(testPodUID), + Labels: map[string]string{ + "k8s-app": "blog", + "version": "v0", + }, + OwnerReferences: []metav1.OwnerReference{ + {Kind: "ReplicationController", Name: "blog", UID: "2c401175-b29f-11e7-9350-020968147796"}, + }, + }, + Spec: corev1.PodSpec{ + NodeName: "k8s-node-1", + ServiceAccountName: "default", + }, + Status: corev1.PodStatus{ + ContainerStatuses: []corev1.ContainerStatus{ + { + Image: "localhost/spiffe/blog:latest", + ImageID: "docker-pullable://localhost/spiffe/blog@sha256:0cfdaced91cb46dd7af48309799a3c351e4ca2d5e1ee9737ca0cbd932cb79898", + }, + { + Image: "localhost/spiffe/ghostunnel:latest", + ImageID: "docker-pullable://localhost/spiffe/ghostunnel@sha256:b2fc20676c92a433b9a91f3f4535faddec0c2c3613849ac12f02c1d5cfcd4c3a", + }, + }, + }, + } +} + +func (s *Suite) TestAttestReferenceWithPIDAllowedByDefault() { + s.startInsecureKubelet() + var reviews []authv1.SubjectAccessReview + p := s.loadInsecurePluginWithBrokerAndKubeClient(fakeKubeClientWithSubjectAccessReview(true, &reviews)) + + s.addPodListResponse(podListFilePath) + s.addGetContainerResponsePidInPod() + + anyRef, err := anypb.New(&broker.WorkloadPIDReference{Pid: int32(pid)}) + s.Require().NoError(err) + + selectors, err := p.AttestReference(testBrokerContext(), anyRef) + s.Require().NoError(err) + s.requireSelectorsEqual(testPodAndContainerSelectors, selectors) + s.Require().Len(reviews, 1) + + review := reviews[0] + assert.Equal(s.T(), testBrokerID, review.Spec.User) + assert.Empty(s.T(), review.Spec.Groups) + if assert.NotNil(s.T(), review.Spec.ResourceAttributes) { + assert.Equal(s.T(), "", review.Spec.ResourceAttributes.Group) + assert.Equal(s.T(), "pods", review.Spec.ResourceAttributes.Resource) + assert.Equal(s.T(), "default", review.Spec.ResourceAttributes.Namespace) + assert.Equal(s.T(), "blog-24ck7", review.Spec.ResourceAttributes.Name) + assert.Equal(s.T(), brokerImpersonationReviewVerb, review.Spec.ResourceAttributes.Verb) + } +} + +func (s *Suite) TestAttestReferenceWithPIDBrokerRBACDenied() { + s.startInsecureKubelet() + p := s.loadInsecurePluginWithBrokerAndKubeClient(fakeKubeClientWithSubjectAccessReview(false, nil)) + + s.addPodListResponse(podListFilePath) + s.addGetContainerResponsePidInPod() + + anyRef, err := anypb.New(&broker.WorkloadPIDReference{Pid: int32(pid)}) + s.Require().NoError(err) + + selectors, err := p.AttestReference(testBrokerContext(), anyRef) + s.RequireGRPCStatusContains(err, codes.PermissionDenied, "Kubernetes authorizer does not allow the broker to use impersonate-via-spire for the referenced object") + s.Require().Nil(selectors) +} + +func (s *Suite) TestAttestReferenceWithPodUID_FoundInKubelet() { + s.startInsecureKubelet() + p := s.loadInsecurePluginWithBrokerAndKubeClient(fakeKubeClientWithSubjectAccessReview(true, nil)) + + s.addPodListResponse(podListFilePath) + + anyRef, err := anypb.New(&broker.KubernetesObjectReference{Type: &broker.KubernetesObjectType{Plural: "pods", Group: "core"}, Uid: testPodUID}) + s.Require().NoError(err) + + selectors, err := p.AttestReference(testBrokerContext(), anyRef) + s.Require().NoError(err) + s.requireSelectorsEqual(testPodSelectors, selectors) +} + +func (s *Suite) TestAttestReferenceWithPodUID_DefaultScopeFallsBackToAPIServerForAgentNodePod() { + s.startInsecureKubelet() + + // Serve an empty pod list so the kubelet lookup finds nothing. + emptyPodList := []byte(`{"items":[]}`) + s.podListMu.Lock() + s.podList = append(s.podList, emptyPodList) + s.podListMu.Unlock() + + fakeClient := fakeKubeClientWithSubjectAccessReview(true, nil, testAPIServerBlogPod()) + cfg := fmt.Sprintf(` + kubelet_read_only_port = %d + max_poll_attempts = 5 + poll_retry_interval = "1s" + node_name = "k8s-node-1" + %s +`, s.kubeletPort(), testBrokerConfig()) + wa := s.loadPluginWithKubeClient(cfg, fakeClient) + + anyRef, err := anypb.New(&broker.KubernetesObjectReference{Type: &broker.KubernetesObjectType{Plural: "pods", Group: "core"}, Uid: testPodUID}) + s.Require().NoError(err) + + selectors, err := wa.AttestReference(testBrokerContext(), anyRef) + s.Require().NoError(err) + s.requireSelectorsEqual(testPodSelectors, selectors) +} + +func (s *Suite) TestAttestReferenceWithPodUID_DefaultScopeRejectsAPIServerPodOnOtherNode() { + s.startInsecureKubelet() + + emptyPodList := []byte(`{"items":[]}`) + s.podListMu.Lock() + s.podList = append(s.podList, emptyPodList) + s.podListMu.Unlock() + + pod := testAPIServerBlogPod() + pod.Spec.NodeName = "other-node" + fakeClient := fakeKubeClientWithSubjectAccessReview(true, nil, pod) + cfg := fmt.Sprintf(` + kubelet_read_only_port = %d + max_poll_attempts = 5 + poll_retry_interval = "1s" + node_name = "k8s-node-1" + %s +`, s.kubeletPort(), testBrokerConfig()) + wa := s.loadPluginWithKubeClient(cfg, fakeClient) + + anyRef, err := anypb.New(&broker.KubernetesObjectReference{Type: &broker.KubernetesObjectType{Plural: "pods", Group: "core"}, Uid: testPodUID}) + s.Require().NoError(err) + + selectors, err := wa.AttestReference(testBrokerContext(), anyRef) + s.RequireGRPCStatusContains(err, codes.PermissionDenied, "pod is not on the agent node") + s.Require().Nil(selectors) +} + +func (s *Suite) TestAttestReferenceWithPodUID_DefaultScopeRequiresAgentNodeNameForAPIServerFallback() { + s.startInsecureKubelet() + + emptyPodList := []byte(`{"items":[]}`) + s.podListMu.Lock() + s.podList = append(s.podList, emptyPodList) + s.podListMu.Unlock() + + fakeClient := fakeKubeClientWithSubjectAccessReview(true, nil, testAPIServerBlogPod()) + cfg := fmt.Sprintf(` + kubelet_read_only_port = %d + max_poll_attempts = 5 + poll_retry_interval = "1s" + %s +`, s.kubeletPort(), testBrokerConfig()) + wa := s.loadPluginWithKubeClient(cfg, fakeClient) + + anyRef, err := anypb.New(&broker.KubernetesObjectReference{Type: &broker.KubernetesObjectType{Plural: "pods", Group: "core"}, Uid: testPodUID}) + s.Require().NoError(err) + + selectors, err := wa.AttestReference(testBrokerContext(), anyRef) + s.RequireGRPCStatusContains(err, codes.Internal, "agent node name is not configured") + s.Require().Nil(selectors) +} + +func (s *Suite) TestAttestReferenceWithPodUID_FallbackToAPIServerWithClusterScope() { + s.startInsecureKubelet() + + // Serve an empty pod list so the kubelet lookup finds nothing. + emptyPodList := []byte(`{"items":[]}`) + s.podListMu.Lock() + s.podList = append(s.podList, emptyPodList) + s.podListMu.Unlock() + + fakeClient := fakeKubeClientWithSubjectAccessReview(true, nil, testAPIServerBlogPod()) + cfg := fmt.Sprintf(` + kubelet_read_only_port = %d + max_poll_attempts = 5 + poll_retry_interval = "1s" + %s +`, s.kubeletPort(), testBrokerConfigWithPodReferenceScope("cluster")) + wa := s.loadPluginWithKubeClient(cfg, fakeClient) + + anyRef, err := anypb.New(&broker.KubernetesObjectReference{Type: &broker.KubernetesObjectType{Plural: "pods", Group: "core"}, Uid: testPodUID}) + s.Require().NoError(err) + + selectors, err := wa.AttestReference(testBrokerContext(), anyRef) + s.Require().NoError(err) + s.requireSelectorsEqual(testPodSelectors, selectors) +} + +func (s *Suite) TestAttestReferenceWithPodUID_NotFound() { + s.startInsecureKubelet() + + emptyPodList := []byte(`{"items":[]}`) + s.podListMu.Lock() + s.podList = append(s.podList, emptyPodList) + s.podListMu.Unlock() + + fakeClient := fakeKubeClientWithSubjectAccessReview(true, nil) + cfg := fmt.Sprintf(` + kubelet_read_only_port = %d + max_poll_attempts = 5 + poll_retry_interval = "1s" + %s +`, s.kubeletPort(), testBrokerConfig()) + wa := s.loadPluginWithKubeClient(cfg, fakeClient) + + anyRef, err := anypb.New(&broker.KubernetesObjectReference{Type: &broker.KubernetesObjectType{Plural: "pods", Group: "core"}, Uid: "nonexistent-uid"}) + s.Require().NoError(err) + + selectors, err := wa.AttestReference(testBrokerContext(), anyRef) + s.RequireGRPCStatusContains(err, codes.NotFound, "not found") + s.Require().Nil(selectors) +} + +func (s *Suite) TestAttestReferenceUnsupportedType() { + s.startInsecureKubelet() + p := s.loadInsecurePluginWithBroker() + + anyRef := &anypb.Any{TypeUrl: "type.googleapis.com/unsupported.Type", Value: []byte{}} + + selectors, err := p.AttestReference(testBrokerContext(), anyRef) + s.RequireGRPCStatusContains(err, codes.InvalidArgument, "unsupported reference type") + s.Require().Nil(selectors) +} + +func (s *Suite) TestAttestReferenceKubernetesObjectValidation() { + s.startInsecureKubelet() + p := s.loadInsecurePluginWithBrokerAndKubeClient(fakeKubeClientWithSubjectAccessReview(true, nil)) + + testCases := []struct { + name string + ref *broker.KubernetesObjectReference + errMsg string + }{ + { + name: "missing type", + ref: &broker.KubernetesObjectReference{Uid: testPodUID}, + errMsg: "object reference is missing type", + }, + { + name: "missing plural", + ref: &broker.KubernetesObjectReference{Type: &broker.KubernetesObjectType{Group: "core"}, Uid: testPodUID}, + errMsg: "object reference type is missing plural", + }, + { + name: "missing group", + ref: &broker.KubernetesObjectReference{Type: &broker.KubernetesObjectType{Plural: "pods"}, Uid: testPodUID}, + errMsg: "object reference type is missing group", + }, + { + name: "missing key and uid", + ref: &broker.KubernetesObjectReference{Type: &broker.KubernetesObjectType{Plural: "pods", Group: "core"}}, + errMsg: "object reference is missing key and UID", + }, + { + name: "key missing name", + ref: &broker.KubernetesObjectReference{ + Type: &broker.KubernetesObjectType{Plural: "pods", Group: "core"}, + Key: &broker.KubernetesObjectKey{Namespace: "shop"}, + }, + errMsg: "object reference key is missing name", + }, + { + name: "pod key missing namespace", + ref: &broker.KubernetesObjectReference{ + Type: &broker.KubernetesObjectType{Plural: "pods", Group: "core"}, + Key: &broker.KubernetesObjectKey{Name: "checkout"}, + }, + errMsg: "namespace is required when name is set for a namespaced resource", + }, + } + + for _, tc := range testCases { + s.T().Run(tc.name, func(t *testing.T) { + anyRef, err := anypb.New(tc.ref) + s.Require().NoError(err) + + selectors, err := p.AttestReference(testBrokerContext(), anyRef) + s.RequireGRPCStatusContains(err, codes.InvalidArgument, tc.errMsg) + s.Require().Nil(selectors) + }) + } +} + +func (s *Suite) TestAttestReferenceRequiresBrokerConfig() { + s.startInsecureKubelet() + p := s.loadInsecurePlugin() + + anyRef, err := anypb.New(&broker.KubernetesObjectReference{Type: &broker.KubernetesObjectType{Plural: "pods", Group: "core"}, Uid: testPodUID}) + s.Require().NoError(err) + + selectors, err := p.AttestReference(testBrokerContext(), anyRef) + s.RequireGRPCStatusContains(err, codes.Internal, "broker configuration missing") + s.Require().Nil(selectors) +} + +func (s *Suite) TestAttestReferenceWithoutBrokerCallerDoesNotRequireBrokerConfig() { + s.startInsecureKubelet() + p := s.loadInsecurePlugin() + s.addPodListResponse(podListFilePath) + + anyRef, err := anypb.New(&broker.KubernetesObjectReference{Type: &broker.KubernetesObjectType{Plural: "pods", Group: "core"}, Uid: testPodUID}) + s.Require().NoError(err) + + selectors, err := p.AttestReference(context.Background(), anyRef) + s.Require().NoError(err) + s.requireSelectorsEqual(testPodSelectors, selectors) +} + +func (s *Suite) TestAttestReferenceWithoutBrokerCallerDoesNotRunBrokerRBAC() { + s.startInsecureKubelet() + var reviews []authv1.SubjectAccessReview + p := s.loadInsecurePluginWithBrokerAndKubeClient(fakeKubeClientWithSubjectAccessReview(false, &reviews)) + s.addPodListResponse(podListFilePath) + + anyRef, err := anypb.New(&broker.KubernetesObjectReference{Type: &broker.KubernetesObjectType{Plural: "pods", Group: "core"}, Uid: testPodUID}) + s.Require().NoError(err) + + selectors, err := p.AttestReference(context.Background(), anyRef) + s.Require().NoError(err) + s.requireSelectorsEqual(testPodSelectors, selectors) + s.Require().Empty(reviews) +} + +func (s *Suite) TestAttestReferenceBrokerRBACDenied() { + s.startInsecureKubelet() + p := s.loadInsecurePluginWithBrokerAndKubeClient(fakeKubeClientWithSubjectAccessReview(false, nil)) + s.addPodListResponse(podListFilePath) + + anyRef, err := anypb.New(&broker.KubernetesObjectReference{Type: &broker.KubernetesObjectType{Plural: "pods", Group: "core"}, Uid: testPodUID}) + s.Require().NoError(err) + + selectors, err := p.AttestReference(testBrokerContext(), anyRef) + s.RequireGRPCStatusContains(err, codes.PermissionDenied, "Kubernetes authorizer does not allow the broker to use impersonate-via-spire for the referenced object") + s.Require().Nil(selectors) +} + +func (s *Suite) TestAttestReferenceBrokerRBACUsesResolvedObject() { + s.startInsecureKubelet() + var reviews []authv1.SubjectAccessReview + p := s.loadInsecurePluginWithBrokerAndKubeClient(fakeKubeClientWithSubjectAccessReview(true, &reviews)) + s.addPodListResponse(podListFilePath) + + anyRef, err := anypb.New(&broker.KubernetesObjectReference{Type: &broker.KubernetesObjectType{Plural: "pods", Group: "core"}, Uid: testPodUID}) + s.Require().NoError(err) + + selectors, err := p.AttestReference(testBrokerContext(), anyRef) + s.Require().NoError(err) + s.requireSelectorsEqual(testPodSelectors, selectors) + s.Require().Len(reviews, 1) + + review := reviews[0] + assert.Equal(s.T(), testBrokerID, review.Spec.User) + assert.Empty(s.T(), review.Spec.Groups) + if assert.NotNil(s.T(), review.Spec.ResourceAttributes) { + assert.Equal(s.T(), "", review.Spec.ResourceAttributes.Group) + assert.Equal(s.T(), "pods", review.Spec.ResourceAttributes.Resource) + assert.Equal(s.T(), "default", review.Spec.ResourceAttributes.Namespace) + assert.Equal(s.T(), "blog-24ck7", review.Spec.ResourceAttributes.Name) + assert.Equal(s.T(), brokerImpersonationReviewVerb, review.Spec.ResourceAttributes.Verb) + } +} + +func (s *Suite) TestAttestReferenceBrokerRBACUsesResolvedGenericObject() { + s.startInsecureKubelet() + + gvk := schema.GroupVersionKind{ + Group: "kustomize.toolkit.fluxcd.io", + Version: "v1", + Kind: "Kustomization", + } + gvr := schema.GroupVersionResource{ + Group: gvk.Group, + Version: gvk.Version, + Resource: "kustomizations", + } + mapper := meta.NewDefaultRESTMapper([]schema.GroupVersion{gvk.GroupVersion()}) + mapper.AddSpecific(gvk, gvr, schema.GroupVersionResource{ + Group: gvk.Group, + Version: gvk.Version, + Resource: "kustomization", + }, meta.RESTScopeNamespace) + + obj := &metav1.PartialObjectMetadata{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tenant-a", + Namespace: "flux-system", + UID: "kustomization-uid", + }, + } + obj.SetGroupVersionKind(gvk) + + var reviews []authv1.SubjectAccessReview + p := s.loadInsecurePluginWithBrokerAndKubeClient(fakeKubeClientWithSubjectAccessReviewAndRESTMapper(true, &reviews, mapper, obj)) + + anyRef, err := anypb.New(&broker.KubernetesObjectReference{ + Type: &broker.KubernetesObjectType{ + Plural: "kustomizations", + Group: "kustomize.toolkit.fluxcd.io", + }, + Uid: "kustomization-uid", + }) + s.Require().NoError(err) + + selectors, err := p.AttestReference(testBrokerContext(), anyRef) + s.Require().NoError(err) + s.Require().Len(reviews, 1) + + selectorValues := make([]string, 0, len(selectors)) + for _, selector := range selectors { + assert.Equal(s.T(), "k8s", selector.Type) + selectorValues = append(selectorValues, selector.Value) + } + assert.Contains(s.T(), selectorValues, "uid:kustomization-uid") + assert.Contains(s.T(), selectorValues, "namespace:flux-system") + assert.Contains(s.T(), selectorValues, "name:tenant-a") + + review := reviews[0] + assert.Equal(s.T(), testBrokerID, review.Spec.User) + assert.Empty(s.T(), review.Spec.Groups) + if assert.NotNil(s.T(), review.Spec.ResourceAttributes) { + assert.Equal(s.T(), "kustomize.toolkit.fluxcd.io", review.Spec.ResourceAttributes.Group) + assert.Equal(s.T(), "kustomizations", review.Spec.ResourceAttributes.Resource) + assert.Equal(s.T(), "flux-system", review.Spec.ResourceAttributes.Namespace) + assert.Equal(s.T(), "tenant-a", review.Spec.ResourceAttributes.Name) + assert.Equal(s.T(), brokerImpersonationReviewVerb, review.Spec.ResourceAttributes.Verb) + } +} + type fakeSigstoreVerifier struct { mu sync.Mutex @@ -1043,3 +1786,81 @@ func (v *fakeSigstoreVerifier) Verify(_ context.Context, imageID string) ([]stri return nil, fmt.Errorf("failed to verify signature for image %s", imageID) } + +func TestGetSelectorValuesFromObjectMeta(t *testing.T) { + truePtr := true + falsePtr := false + + for _, tc := range []struct { + name string + objType *broker.KubernetesObjectType + gvk schema.GroupVersionKind + obj *metav1.PartialObjectMetadata + expected []string + }{ + { + name: "namespaced with all fields", + objType: &broker.KubernetesObjectType{Plural: "deployments", Group: "apps"}, + gvk: schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "Deployment"}, + obj: &metav1.PartialObjectMetadata{ + ObjectMeta: metav1.ObjectMeta{ + Name: "checkout", + Namespace: "shop", + UID: "a1b2c3", + Labels: map[string]string{"app": "checkout"}, + Annotations: map[string]string{"team": "payments"}, // ignored — annotations are not used as selectors + OwnerReferences: []metav1.OwnerReference{ + {APIVersion: "apps/v1", Kind: "ReplicaSet", Name: "checkout-rs", UID: "owner-uid-1", Controller: &truePtr}, + {APIVersion: "v1", Kind: "ConfigMap", Name: "checkout-cm", UID: "owner-uid-2", Controller: &falsePtr}, + }, + }, + }, + expected: []string{ + "uid:a1b2c3", + "resource:deployments.apps", + "plural:deployments", + "group:apps", + "version:v1", + "apiVersion:apps/v1", + "kind:Deployment", + "name:checkout", + "namespace:shop", + "key:shop/checkout", + "label:app:checkout", + "owner-key:apps/ReplicaSet/checkout-rs", + "owner-uid:apps/ReplicaSet/owner-uid-1", + "controller-key:apps/ReplicaSet/checkout-rs", + "controller-uid:apps/ReplicaSet/owner-uid-1", + "owner-key:core/ConfigMap/checkout-cm", + "owner-uid:core/ConfigMap/owner-uid-2", + }, + }, + { + name: "cluster-scoped, no owners/labels", + objType: &broker.KubernetesObjectType{Plural: "nodes", Group: "core"}, + gvk: schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Node"}, + obj: &metav1.PartialObjectMetadata{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ip-10-0-1-42.ec2.internal", + UID: "node-uid", + }, + }, + expected: []string{ + "uid:node-uid", + "resource:nodes.core", + "plural:nodes", + "group:core", + "version:v1", + "apiVersion:v1", + "kind:Node", + "name:ip-10-0-1-42.ec2.internal", + "key:ip-10-0-1-42.ec2.internal", + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + got := getSelectorValuesFromObjectMeta(tc.objType, tc.gvk, tc.obj) + assert.ElementsMatch(t, tc.expected, got) + }) + } +} diff --git a/pkg/agent/plugin/workloadattestor/systemd/systemd_posix.go b/pkg/agent/plugin/workloadattestor/systemd/systemd_posix.go index 4fbfeef9dc..e2d017c463 100644 --- a/pkg/agent/plugin/workloadattestor/systemd/systemd_posix.go +++ b/pkg/agent/plugin/workloadattestor/systemd/systemd_posix.go @@ -75,6 +75,13 @@ func (p *Plugin) Attest(ctx context.Context, req *workloadattestorv1.AttestReque }, nil } +// AttestReference returns Unimplemented. This plugin does not handle +// reference-based workload attestation; the host falls back to PID-based +// Attest when the reference is a WorkloadPIDReference. +func (p *Plugin) AttestReference(_ context.Context, _ *workloadattestorv1.AttestReferenceRequest) (*workloadattestorv1.AttestReferenceResponse, error) { + return nil, status.Error(codes.Unimplemented, "AttestReference not implemented") +} + func (p *Plugin) Close() error { p.dbusMutex.Lock() defer p.dbusMutex.Unlock() diff --git a/pkg/agent/plugin/workloadattestor/unix/unix_posix.go b/pkg/agent/plugin/workloadattestor/unix/unix_posix.go index 5f13b3e218..da5599fa6a 100644 --- a/pkg/agent/plugin/workloadattestor/unix/unix_posix.go +++ b/pkg/agent/plugin/workloadattestor/unix/unix_posix.go @@ -205,6 +205,13 @@ func (p *Plugin) Attest(_ context.Context, req *workloadattestorv1.AttestRequest }, nil } +// AttestReference returns Unimplemented. This plugin does not handle +// reference-based workload attestation; the host falls back to PID-based +// Attest when the reference is a WorkloadPIDReference. +func (p *Plugin) AttestReference(_ context.Context, _ *workloadattestorv1.AttestReferenceRequest) (*workloadattestorv1.AttestReferenceResponse, error) { + return nil, status.Error(codes.Unimplemented, "AttestReference not implemented") +} + func (p *Plugin) Configure(_ context.Context, req *configv1.ConfigureRequest) (*configv1.ConfigureResponse, error) { newConfig, _, err := pluginconf.Build(req, buildConfig) if err != nil { diff --git a/pkg/agent/plugin/workloadattestor/v1.go b/pkg/agent/plugin/workloadattestor/v1.go index 329a7a17de..f804c80cea 100644 --- a/pkg/agent/plugin/workloadattestor/v1.go +++ b/pkg/agent/plugin/workloadattestor/v1.go @@ -2,12 +2,19 @@ package workloadattestor import ( "context" + "errors" "fmt" + "github.com/spiffe/go-spiffe/v2/exp/proto/spiffe/broker" workloadattestorv1 "github.com/spiffe/spire-plugin-sdk/proto/spire/plugin/agent/workloadattestor/v1" + "github.com/spiffe/spire/pkg/agent/broker/brokercontext" "github.com/spiffe/spire/pkg/common/plugin" "github.com/spiffe/spire/pkg/common/util" "github.com/spiffe/spire/proto/spire/common" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" ) type V1 struct { @@ -15,27 +22,98 @@ type V1 struct { workloadattestorv1.WorkloadAttestorPluginClient } +// Attest invokes the plugin's PID-based Attest RPC. Older plugins implement +// only this RPC; newer plugins may instead implement only AttestReference. +// To bridge both styles, if Attest returns Unimplemented, we pack the PID as +// a WorkloadPIDReference and retry through AttestReference. func (v1 *V1) Attest(ctx context.Context, pid int) ([]*common.Selector, error) { pidInt32, err := util.CheckedCast[int32](pid) if err != nil { return nil, v1.WrapErr(fmt.Errorf("invalid value for PID: %w", err)) } + + selectors, err := v1.attestByPID(ctx, pidInt32) + if status.Code(err) != codes.Unimplemented { + return selectors, err + } + + // Plugin doesn't implement Attest; try AttestReference with a packed + // WorkloadPIDReference. We call the raw method directly to avoid an + // infinite loop: if AttestReference is also unimplemented we surface the + // original Unimplemented to the caller. + ref, packErr := anypb.New(&broker.WorkloadPIDReference{Pid: pidInt32}) + if packErr != nil { + return nil, v1.WrapErr(packErr) + } + return v1.attestByReference(ctx, ref) +} + +// AttestReference invokes the plugin's reference-based AttestReference RPC. +// If the plugin returns Unimplemented and the supplied reference is a +// WorkloadPIDReference, we fall back to the legacy PID-based Attest. Other +// reference types surface the Unimplemented error directly because there's +// no equivalent legacy path. +func (v1 *V1) AttestReference(ctx context.Context, ref *anypb.Any) ([]*common.Selector, error) { + selectors, err := v1.attestByReference(ctx, ref) + if status.Code(err) != codes.Unimplemented { + return selectors, err + } + + pid, extractErr := extractPIDReference(ref) + if extractErr != nil { + // Plugin doesn't implement AttestReference and the reference isn't a + // PID we can fall back on; surface the Unimplemented to the caller. + return nil, err + } + return v1.attestByPID(ctx, pid) +} + +// attestByPID is the raw plugin call without fallback. The wrapping +// public methods orchestrate the cross-RPC fallback so neither raw helper +// can recurse. +func (v1 *V1) attestByPID(ctx context.Context, pid int32) ([]*common.Selector, error) { resp, err := v1.WorkloadAttestorPluginClient.Attest(ctx, &workloadattestorv1.AttestRequest{ - Pid: pidInt32, + Pid: pid, }) if err != nil { return nil, v1.WrapErr(err) } + return v1.selectorsFrom(resp.SelectorValues), nil +} + +// attestByReference is the raw plugin call without fallback. +func (v1 *V1) attestByReference(ctx context.Context, ref *anypb.Any) ([]*common.Selector, error) { + ctx = brokercontext.AppendCallerIDToOutgoingContext(ctx) + resp, err := v1.WorkloadAttestorPluginClient.AttestReference(ctx, &workloadattestorv1.AttestReferenceRequest{ + Reference: ref, + }) + if err != nil { + return nil, v1.WrapErr(err) + } + return v1.selectorsFrom(resp.SelectorValues), nil +} + +func (v1 *V1) selectorsFrom(values []string) []*common.Selector { + if values == nil { + return nil + } + selectors := make([]*common.Selector, 0, len(values)) + for _, value := range values { + selectors = append(selectors, &common.Selector{ + Type: v1.Name(), + Value: value, + }) + } + return selectors +} - var selectors []*common.Selector - if resp.SelectorValues != nil { - selectors = make([]*common.Selector, 0, len(resp.SelectorValues)) - for _, selectorValue := range resp.SelectorValues { - selectors = append(selectors, &common.Selector{ - Type: v1.Name(), - Value: selectorValue, - }) +func extractPIDReference(ref *anypb.Any) (int32, error) { + if ref.GetTypeUrl() == "type.googleapis.com/spiffe.broker.WorkloadPIDReference" { + var pidRef broker.WorkloadPIDReference + if err := anypb.UnmarshalTo(ref, &pidRef, proto.UnmarshalOptions{}); err != nil { + return 0, fmt.Errorf("unmarshaling PID reference: %w", err) } + return pidRef.Pid, nil } - return selectors, nil + return 0, errors.New("PID reference not found") } diff --git a/pkg/agent/plugin/workloadattestor/v1_test.go b/pkg/agent/plugin/workloadattestor/v1_test.go index 854c669b68..b370696240 100644 --- a/pkg/agent/plugin/workloadattestor/v1_test.go +++ b/pkg/agent/plugin/workloadattestor/v1_test.go @@ -4,7 +4,9 @@ import ( "context" "testing" + "github.com/spiffe/go-spiffe/v2/spiffeid" workloadattestorv1 "github.com/spiffe/spire-plugin-sdk/proto/spire/plugin/agent/workloadattestor/v1" + "github.com/spiffe/spire/pkg/agent/broker/brokercontext" "github.com/spiffe/spire/pkg/agent/plugin/workloadattestor" "github.com/spiffe/spire/pkg/common/catalog" "github.com/spiffe/spire/proto/spire/common" @@ -13,6 +15,7 @@ import ( "github.com/stretchr/testify/require" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/anypb" ) func TestV1(t *testing.T) { @@ -47,6 +50,17 @@ func TestV1(t *testing.T) { }) } +func TestV1AttestReferencePropagatesBrokerCallerID(t *testing.T) { + expectedID := spiffeid.RequireFromString("spiffe://example.org/broker") + fake := &fakeReferencePluginV1{expectedID: expectedID} + plugin := new(workloadattestor.V1) + plugintest.Load(t, catalog.MakeBuiltIn("test", workloadattestorv1.WorkloadAttestorPluginServer(fake)), plugin) + + selectors, err := plugin.AttestReference(brokercontext.WithCallerID(context.Background(), expectedID), &anypb.Any{TypeUrl: "test"}) + require.NoError(t, err) + spiretest.RequireProtoListEqual(t, []*common.Selector{{Type: "test", Value: "broker-id-ok"}}, selectors) +} + func makeFakeV1Plugin(t *testing.T, selectorValues map[int][]string) workloadattestor.WorkloadAttestor { fake := &fakePluginV1{selectorValues: selectorValues} server := workloadattestorv1.WorkloadAttestorPluginServer(fake) @@ -72,3 +86,22 @@ func (plugin fakePluginV1) Attest(_ context.Context, req *workloadattestorv1.Att SelectorValues: selectorValues, }, nil } + +type fakeReferencePluginV1 struct { + workloadattestorv1.UnimplementedWorkloadAttestorServer + expectedID spiffeid.ID +} + +func (plugin *fakeReferencePluginV1) AttestReference(ctx context.Context, _ *workloadattestorv1.AttestReferenceRequest) (*workloadattestorv1.AttestReferenceResponse, error) { + actualID, ok, err := brokercontext.CallerIDFromContext(ctx) + if err != nil { + return nil, err + } + if !ok { + return nil, status.Error(codes.Internal, "missing broker caller") + } + if actualID != plugin.expectedID { + return nil, status.Errorf(codes.Internal, "got broker caller %q", actualID.String()) + } + return &workloadattestorv1.AttestReferenceResponse{SelectorValues: []string{"broker-id-ok"}}, nil +} diff --git a/pkg/agent/plugin/workloadattestor/windows/windows_windows.go b/pkg/agent/plugin/workloadattestor/windows/windows_windows.go index edfcfebf78..97128f07ab 100644 --- a/pkg/agent/plugin/workloadattestor/windows/windows_windows.go +++ b/pkg/agent/plugin/workloadattestor/windows/windows_windows.go @@ -112,6 +112,13 @@ func (p *Plugin) Attest(_ context.Context, req *workloadattestorv1.AttestRequest }, nil } +// AttestReference returns Unimplemented. This plugin does not handle +// reference-based workload attestation; the host falls back to PID-based +// Attest when the reference is a WorkloadPIDReference. +func (p *Plugin) AttestReference(_ context.Context, _ *workloadattestorv1.AttestReferenceRequest) (*workloadattestorv1.AttestReferenceResponse, error) { + return nil, status.Error(codes.Unimplemented, "AttestReference not implemented") +} + func (p *Plugin) newProcessInfo(pid int32, queryPath bool) (*processInfo, error) { p.log = p.log.With(telemetry.PID, pid) diff --git a/pkg/agent/plugin/workloadattestor/workloadattestor.go b/pkg/agent/plugin/workloadattestor/workloadattestor.go index b17a586866..ba4458f0d9 100644 --- a/pkg/agent/plugin/workloadattestor/workloadattestor.go +++ b/pkg/agent/plugin/workloadattestor/workloadattestor.go @@ -5,10 +5,12 @@ import ( "github.com/spiffe/spire/pkg/common/catalog" "github.com/spiffe/spire/proto/spire/common" + "google.golang.org/protobuf/types/known/anypb" ) type WorkloadAttestor interface { catalog.PluginInfo Attest(ctx context.Context, pid int) ([]*common.Selector, error) + AttestReference(ctx context.Context, reference *anypb.Any) ([]*common.Selector, error) } diff --git a/pkg/common/telemetry/names.go b/pkg/common/telemetry/names.go index 4d71a0f569..35c16cd8b7 100644 --- a/pkg/common/telemetry/names.go +++ b/pkg/common/telemetry/names.go @@ -490,6 +490,9 @@ const ( // Reconfigurable tags whether something is reconfigurable. Reconfigurable = "reconfigurable" + // ReferenceType tags a workload reference type URL + ReferenceType = "reference_type" + // RefreshHint tags a bundle refresh hint RefreshHint = "refresh_hint" @@ -849,6 +852,9 @@ const ( // AuthorizeCall functionality related to authorizing an incoming call AuthorizeCall = "authorize_call" + // BrokerAPI functionality related to broker endpoints + BrokerAPI = "broker_api" + // CreateFederatedBundle functionality related to creating a federated bundle CreateFederatedBundle = "create_federated_bundle" diff --git a/test/integration/setup/brokerclient/build.sh b/test/integration/setup/brokerclient/build.sh new file mode 100755 index 0000000000..f1f5e97960 --- /dev/null +++ b/test/integration/setup/brokerclient/build.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -e + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +cd "${DIR}" && CGO_ENABLED=0 GOOS=linux go build -o "$1" diff --git a/test/integration/setup/brokerclient/main.go b/test/integration/setup/brokerclient/main.go new file mode 100644 index 0000000000..35eecf0543 --- /dev/null +++ b/test/integration/setup/brokerclient/main.go @@ -0,0 +1,168 @@ +// brokerclient is the e2e test driver for the SPIFFE Broker API. It fetches +// its own SVID from the Workload API, then dials the agent's broker endpoint +// with mTLS and exercises the requested scenario (PID reference, Kubernetes +// object reference). It asserts either a specific SPIFFE ID in the response +// or a specific gRPC error code. +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "log" + "time" + + "github.com/spiffe/go-spiffe/v2/exp/proto/spiffe/broker" + "github.com/spiffe/go-spiffe/v2/spiffeid" + "github.com/spiffe/go-spiffe/v2/spiffetls/tlsconfig" + "github.com/spiffe/go-spiffe/v2/workloadapi" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/anypb" +) + +var ( + workloadAPIAddr = flag.String("workload-api", "unix:///run/spire/agent-sockets/api.sock", "Workload API socket URI") + brokerAddr = flag.String("broker-addr", "unix:///run/spire/broker-sockets/broker.sock", "Broker API socket URI") + trustDomain = flag.String("trust-domain", "example.org", "Trust domain to authorize when dialing the broker") + refType = flag.String("ref-type", "", "Reference type: pid|object") + pid = flag.Int("pid", 0, "PID for pid reference") + plural = flag.String("plural", "", "K8s resource plural (e.g. pods, deployments, kustomizations)") + group = flag.String("group", "", "K8s resource group (e.g. core, apps, kustomize.toolkit.fluxcd.io)") + namespace = flag.String("namespace", "", "K8s object namespace") + name = flag.String("name", "", "K8s object name") + uid = flag.String("uid", "", "K8s object UID") + expectedSPIFFE = flag.String("expected-spiffe", "", "Expected SPIFFE ID in the response (omit when expecting an error)") + expectErr = flag.String("expect-err", "", "Expected gRPC code (e.g. PermissionDenied, Unavailable)") + skipBrokerDial = flag.Bool("skip-broker", false, "Only verify Workload API fetches an SVID; do not call the broker API") +) + +func main() { + flag.Parse() + if err := run(); err != nil { + log.Fatalf("brokerclient: %v", err) + } + log.Print("brokerclient: OK") +} + +func run() error { + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + + src, err := workloadapi.NewX509Source(ctx, + workloadapi.WithClientOptions(workloadapi.WithAddr(*workloadAPIAddr)), + ) + if err != nil { + return fmt.Errorf("workload API: %w", err) + } + defer src.Close() + + own, err := src.GetX509SVID() + if err != nil { + return fmt.Errorf("get own SVID: %w", err) + } + log.Printf("own SPIFFE ID: %s", own.ID) + + if *skipBrokerDial { + return nil + } + + req, err := buildRequest() + if err != nil { + return err + } + + td, err := spiffeid.TrustDomainFromString(*trustDomain) + if err != nil { + return fmt.Errorf("parse trust domain: %w", err) + } + tlsCfg := tlsconfig.MTLSClientConfig(src, src, tlsconfig.AuthorizeMemberOf(td)) + + conn, err := grpc.NewClient(*brokerAddr, + grpc.WithTransportCredentials(credentials.NewTLS(tlsCfg)), + ) + if err != nil { + return fmt.Errorf("dial broker: %w", err) + } + defer conn.Close() + + client := broker.NewAPIClient(conn) + ctx = metadata.AppendToOutgoingContext(ctx, "broker.spiffe.io", "true") + + stream, err := client.SubscribeToX509SVID(ctx, req) + if err != nil { + return checkErr(err) + } + resp, err := stream.Recv() + if err != nil { + return checkErr(err) + } + return checkResponse(resp) +} + +func buildRequest() (*broker.SubscribeToX509SVIDRequest, error) { + var packed *anypb.Any + var err error + switch *refType { + case "pid": + packed, err = anypb.New(&broker.WorkloadPIDReference{Pid: int32(*pid)}) + case "object": + ref := &broker.KubernetesObjectReference{ + Type: &broker.KubernetesObjectType{Plural: *plural, Group: *group}, + } + if *namespace != "" || *name != "" { + ref.Key = &broker.KubernetesObjectKey{Namespace: *namespace, Name: *name} + } + if *uid != "" { + ref.Uid = *uid + } + packed, err = anypb.New(ref) + default: + return nil, fmt.Errorf("unknown ref-type %q", *refType) + } + if err != nil { + return nil, fmt.Errorf("packing reference: %w", err) + } + return &broker.SubscribeToX509SVIDRequest{ + Reference: &broker.WorkloadReference{Reference: packed}, + }, nil +} + +func checkErr(err error) error { + code := status.Code(err).String() + if *expectErr != "" { + if code == *expectErr { + log.Printf("got expected error code %s: %v", code, err) + return nil + } + return fmt.Errorf("expected error code %s, got %s (%w)", *expectErr, code, err) + } + return fmt.Errorf("unexpected error (code %s): %w", code, err) +} + +func checkResponse(resp *broker.SubscribeToX509SVIDResponse) error { + if *expectErr != "" { + return fmt.Errorf("expected error %s, but got response with %d SVIDs", *expectErr, len(resp.Svids)) + } + log.Printf("got %d SVIDs in response", len(resp.Svids)) + if *expectedSPIFFE == "" { + if len(resp.Svids) == 0 { + return errors.New("response carried no SVIDs") + } + return nil + } + for _, svid := range resp.Svids { + if svid.SpiffeId == *expectedSPIFFE { + log.Printf("found expected SVID: %s", svid.SpiffeId) + return nil + } + } + got := []string{} + for _, svid := range resp.Svids { + got = append(got, svid.SpiffeId) + } + return fmt.Errorf("expected SVID %s, response had: %v", *expectedSPIFFE, got) +} diff --git a/test/integration/suites/broker-api-k8s/00-setup b/test/integration/suites/broker-api-k8s/00-setup new file mode 100755 index 0000000000..c74edff085 --- /dev/null +++ b/test/integration/suites/broker-api-k8s/00-setup @@ -0,0 +1,27 @@ +#!/bin/bash + +set -e + +mkdir -p ./bin + +KIND_PATH=./bin/kind +KUBECTL_PATH=./bin/kubectl + +download-kind "${KIND_PATH}" +download-kubectl "${KUBECTL_PATH}" + +start-kind-cluster "${KIND_PATH}" broker-api-k8s ./conf/kind-config.yaml + +# Build the brokerclient binary and bake it into a tiny image. +log-info "building brokerclient and broker-client image..." +"${ROOTDIR}/setup/brokerclient/build.sh" "$(pwd)/conf/brokerclient" +docker build -t broker-client:latest-local -f ./conf/broker-client.Dockerfile ./conf + +container_images=( + "spire-server:latest-local" + "spire-agent:latest-local" + "broker-client:latest-local" +) +load-images "${KIND_PATH}" broker-api-k8s "${container_images[@]}" + +set-kubectl-context "${KUBECTL_PATH}" kind-broker-api-k8s diff --git a/test/integration/suites/broker-api-k8s/01-apply-config b/test/integration/suites/broker-api-k8s/01-apply-config new file mode 100755 index 0000000000..3e86c28f56 --- /dev/null +++ b/test/integration/suites/broker-api-k8s/01-apply-config @@ -0,0 +1,63 @@ +#!/bin/bash + +set -e + +source init-kubectl + +wait-for-rollout() { + ns=$1 + obj=$2 + MAXROLLOUTCHECKS=12 + ROLLOUTCHECKINTERVAL=15s + for ((i=0; i<${MAXROLLOUTCHECKS}; i++)); do + log-info "checking rollout status for ${ns} ${obj}..." + if ./bin/kubectl "-n${ns}" rollout status "${obj}" --timeout="${ROLLOUTCHECKINTERVAL}"; then + return + fi + log-warn "describing ${ns} ${obj}..." + ./bin/kubectl "-n${ns}" describe "${obj}" || true + log-warn "logs for ${ns} ${obj}..." + ./bin/kubectl "-n${ns}" logs --all-containers "${obj}" || true + done + fail-now "Failed waiting for ${obj} to roll out." +} + +wait-for-pod-ready() { + ns=$1 + pod=$2 + MAXCHECKS=24 + INTERVAL=5 + for ((i=0; i<${MAXCHECKS}; i++)); do + log-info "checking readiness of ${ns}/${pod}..." + if ./bin/kubectl "-n${ns}" wait --for=condition=Ready "pod/${pod}" --timeout=10s; then + return + fi + sleep "${INTERVAL}" + done + ./bin/kubectl "-n${ns}" describe "pod/${pod}" || true + fail-now "Pod ${ns}/${pod} did not become ready" +} + +./bin/kubectl create namespace spire + +# CRD has to be Established before any Kustomization CR (used by workloads.yaml) is applied. +./bin/kubectl apply -f ./conf/flux-kustomization-crd.yaml +./bin/kubectl wait --for=condition=Established --timeout=60s \ + crd/kustomizations.kustomize.toolkit.fluxcd.io + +./bin/kubectl apply -k ./conf/server +wait-for-rollout spire deployment/spire-server + +./bin/kubectl apply -k ./conf/agent +wait-for-rollout spire daemonset/spire-agent + +./bin/kubectl apply -f ./conf/workloads.yaml +wait-for-pod-ready spire target-pod +wait-for-pod-ready spire same-node-target-pod +wait-for-pod-ready spire other-node-target-pod +wait-for-pod-ready spire broker +wait-for-pod-ready spire cluster-broker +wait-for-pod-ready spire agent-node-broker +wait-for-pod-ready spire restricted-broker +wait-for-pod-ready spire rbac-denied-broker +wait-for-pod-ready spire unauthorized-workload diff --git a/test/integration/suites/broker-api-k8s/02-create-entries b/test/integration/suites/broker-api-k8s/02-create-entries new file mode 100755 index 0000000000..708196281b --- /dev/null +++ b/test/integration/suites/broker-api-k8s/02-create-entries @@ -0,0 +1,57 @@ +#!/bin/bash + +set -e + +source init-kubectl + +readarray -t NODEUIDS < <(./bin/kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.uid}{"\n"}{end}') +PARENTS=() +for nodeUID in "${NODEUIDS[@]}"; do + if [ -n "${nodeUID}" ]; then + PARENTS+=("spiffe://example.org/spire/agent/k8s_psat/example-cluster/${nodeUID}") + fi +done + +if [ "${#PARENTS[@]}" -eq 0 ]; then + fail-now "no Kubernetes nodes found for agent parent IDs" +fi + +create-entry() { + local spiffeID=$1; shift + local parent + for parent in "${PARENTS[@]}"; do + log-info "creating entry ${spiffeID} under ${parent}..." + ./bin/kubectl -nspire exec -t deployment/spire-server -- \ + /opt/spire/bin/spire-server entry create \ + -spiffeID "${spiffeID}" \ + -parentID "${parent}" \ + "$@" + done +} + +# Brokers — registered by their pod-name selectors. +create-entry "spiffe://example.org/broker" -selector "k8s:pod-name:broker" +create-entry "spiffe://example.org/cluster-broker" -selector "k8s:pod-name:cluster-broker" +create-entry "spiffe://example.org/agent-node-broker" -selector "k8s:pod-name:agent-node-broker" +create-entry "spiffe://example.org/restricted-broker" -selector "k8s:pod-name:restricted-broker" +create-entry "spiffe://example.org/rbac-denied-broker" -selector "k8s:pod-name:rbac-denied-broker" +create-entry "spiffe://example.org/unauthorized-workload" -selector "k8s:pod-name:unauthorized-workload" + +# Target pods — matched by the legacy PID path and/or by the +# KubernetesObjectReference(pods/core) path. The k8s attestor emits +# `pod-name` in both cases. +create-entry "spiffe://example.org/target-pod" -selector "k8s:pod-name:target-pod" +create-entry "spiffe://example.org/same-node-target-pod" -selector "k8s:pod-name:same-node-target-pod" +create-entry "spiffe://example.org/other-node-target-pod" -selector "k8s:pod-name:other-node-target-pod" + +# Target Kustomization — matched by the uniform object-meta selector +# vocabulary emitted by getSelectorValuesFromObjectMeta. +create-entry "spiffe://example.org/target-kustomization" \ + -selector "k8s:resource:kustomizations.kustomize.toolkit.fluxcd.io" \ + -selector "k8s:namespace:spire" \ + -selector "k8s:name:target-kustomization" + +# Give the agent a moment to sync entries before the test steps invoke +# brokerclient (the binary will also retry internally via the workloadapi +# source, so this is just a fast-path comfort wait). +sleep 15 diff --git a/test/integration/suites/broker-api-k8s/03-test-broker-by-pid b/test/integration/suites/broker-api-k8s/03-test-broker-by-pid new file mode 100755 index 0000000000..0818500498 --- /dev/null +++ b/test/integration/suites/broker-api-k8s/03-test-broker-by-pid @@ -0,0 +1,32 @@ +#!/bin/bash + +set -e + +source init-kubectl + +log-info "Test: broker fetches target-pod SVID via WorkloadPIDReference" + +# Resolve the target pod's PID from the node (hostPID exposes it). The target +# container runs `spire-agent api watch` as PID 1 inside its namespace, but on +# the host we need its real PID. Get the container's pid via crictl-like +# path: read the pod-uid, look up the pid via /proc on the node. +TARGET_POD_UID=$(./bin/kubectl -nspire get pod target-pod -o jsonpath='{.metadata.uid}') + +# Use the broker pod (which has hostPID) to scan /proc for the container. +# The brokerclient's own PID inside the pod IS the host PID because hostPID +# is true. Easiest: ps -ef on the broker pod and grep for the target command. +TARGET_PID=$(./bin/kubectl -nspire exec -t broker -- sh -c \ + "ps -ef | grep '[s]pire-agent api watch' | awk '{print \$1}' | head -n1") + +if [ -z "${TARGET_PID}" ]; then + fail-now "could not resolve target-pod PID from broker container" +fi +log-info "target-pod PID on host: ${TARGET_PID}" + +./bin/kubectl -nspire exec -t broker -- /brokerclient \ + -ref-type pid \ + -pid "${TARGET_PID}" \ + -expected-spiffe spiffe://example.org/target-pod \ + || fail-now "broker failed to fetch SVID via PID reference" + +log-success "broker fetched target-pod SVID via WorkloadPIDReference" diff --git a/test/integration/suites/broker-api-k8s/04-test-broker-by-pod-obj b/test/integration/suites/broker-api-k8s/04-test-broker-by-pod-obj new file mode 100755 index 0000000000..4449e337d1 --- /dev/null +++ b/test/integration/suites/broker-api-k8s/04-test-broker-by-pod-obj @@ -0,0 +1,16 @@ +#!/bin/bash + +set -e + +source init-kubectl + +log-info "Test: broker fetches target-pod SVID via KubernetesObjectReference (pods/core)" + +./bin/kubectl -nspire exec -t broker -- /brokerclient \ + -ref-type object \ + -plural pods -group core \ + -namespace spire -name target-pod \ + -expected-spiffe spiffe://example.org/target-pod \ + || fail-now "broker failed to fetch SVID via pod object reference" + +log-success "broker fetched target-pod SVID via KubernetesObjectReference" diff --git a/test/integration/suites/broker-api-k8s/04-test-broker-pod-reference-scopes b/test/integration/suites/broker-api-k8s/04-test-broker-pod-reference-scopes new file mode 100755 index 0000000000..bb9b5afeff --- /dev/null +++ b/test/integration/suites/broker-api-k8s/04-test-broker-pod-reference-scopes @@ -0,0 +1,43 @@ +#!/bin/bash + +set -e + +source init-kubectl + +log-info "Test: cluster-scoped broker fetches same-node pod SVID" + +./bin/kubectl -nspire exec -t cluster-broker -- /brokerclient \ + -ref-type object \ + -plural pods -group core \ + -namespace spire -name same-node-target-pod \ + -expected-spiffe spiffe://example.org/same-node-target-pod \ + || fail-now "cluster-scoped broker failed to fetch same-node pod SVID" + +log-info "Test: agent-node-scoped broker fetches same-node pod SVID" + +./bin/kubectl -nspire exec -t agent-node-broker -- /brokerclient \ + -ref-type object \ + -plural pods -group core \ + -namespace spire -name same-node-target-pod \ + -expected-spiffe spiffe://example.org/same-node-target-pod \ + || fail-now "agent-node-scoped broker failed to fetch same-node pod SVID" + +log-info "Test: cluster-scoped broker fetches other-node pod SVID" + +./bin/kubectl -nspire exec -t cluster-broker -- /brokerclient \ + -ref-type object \ + -plural pods -group core \ + -namespace spire -name other-node-target-pod \ + -expected-spiffe spiffe://example.org/other-node-target-pod \ + || fail-now "cluster-scoped broker failed to fetch other-node pod SVID" + +log-info "Test: agent-node-scoped broker cannot fetch other-node pod SVID" + +./bin/kubectl -nspire exec -t agent-node-broker -- /brokerclient \ + -ref-type object \ + -plural pods -group core \ + -namespace spire -name other-node-target-pod \ + -expect-err PermissionDenied \ + || fail-now "agent-node-scoped broker was not denied for other-node pod SVID" + +log-success "pod reference scope scenarios behaved as expected" diff --git a/test/integration/suites/broker-api-k8s/05-test-broker-by-flux-kustomization b/test/integration/suites/broker-api-k8s/05-test-broker-by-flux-kustomization new file mode 100755 index 0000000000..d3f9f9c1b7 --- /dev/null +++ b/test/integration/suites/broker-api-k8s/05-test-broker-by-flux-kustomization @@ -0,0 +1,16 @@ +#!/bin/bash + +set -e + +source init-kubectl + +log-info "Test: broker fetches Flux Kustomization SVID via KubernetesObjectReference" + +./bin/kubectl -nspire exec -t broker -- /brokerclient \ + -ref-type object \ + -plural kustomizations -group kustomize.toolkit.fluxcd.io \ + -namespace spire -name target-kustomization \ + -expected-spiffe spiffe://example.org/target-kustomization \ + || fail-now "broker failed to fetch SVID for Flux Kustomization" + +log-success "broker fetched Flux Kustomization SVID via KubernetesObjectReference" diff --git a/test/integration/suites/broker-api-k8s/06-test-restricted-broker-denied b/test/integration/suites/broker-api-k8s/06-test-restricted-broker-denied new file mode 100755 index 0000000000..b60bb81075 --- /dev/null +++ b/test/integration/suites/broker-api-k8s/06-test-restricted-broker-denied @@ -0,0 +1,33 @@ +#!/bin/bash + +set -e + +source init-kubectl + +log-info "Test: restricted-broker (PID-only allowlist) is denied for object reference" + +./bin/kubectl -nspire exec -t restricted-broker -- /brokerclient \ + -ref-type object \ + -plural pods -group core \ + -namespace spire -name target-pod \ + -expect-err PermissionDenied \ + || fail-now "restricted-broker did not get PermissionDenied for object reference" + +log-success "restricted-broker rejected with PermissionDenied as expected" + +log-info "Test: restricted-broker IS still allowed to use PID reference" + +TARGET_PID=$(./bin/kubectl -nspire exec -t restricted-broker -- sh -c \ + "ps -ef | grep '[s]pire-agent api watch' | awk '{print \$1}' | head -n1") + +if [ -z "${TARGET_PID}" ]; then + fail-now "could not resolve target-pod PID from restricted-broker" +fi + +./bin/kubectl -nspire exec -t restricted-broker -- /brokerclient \ + -ref-type pid \ + -pid "${TARGET_PID}" \ + -expected-spiffe spiffe://example.org/target-pod \ + || fail-now "restricted-broker failed PID reference within its allowlist" + +log-success "restricted-broker successfully used the reference type within its allowlist" diff --git a/test/integration/suites/broker-api-k8s/07-test-broker-rbac-denied b/test/integration/suites/broker-api-k8s/07-test-broker-rbac-denied new file mode 100755 index 0000000000..4cf929ba28 --- /dev/null +++ b/test/integration/suites/broker-api-k8s/07-test-broker-rbac-denied @@ -0,0 +1,33 @@ +#!/bin/bash + +set -e + +source init-kubectl + +log-info "Test: broker allowed to use object references is denied by Kubernetes authorization" + +./bin/kubectl -nspire exec -t rbac-denied-broker -- /brokerclient \ + -ref-type object \ + -plural pods -group core \ + -namespace spire -name target-pod \ + -expect-err PermissionDenied \ + || fail-now "rbac-denied-broker did not get PermissionDenied from Kubernetes authorization" + +log-success "rbac-denied-broker rejected by Kubernetes authorization as expected" + +log-info "Test: broker allowed to use PID references is denied by Kubernetes authorization" + +TARGET_PID=$(./bin/kubectl -nspire exec -t rbac-denied-broker -- sh -c \ + "ps -ef | grep '[s]pire-agent api watch' | awk '{print \$1}' | head -n1") + +if [ -z "${TARGET_PID}" ]; then + fail-now "could not resolve target-pod PID from rbac-denied-broker" +fi + +./bin/kubectl -nspire exec -t rbac-denied-broker -- /brokerclient \ + -ref-type pid \ + -pid "${TARGET_PID}" \ + -expect-err PermissionDenied \ + || fail-now "rbac-denied-broker PID reference did not get PermissionDenied from Kubernetes authorization" + +log-success "rbac-denied-broker PID reference rejected by Kubernetes authorization as expected" diff --git a/test/integration/suites/broker-api-k8s/07-test-unauthorized-workload-rejected b/test/integration/suites/broker-api-k8s/07-test-unauthorized-workload-rejected new file mode 100755 index 0000000000..80fc3e1856 --- /dev/null +++ b/test/integration/suites/broker-api-k8s/07-test-unauthorized-workload-rejected @@ -0,0 +1,26 @@ +#!/bin/bash + +set -e + +source init-kubectl + +log-info "Test: unauthorized workload CAN still fetch its own SVID via Workload API" + +./bin/kubectl -nspire exec -t unauthorized-workload -- /brokerclient \ + -skip-broker \ + || fail-now "unauthorized workload should still be able to use the Workload API" + +log-success "unauthorized workload fetched its own SVID via Workload API" + +log-info "Test: unauthorized workload is REJECTED at mTLS when dialing the broker endpoint" + +# Its SPIFFE ID is not in the agent's broker allowlist, so the server's mTLS +# authorizer fails the cert verification. gRPC surfaces this as Unavailable. +./bin/kubectl -nspire exec -t unauthorized-workload -- /brokerclient \ + -ref-type object \ + -plural pods -group core \ + -namespace spire -name target-pod \ + -expect-err Unavailable \ + || fail-now "unauthorized workload was not rejected at mTLS layer" + +log-success "unauthorized workload rejected at mTLS layer as expected" diff --git a/test/integration/suites/broker-api-k8s/08-test-broker-by-tcp b/test/integration/suites/broker-api-k8s/08-test-broker-by-tcp new file mode 100755 index 0000000000..d46c433d30 --- /dev/null +++ b/test/integration/suites/broker-api-k8s/08-test-broker-by-tcp @@ -0,0 +1,21 @@ +#!/bin/bash + +set -e + +source init-kubectl + +log-info "Test: broker fetches Flux Kustomization SVID via the TCP broker listener" + +# The broker pod normally dials the agent over the local UDS. For this +# step we override -broker-addr to point at the spire-broker ClusterIP +# Service, which fronts the agent's TCP listener on port 8443. mTLS, +# allowlist, and reference-type semantics are identical to the UDS path. +./bin/kubectl -nspire exec -t broker -- /brokerclient \ + -broker-addr dns:///spire-broker:8443 \ + -ref-type object \ + -plural kustomizations -group kustomize.toolkit.fluxcd.io \ + -namespace spire -name target-kustomization \ + -expected-spiffe spiffe://example.org/target-kustomization \ + || fail-now "broker failed to fetch SVID over TCP" + +log-success "broker fetched SVID over TCP via the spire-broker Service" diff --git a/test/integration/suites/broker-api-k8s/09-test-broker-pid-over-tcp-denied b/test/integration/suites/broker-api-k8s/09-test-broker-pid-over-tcp-denied new file mode 100755 index 0000000000..12481a3967 --- /dev/null +++ b/test/integration/suites/broker-api-k8s/09-test-broker-pid-over-tcp-denied @@ -0,0 +1,26 @@ +#!/bin/bash + +set -e + +source init-kubectl + +log-info "Test: PID reference over TCP is denied by allowed_reference_types_over_tcp" + +# The agent's TCP allowlist enumerates only KubernetesObjectReference. A +# PID reference passed over the TCP listener must be rejected with +# PermissionDenied, even though the broker's per-broker allowlist is +# wildcard ("*"). +TARGET_PID=$(./bin/kubectl -nspire exec -t broker -- sh -c \ + "ps -ef | grep '[s]pire-agent api watch' | awk '{print \$1}' | head -n1") + +if [ -z "${TARGET_PID}" ]; then + fail-now "could not resolve target-pod PID from broker" +fi + +./bin/kubectl -nspire exec -t broker -- /brokerclient \ + -broker-addr dns:///spire-broker:8443 \ + -ref-type pid -pid "${TARGET_PID}" \ + -expect-err PermissionDenied \ + || fail-now "PID-over-TCP was not denied" + +log-success "PID reference over TCP rejected with PermissionDenied" diff --git a/test/integration/suites/broker-api-k8s/README.md b/test/integration/suites/broker-api-k8s/README.md new file mode 100644 index 0000000000..94efa30fea --- /dev/null +++ b/test/integration/suites/broker-api-k8s/README.md @@ -0,0 +1,41 @@ +# SPIFFE Broker API (Kubernetes) Suite + +## Description + +Spins up a KIND cluster and exercises the SPIFFE Broker API end-to-end against +real workloads. Covers: + +* A broker fetches a pod workload's SVID via `WorkloadPIDReference` (legacy + PID-based path). +* A broker fetches a pod workload's SVID via `KubernetesObjectReference` + (`pods/core`). +* Cluster-scoped and agent-node-scoped brokers fetch same-node pod SVIDs, while + only the cluster-scoped broker can fetch an other-node pod SVID. +* A broker fetches a **non-pod** object's SVID via `KubernetesObjectReference` + (`kustomizations.kustomize.toolkit.fluxcd.io`) — exercises the generic + object-attestation path that resolves the resource via the REST mapper and + emits the uniform object-meta selector vocabulary. +* A broker whose `allowed_reference_types` is restricted to PID references + gets `PermissionDenied` at the gRPC layer when it asks for a + `KubernetesObjectReference`. +* A broker whose `allowed_reference_types` permits PID and + `KubernetesObjectReference` requests, but whose broker SPIFFE ID lacks + `impersonate-via-spire` authorization, gets `PermissionDenied` from the k8s + attestor's `SubjectAccessReview`. +* A workload that is not in the agent's broker allowlist can still use the + Workload API to fetch its own SVID, but is rejected at the mTLS layer when + it tries to dial the broker endpoint as a broker. +* A broker can also reach the agent over the **TCP** broker listener (via a + ClusterIP Service in front of the agent daemonset), proving the same mTLS + and reference-type semantics apply to remote-style brokers. +* The global `allowed_reference_types_over_tcp` setting denies PID + references over TCP even when the broker's per-broker allowlist would + otherwise permit them — fail-closed against remote use of local-only + reference types. + +Only the Flux Kustomization CRD is installed (no controllers); the resource +just needs to exist in the API server so the broker can reference it. + +The agent RBAC fixture grants `create` on +`subjectaccessreviews.authorization.k8s.io` because the k8s workload attestor +uses SubjectAccessReview API calls to authorize Broker API reference requests. diff --git a/test/integration/suites/broker-api-k8s/conf/agent/kustomization.yaml b/test/integration/suites/broker-api-k8s/conf/agent/kustomization.yaml new file mode 100644 index 0000000000..17d0a0d8e3 --- /dev/null +++ b/test/integration/suites/broker-api-k8s/conf/agent/kustomization.yaml @@ -0,0 +1,10 @@ +# kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# list of Resource Config to be Applied +resources: +- spire-agent.yaml + +# namespace to deploy all Resources to +namespace: spire diff --git a/test/integration/suites/broker-api-k8s/conf/agent/spire-agent.yaml b/test/integration/suites/broker-api-k8s/conf/agent/spire-agent.yaml new file mode 100644 index 0000000000..bac678323c --- /dev/null +++ b/test/integration/suites/broker-api-k8s/conf/agent/spire-agent.yaml @@ -0,0 +1,303 @@ +# ServiceAccount for the SPIRE agent +apiVersion: v1 +kind: ServiceAccount +metadata: + name: spire-agent + namespace: spire + +--- +# Cluster role granting the agent permission to: +# - read pods/nodes for the legacy PID/k8s_psat path +# - read Kustomization CRs so the broker API can attest non-pod object references +# - create SubjectAccessReviews for broker authorization checks +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: spire-agent-cluster-role +rules: + - apiGroups: [""] + resources: ["pods", "nodes", "nodes/proxy"] + verbs: ["get", "list"] + - apiGroups: ["kustomize.toolkit.fluxcd.io"] + resources: ["kustomizations"] + verbs: ["get", "list"] + - apiGroups: ["authorization.k8s.io"] + resources: ["subjectaccessreviews"] + verbs: ["create"] + +--- +# Cluster role used by the k8s workload attestor's SubjectAccessReview check. +# The broker SPIFFE ID is used as the SubjectAccessReview username. +# Use SPIRE's custom verb, not Kubernetes' built-in `impersonate` verb. +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: spire-broker-impersonation-role +rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["impersonate-via-spire"] + - apiGroups: ["kustomize.toolkit.fluxcd.io"] + resources: ["kustomizations"] + verbs: ["impersonate-via-spire"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: spire-broker-impersonation-role-binding +subjects: + - kind: User + name: spiffe://example.org/broker + - kind: User + name: spiffe://example.org/cluster-broker + - kind: User + name: spiffe://example.org/agent-node-broker + - kind: User + name: spiffe://example.org/restricted-broker +roleRef: + kind: ClusterRole + name: spire-broker-impersonation-role + apiGroup: rbac.authorization.k8s.io + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: spire-agent-cluster-role-binding +subjects: + - kind: ServiceAccount + name: spire-agent + namespace: spire +roleRef: + kind: ClusterRole + name: spire-agent-cluster-role + apiGroup: rbac.authorization.k8s.io + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: spire-agent + namespace: spire +data: + agent.conf: | + agent { + data_dir = "/run/spire" + log_level = "DEBUG" + server_address = "spire-server" + server_port = "8081" + socket_path = "/run/spire/agent-sockets/api.sock" + trust_bundle_path = "/run/spire/bundle/bundle.crt" + trust_domain = "example.org" + + experimental { + # Enable the SPIFFE Broker API endpoint on both a UDS (for local + # brokers) and a TCP listener (for remote brokers). The TCP port + # is fronted by the spire-broker Service. Each authorized broker + # presents a SPIFFE ID; mTLS at the listener gates the set. + broker { + socket_path = "/run/spire/broker-sockets/broker.sock" + bind_address = "0.0.0.0:8443" + # Fail-closed for TCP except KubernetesObjectReference (so the + # TCP test in this suite succeeds while PID-over-TCP is denied). + allowed_reference_types_over_tcp = [ + "type.googleapis.com/spiffe.broker.KubernetesObjectReference", + ] + brokers = [ + { + id = "spiffe://example.org/broker" + allowed_reference_types = ["*"] + }, + { + id = "spiffe://example.org/cluster-broker" + allowed_reference_types = [ + "type.googleapis.com/spiffe.broker.KubernetesObjectReference", + ] + }, + { + id = "spiffe://example.org/agent-node-broker" + allowed_reference_types = [ + "type.googleapis.com/spiffe.broker.KubernetesObjectReference", + ] + }, + { + id = "spiffe://example.org/restricted-broker" + allowed_reference_types = [ + "type.googleapis.com/spiffe.broker.WorkloadPIDReference", + ] + }, + { + id = "spiffe://example.org/rbac-denied-broker" + allowed_reference_types = [ + "type.googleapis.com/spiffe.broker.WorkloadPIDReference", + "type.googleapis.com/spiffe.broker.KubernetesObjectReference", + ] + }, + ] + } + } + } + + plugins { + NodeAttestor "k8s_psat" { + plugin_data { + cluster = "example-cluster" + } + } + + KeyManager "memory" { + plugin_data { + } + } + + WorkloadAttestor "k8s" { + plugin_data { + # Skip kubelet cert verification (kind doesn't ship a CA bundle entry for it). + skip_kubelet_verification = true + node_name_env = "MY_NODE_NAME" + + api_server_cache { + enabled = true + } + + # Broker-specific Kubernetes authorization config. + # The broker SPIFFE ID is used as the SubjectAccessReview username. + broker { + brokers = [ + { + id = "spiffe://example.org/broker" + }, + { + id = "spiffe://example.org/cluster-broker" + pod_reference_scope = "cluster" + }, + { + id = "spiffe://example.org/agent-node-broker" + pod_reference_scope = "agent_node" + }, + { + id = "spiffe://example.org/restricted-broker" + }, + { + id = "spiffe://example.org/rbac-denied-broker" + }, + ] + } + } + } + } + + health_checks { + listener_enabled = true + bind_address = "0.0.0.0" + bind_port = "8080" + live_path = "/live" + ready_path = "/ready" + } + +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: spire-agent + namespace: spire + labels: + app: spire-agent +spec: + selector: + matchLabels: + app: spire-agent + updateStrategy: + type: RollingUpdate + template: + metadata: + namespace: spire + labels: + app: spire-agent + spec: + hostPID: true + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: spire-agent + containers: + - name: spire-agent + image: spire-agent:latest-local + imagePullPolicy: Never + args: ["-config", "/run/spire/config/agent.conf"] + env: + - name: MY_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + ports: + - name: broker + containerPort: 8443 + volumeMounts: + - name: spire-config + mountPath: /run/spire/config + readOnly: true + - name: spire-bundle + mountPath: /run/spire/bundle + readOnly: true + - name: spire-agent-socket + mountPath: /run/spire/agent-sockets + readOnly: false + - name: spire-broker-socket + mountPath: /run/spire/broker-sockets + readOnly: false + - name: spire-token + mountPath: /var/run/secrets/tokens + livenessProbe: + httpGet: + path: /live + port: 8080 + initialDelaySeconds: 10 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /ready + port: 8080 + initialDelaySeconds: 10 + periodSeconds: 10 + volumes: + - name: spire-config + configMap: + name: spire-agent + - name: spire-bundle + configMap: + name: spire-bundle + - name: spire-agent-socket + hostPath: + path: /run/spire/agent-sockets + type: DirectoryOrCreate + - name: spire-broker-socket + hostPath: + path: /run/spire/broker-sockets + type: DirectoryOrCreate + - name: spire-token + projected: + sources: + - serviceAccountToken: + path: spire-agent + expirationSeconds: 7200 + audience: spire-server + +--- +# Fronts the agent's broker TCP listener so remote-style brokers can reach +# it without depending on hostNetwork or the node IP. Targets the daemonset +# pods directly via the `app=spire-agent` selector. +apiVersion: v1 +kind: Service +metadata: + name: spire-broker + namespace: spire +spec: + type: ClusterIP + selector: + app: spire-agent + ports: + - name: broker + port: 8443 + targetPort: 8443 + protocol: TCP diff --git a/test/integration/suites/broker-api-k8s/conf/broker-client.Dockerfile b/test/integration/suites/broker-api-k8s/conf/broker-client.Dockerfile new file mode 100644 index 0000000000..dd7a86eca2 --- /dev/null +++ b/test/integration/suites/broker-api-k8s/conf/broker-client.Dockerfile @@ -0,0 +1,5 @@ +# Tiny image that just carries the brokerclient binary built on the host. +# We base on busybox so `sleep infinity` is available; the suite scripts +# `kubectl exec` into the pod and invoke /brokerclient with per-test flags. +FROM busybox:1.37 +COPY brokerclient /brokerclient diff --git a/test/integration/suites/broker-api-k8s/conf/flux-kustomization-crd.yaml b/test/integration/suites/broker-api-k8s/conf/flux-kustomization-crd.yaml new file mode 100644 index 0000000000..102c9a3117 --- /dev/null +++ b/test/integration/suites/broker-api-k8s/conf/flux-kustomization-crd.yaml @@ -0,0 +1,25 @@ +# Minimal stand-in for the Flux source-controller Kustomization CRD. +# The broker API doesn't need the Flux controller running — only the CRD has +# to be registered so the agent's REST mapper can resolve +# `kustomizations.kustomize.toolkit.fluxcd.io` and fetch PartialObjectMetadata. +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: kustomizations.kustomize.toolkit.fluxcd.io +spec: + group: kustomize.toolkit.fluxcd.io + scope: Namespaced + names: + kind: Kustomization + plural: kustomizations + singular: kustomization + shortNames: + - ks + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true diff --git a/test/integration/suites/broker-api-k8s/conf/kind-config.yaml b/test/integration/suites/broker-api-k8s/conf/kind-config.yaml new file mode 100644 index 0000000000..ae8cd3041f --- /dev/null +++ b/test/integration/suites/broker-api-k8s/conf/kind-config.yaml @@ -0,0 +1,6 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: + - role: control-plane + - role: worker + - role: worker diff --git a/test/integration/suites/broker-api-k8s/conf/server/kustomization.yaml b/test/integration/suites/broker-api-k8s/conf/server/kustomization.yaml new file mode 100644 index 0000000000..61ec1abdc4 --- /dev/null +++ b/test/integration/suites/broker-api-k8s/conf/server/kustomization.yaml @@ -0,0 +1,10 @@ +# kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# list of Resource Config to be Applied +resources: +- spire-server.yaml + +# namespace to deploy all Resources to +namespace: spire diff --git a/test/integration/suites/broker-api-k8s/conf/server/spire-server.yaml b/test/integration/suites/broker-api-k8s/conf/server/spire-server.yaml new file mode 100644 index 0000000000..bad6a2a7b0 --- /dev/null +++ b/test/integration/suites/broker-api-k8s/conf/server/spire-server.yaml @@ -0,0 +1,235 @@ +# ServiceAccount used by the SPIRE server. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: spire-server + namespace: spire + +--- + +# Required cluster role to allow spire-server to query k8s API server +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: spire-server-cluster-role +rules: +- apiGroups: [""] + resources: ["pods", "nodes"] + verbs: ["get", "list", "watch"] + # allow TokenReview requests (to verify service account tokens for PSAT + # attestation) +- apiGroups: ["authentication.k8s.io"] + resources: ["tokenreviews"] + verbs: ["get", "create"] + +--- + +# Binds above cluster role to spire-server service account +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: spire-server-cluster-role-binding + namespace: spire +subjects: +- kind: ServiceAccount + name: spire-server + namespace: spire +roleRef: + kind: ClusterRole + name: spire-server-cluster-role + apiGroup: rbac.authorization.k8s.io + +--- + +# Role for the SPIRE server +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + namespace: spire + name: spire-server-role +rules: + # allow "get" access to pods (to resolve selectors for PSAT attestation) +- apiGroups: [""] + resources: ["pods"] + verbs: ["get"] + # allow access to "get" and "patch" the spire-bundle ConfigMap (for SPIRE + # agent bootstrapping, see the spire-bundle ConfigMap below) +- apiGroups: [""] + resources: ["configmaps"] + resourceNames: ["spire-bundle"] + verbs: ["get", "patch"] +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["create"] +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["create", "update", "get"] +- apiGroups: [""] + resources: ["events"] + verbs: ["create"] + +--- + +# RoleBinding granting the spire-server-role to the SPIRE server +# service account. +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: spire-server-role-binding + namespace: spire +subjects: +- kind: ServiceAccount + name: spire-server + namespace: spire +roleRef: + kind: Role + name: spire-server-role + apiGroup: rbac.authorization.k8s.io + +--- + +# ConfigMap containing the latest trust bundle for the trust domain. It is +# updated by SPIRE using the k8sbundle notifier plugin. SPIRE agents mount +# this config map and use the certificate to bootstrap trust with the SPIRE +# server during attestation. +apiVersion: v1 +kind: ConfigMap +metadata: + name: spire-bundle + namespace: spire + +--- + +# ConfigMap containing the SPIRE server configuration. +apiVersion: v1 +kind: ConfigMap +metadata: + name: spire-server + namespace: spire +data: + server.conf: | + server { + bind_address = "0.0.0.0" + bind_port = "8081" + trust_domain = "example.org" + data_dir = "/run/spire/data" + log_level = "DEBUG" + default_x509_svid_ttl = "1h" + ca_ttl = "12h" + ca_subject { + country = ["US"] + organization = ["SPIFFE"] + common_name = "" + } + } + + plugins { + DataStore "sql" { + plugin_data { + database_type = "sqlite3" + connection_string = "/run/spire/data/datastore.sqlite3" + } + } + + NodeAttestor "k8s_psat" { + plugin_data { + clusters = { + "example-cluster" = { + service_account_allow_list = ["spire:spire-agent"] + } + } + } + } + + KeyManager "disk" { + plugin_data { + keys_path = "/run/spire/data/keys.json" + } + } + + Notifier "k8sbundle" { + plugin_data { + # This plugin updates the bundle.crt value in the spire:spire-bundle + # ConfigMap by default, so no additional configuration is necessary. + } + } + } + + health_checks { + listener_enabled = true + bind_address = "0.0.0.0" + bind_port = "8080" + live_path = "/live" + ready_path = "/ready" + } + +--- + +# This is the Deployment for the SPIRE server. It waits for SPIRE database to +# initialize and uses the SPIRE healthcheck command for liveness/readiness +# probes. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: spire-server + namespace: spire + labels: + app: spire-server +spec: + replicas: 1 + selector: + matchLabels: + app: spire-server + template: + metadata: + namespace: spire + labels: + app: spire-server + spec: + serviceAccountName: spire-server + shareProcessNamespace: true + containers: + - name: spire-server + image: spire-server:latest-local + imagePullPolicy: Never + args: ["-config", "/run/spire/config/server.conf"] + ports: + - containerPort: 8081 + volumeMounts: + - name: spire-config + mountPath: /run/spire/config + readOnly: true + livenessProbe: + httpGet: + path: /live + port: 8080 + initialDelaySeconds: 5 + periodSeconds: 5 + readinessProbe: + httpGet: + path: /ready + port: 8080 + initialDelaySeconds: 5 + periodSeconds: 5 + volumes: + - name: spire-config + configMap: + name: spire-server + +--- + +# Service definition for SPIRE server defining the gRPC port. +apiVersion: v1 +kind: Service +metadata: + name: spire-server + namespace: spire +spec: + type: NodePort + ports: + - name: grpc + port: 8081 + targetPort: 8081 + protocol: TCP + selector: + app: spire-server diff --git a/test/integration/suites/broker-api-k8s/conf/workloads.yaml b/test/integration/suites/broker-api-k8s/conf/workloads.yaml new file mode 100644 index 0000000000..d3f0731ffa --- /dev/null +++ b/test/integration/suites/broker-api-k8s/conf/workloads.yaml @@ -0,0 +1,291 @@ +# The "target" workload — a regular pod the broker(s) will fetch SVIDs for +# via PID reference and via KubernetesObjectReference(pods/core). +# It runs `spire-agent api watch` so it stays alive with a steady PID. +apiVersion: v1 +kind: Pod +metadata: + name: target-pod + namespace: spire + labels: + app: target-pod +spec: + hostPID: true + nodeName: broker-api-k8s-worker + containers: + - name: target + image: spire-agent:latest-local + imagePullPolicy: Never + command: ["/opt/spire/bin/spire-agent", "api", "watch"] + args: ["-socketPath", "/run/spire/agent-sockets/api.sock"] + volumeMounts: + - name: spire-agent-socket + mountPath: /run/spire/agent-sockets + readOnly: false + volumes: + - name: spire-agent-socket + hostPath: + path: /run/spire/agent-sockets + type: Directory + +--- +# Pod on the same node as the scoped broker pods. It only needs to +# exist for KubernetesObjectReference lookups. +apiVersion: v1 +kind: Pod +metadata: + name: same-node-target-pod + namespace: spire + labels: + app: same-node-target-pod +spec: + nodeName: broker-api-k8s-worker + containers: + - name: target + image: broker-client:latest-local + imagePullPolicy: Never + command: ["/bin/sleep", "infinity"] + +--- +# Pod on a different node than the scoped broker pods. It only needs to +# exist for KubernetesObjectReference lookups. +apiVersion: v1 +kind: Pod +metadata: + name: other-node-target-pod + namespace: spire + labels: + app: other-node-target-pod +spec: + nodeName: broker-api-k8s-worker2 + containers: + - name: target + image: broker-client:latest-local + imagePullPolicy: Never + command: ["/bin/sleep", "infinity"] + +--- +# The "target" non-pod object — a Flux Kustomization CR. No controller is +# required; the resource just needs to exist in the API server so the broker +# can reference it. +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: target-kustomization + namespace: spire +spec: + interval: 1m + path: ./apps + prune: true + sourceRef: + kind: GitRepository + name: dummy + +--- +# The "broker" — authorized in the agent config to use any reference type. +apiVersion: v1 +kind: Pod +metadata: + name: broker + namespace: spire + labels: + app: broker +spec: + hostPID: true + nodeName: broker-api-k8s-worker + containers: + - name: broker + image: broker-client:latest-local + imagePullPolicy: Never + command: ["/bin/sleep", "infinity"] + volumeMounts: + - name: spire-agent-socket + mountPath: /run/spire/agent-sockets + readOnly: false + - name: spire-broker-socket + mountPath: /run/spire/broker-sockets + readOnly: false + volumes: + - name: spire-agent-socket + hostPath: + path: /run/spire/agent-sockets + type: Directory + - name: spire-broker-socket + hostPath: + path: /run/spire/broker-sockets + type: Directory + +--- +# Broker whose k8s attestor pod reference scope is `cluster`. +apiVersion: v1 +kind: Pod +metadata: + name: cluster-broker + namespace: spire + labels: + app: cluster-broker +spec: + hostPID: true + nodeName: broker-api-k8s-worker + containers: + - name: cluster-broker + image: broker-client:latest-local + imagePullPolicy: Never + command: ["/bin/sleep", "infinity"] + volumeMounts: + - name: spire-agent-socket + mountPath: /run/spire/agent-sockets + readOnly: false + - name: spire-broker-socket + mountPath: /run/spire/broker-sockets + readOnly: false + volumes: + - name: spire-agent-socket + hostPath: + path: /run/spire/agent-sockets + type: Directory + - name: spire-broker-socket + hostPath: + path: /run/spire/broker-sockets + type: Directory + +--- +# Broker whose k8s attestor pod reference scope is `agent_node`. +apiVersion: v1 +kind: Pod +metadata: + name: agent-node-broker + namespace: spire + labels: + app: agent-node-broker +spec: + hostPID: true + nodeName: broker-api-k8s-worker + containers: + - name: agent-node-broker + image: broker-client:latest-local + imagePullPolicy: Never + command: ["/bin/sleep", "infinity"] + volumeMounts: + - name: spire-agent-socket + mountPath: /run/spire/agent-sockets + readOnly: false + - name: spire-broker-socket + mountPath: /run/spire/broker-sockets + readOnly: false + volumes: + - name: spire-agent-socket + hostPath: + path: /run/spire/agent-sockets + type: Directory + - name: spire-broker-socket + hostPath: + path: /run/spire/broker-sockets + type: Directory + +--- +# The "restricted broker" — authorized in the agent config but limited to +# WorkloadPIDReference only. KubernetesObjectReference attempts should be +# rejected with PermissionDenied at the gRPC layer. +apiVersion: v1 +kind: Pod +metadata: + name: restricted-broker + namespace: spire + labels: + app: restricted-broker +spec: + hostPID: true + nodeName: broker-api-k8s-worker + containers: + - name: restricted-broker + image: broker-client:latest-local + imagePullPolicy: Never + command: ["/bin/sleep", "infinity"] + volumeMounts: + - name: spire-agent-socket + mountPath: /run/spire/agent-sockets + readOnly: false + - name: spire-broker-socket + mountPath: /run/spire/broker-sockets + readOnly: false + volumes: + - name: spire-agent-socket + hostPath: + path: /run/spire/agent-sockets + type: Directory + - name: spire-broker-socket + hostPath: + path: /run/spire/broker-sockets + type: Directory + +--- +# The "RBAC denied broker" is authorized by the Broker API config to use +# KubernetesObjectReference, but its broker SPIFFE ID is not allowed to use +# `impersonate-via-spire` by the Kubernetes authorizer. +apiVersion: v1 +kind: Pod +metadata: + name: rbac-denied-broker + namespace: spire + labels: + app: rbac-denied-broker +spec: + hostPID: true + nodeName: broker-api-k8s-worker + containers: + - name: rbac-denied-broker + image: broker-client:latest-local + imagePullPolicy: Never + command: ["/bin/sleep", "infinity"] + volumeMounts: + - name: spire-agent-socket + mountPath: /run/spire/agent-sockets + readOnly: false + - name: spire-broker-socket + mountPath: /run/spire/broker-sockets + readOnly: false + volumes: + - name: spire-agent-socket + hostPath: + path: /run/spire/agent-sockets + type: Directory + - name: spire-broker-socket + hostPath: + path: /run/spire/broker-sockets + type: Directory + +--- +# The "unauthorized workload" — gets its own SVID via the Workload API just +# fine, but isn't in the agent's broker allowlist. Attempts to dial the +# broker endpoint should fail at the mTLS layer. +apiVersion: v1 +kind: Pod +metadata: + name: unauthorized-workload + namespace: spire + labels: + app: unauthorized-workload +spec: + hostPID: true + nodeName: broker-api-k8s-worker + containers: + - name: unauthorized-workload + image: broker-client:latest-local + imagePullPolicy: Never + command: ["/bin/sleep", "infinity"] + volumeMounts: + - name: spire-agent-socket + mountPath: /run/spire/agent-sockets + readOnly: false + - name: spire-broker-socket + mountPath: /run/spire/broker-sockets + readOnly: false + volumes: + - name: spire-agent-socket + hostPath: + path: /run/spire/agent-sockets + type: Directory + - name: spire-broker-socket + hostPath: + path: /run/spire/broker-sockets + type: Directory diff --git a/test/integration/suites/broker-api-k8s/init-kubectl b/test/integration/suites/broker-api-k8s/init-kubectl new file mode 100755 index 0000000000..2b7d453a06 --- /dev/null +++ b/test/integration/suites/broker-api-k8s/init-kubectl @@ -0,0 +1,8 @@ +#!/bin/bash + +KUBECONFIG="${RUNDIR}/kubeconfig" +if [ ! -f "${RUNDIR}/kubeconfig" ]; then + ./bin/kind get kubeconfig --name=broker-api-k8s > "${RUNDIR}/kubeconfig" +fi +export KUBECONFIG + diff --git a/test/integration/suites/broker-api-k8s/teardown b/test/integration/suites/broker-api-k8s/teardown new file mode 100755 index 0000000000..a14acc0b23 --- /dev/null +++ b/test/integration/suites/broker-api-k8s/teardown @@ -0,0 +1,15 @@ +#!/bin/bash + +source init-kubectl + +if [ -z "$SUCCESS" ]; then + ./bin/kubectl -nspire logs deployment/spire-server --all-containers || true + ./bin/kubectl -nspire logs daemonset/spire-agent --all-containers || true + for pod in broker cluster-broker agent-node-broker restricted-broker rbac-denied-broker unauthorized-workload target-pod same-node-target-pod other-node-target-pod; do + ./bin/kubectl -nspire logs "pod/${pod}" --all-containers || true + ./bin/kubectl -nspire describe "pod/${pod}" || true + done +fi + +export KUBECONFIG= +./bin/kind delete cluster --name broker-api-k8s