From cef78bb9e42ad0cd69caa9f90a136715fb786d0c Mon Sep 17 00:00:00 2001 From: Matheus Pimenta Date: Sat, 4 Jul 2026 16:12:27 +0100 Subject: [PATCH] Support agent deployments for only Broker API Signed-off-by: Matheus Pimenta --- cmd/spire-agent/cli/run/run.go | 86 ++++++++++++------- cmd/spire-agent/cli/run/run_posix.go | 42 +++++---- cmd/spire-agent/cli/run/run_posix_test.go | 46 ++++++++++ cmd/spire-agent/cli/run/run_test.go | 41 +++++++++ conf/agent/agent_full.conf | 8 ++ doc/plugin_agent_nodeattestor_k8s_psat.md | 8 +- doc/plugin_server_nodeattestor_k8s_psat.md | 9 +- doc/spire_agent.md | 29 ++++++- pkg/agent/agent.go | 41 +++++++-- pkg/agent/agent_test.go | 28 ++++++ pkg/agent/broker/endpoints.go | 23 ++++- pkg/agent/config.go | 2 +- .../plugin/nodeattestor/k8spsat/psat.go | 12 ++- .../plugin/nodeattestor/k8spsat/psat_test.go | 61 +++++++++++-- 14 files changed, 361 insertions(+), 75 deletions(-) create mode 100644 pkg/agent/agent_test.go diff --git a/cmd/spire-agent/cli/run/run.go b/cmd/spire-agent/cli/run/run.go index b5165ced04..310adf8683 100644 --- a/cmd/spire-agent/cli/run/run.go +++ b/cmd/spire-agent/cli/run/run.go @@ -67,33 +67,34 @@ type Config struct { } type agentConfig struct { - DataDir string `hcl:"data_dir"` - AdminSocketPath string `hcl:"admin_socket_path"` - InsecureBootstrap bool `hcl:"insecure_bootstrap"` - RebootstrapMode string `hcl:"rebootstrap_mode"` - RebootstrapDelay string `hcl:"rebootstrap_delay"` - JoinToken string `hcl:"join_token"` - JoinTokenFile string `hcl:"join_token_file"` - LogFile string `hcl:"log_file"` - LogFormat string `hcl:"log_format"` - LogLevel string `hcl:"log_level"` - LogSelectors []string `hcl:"log_selectors"` - LogSourceLocation bool `hcl:"log_source_location"` - SDS sdsConfig `hcl:"sds"` - ServerAddress string `hcl:"server_address"` - ServerPort int `hcl:"server_port"` - SocketPath string `hcl:"socket_path"` - WorkloadX509SVIDKeyType string `hcl:"workload_x509_svid_key_type"` - TrustBundleFormat string `hcl:"trust_bundle_format"` - TrustBundlePath string `hcl:"trust_bundle_path"` - TrustBundleUnixSocket string `hcl:"trust_bundle_unix_socket"` - TrustBundleURL string `hcl:"trust_bundle_url"` - TrustDomain string `hcl:"trust_domain"` - AllowUnauthenticatedVerifiers bool `hcl:"allow_unauthenticated_verifiers"` - AllowedForeignJWTClaims []string `hcl:"allowed_foreign_jwt_claims"` - AvailabilityTarget string `hcl:"availability_target"` - X509SVIDCacheMaxSize int `hcl:"x509_svid_cache_max_size"` - JWTSVIDCacheMaxSize int `hcl:"jwt_svid_cache_max_size"` + DataDir string `hcl:"data_dir"` + AdminSocketPath string `hcl:"admin_socket_path"` + InsecureBootstrap bool `hcl:"insecure_bootstrap"` + RebootstrapMode string `hcl:"rebootstrap_mode"` + RebootstrapDelay string `hcl:"rebootstrap_delay"` + JoinToken string `hcl:"join_token"` + JoinTokenFile string `hcl:"join_token_file"` + LogFile string `hcl:"log_file"` + LogFormat string `hcl:"log_format"` + LogLevel string `hcl:"log_level"` + LogSelectors []string `hcl:"log_selectors"` + LogSourceLocation bool `hcl:"log_source_location"` + SDS sdsConfig `hcl:"sds"` + ServerAddress string `hcl:"server_address"` + ServerPort int `hcl:"server_port"` + SocketPath string `hcl:"socket_path"` + WorkloadX509SVIDKeyType string `hcl:"workload_x509_svid_key_type"` + TrustBundleFormat string `hcl:"trust_bundle_format"` + TrustBundlePath string `hcl:"trust_bundle_path"` + TrustBundleUnixSocket string `hcl:"trust_bundle_unix_socket"` + TrustBundleURL string `hcl:"trust_bundle_url"` + TrustDomain string `hcl:"trust_domain"` + AllowUnauthenticatedVerifiers bool `hcl:"allow_unauthenticated_verifiers"` + AllowedForeignJWTClaims []string `hcl:"allowed_foreign_jwt_claims"` + AvailabilityTarget string `hcl:"availability_target"` + X509SVIDCacheMaxSize int `hcl:"x509_svid_cache_max_size"` + JWTSVIDCacheMaxSize int `hcl:"jwt_svid_cache_max_size"` + WorkloadAPI *workloadAPIConfig `hcl:"workload_api"` AuthorizedDelegates []string `hcl:"authorized_delegates"` @@ -228,6 +229,15 @@ type sdsConfig struct { DisableSPIFFECertValidation bool `hcl:"disable_spiffe_cert_validation"` } +type workloadAPIConfig struct { + // Enabled controls whether the agent serves the Workload API and SDS + // endpoint. It defaults to true. Setting it to false requires the Broker + // API endpoint to be configured. + Enabled *bool `hcl:"enabled"` + + UnusedKeyPositions map[string][]token.Pos `hcl:",unusedKeyPositions"` +} + type workloadAPIRateLimitConfig struct { FetchX509SVID *int `hcl:"fetch_x509_svid"` FetchJWTSVID *int `hcl:"fetch_jwt_svid"` @@ -359,6 +369,10 @@ func (c *agentConfig) validate() error { return errors.New("agent section must be configured") } + if !c.workloadAPIEnabled() && c.Experimental.Broker == nil { + return errors.New("workload_api.enabled=false requires experimental.broker to be configured") + } + // Validate join token configuration if c.JoinToken != "" && c.JoinTokenFile != "" { return errors.New("only one of join_token or join_token_file can be specified, not both") @@ -431,6 +445,10 @@ func (c *agentConfig) validate() error { return c.validateOS() } +func (c *agentConfig) workloadAPIEnabled() bool { + return c.WorkloadAPI == nil || c.WorkloadAPI.Enabled == nil || *c.WorkloadAPI.Enabled +} + func ParseFile(path string, expandEnv bool) (*Config, error) { c := &Config{} @@ -632,11 +650,13 @@ func NewAgentConfig(c *Config, logOptions []log.Option, allowUnknownConfig bool) } ac.TrustDomain = td - addr, err := c.Agent.getAddr() - if err != nil { - return nil, err + if c.Agent.workloadAPIEnabled() { + addr, err := c.Agent.getAddr() + if err != nil { + return nil, err + } + ac.BindAddress = addr } - ac.BindAddress = addr if c.Agent.hasAdminAddr() { adminAddr, err := c.Agent.getAdminAddr() @@ -856,6 +876,10 @@ func checkForUnknownConfig(c *Config, l logrus.FieldLogger) (err error) { detectedUnknown("agent", a.UnusedKeyPositions) } + if a := c.Agent; a != nil && a.WorkloadAPI != nil && len(a.WorkloadAPI.UnusedKeyPositions) != 0 { + detectedUnknown("agent.workload_api", a.WorkloadAPI.UnusedKeyPositions) + } + if a := c.Agent; a != nil && a.Experimental.Broker != nil { if len(a.Experimental.Broker.UnusedKeyPositions) != 0 { detectedUnknown("experimental.broker", a.Experimental.Broker.UnusedKeyPositions) diff --git a/cmd/spire-agent/cli/run/run_posix.go b/cmd/spire-agent/cli/run/run_posix.go index 8a63a48e97..0cc4aa1eef 100644 --- a/cmd/spire-agent/cli/run/run_posix.go +++ b/cmd/spire-agent/cli/run/run_posix.go @@ -30,17 +30,19 @@ func (c *agentConfig) getAddr() (net.Addr, error) { } func (c *agentConfig) getAdminAddr() (net.Addr, error) { - socketPathAbs, err := filepath.Abs(c.SocketPath) - if err != nil { - return nil, fmt.Errorf("failed to get absolute path for socket_path: %w", err) - } adminSocketPathAbs, err := filepath.Abs(c.AdminSocketPath) if err != nil { return nil, fmt.Errorf("failed to get absolute path for admin_socket_path: %w", err) } - if strings.HasPrefix(adminSocketPathAbs, filepath.Dir(socketPathAbs)+"/") { - return nil, errors.New("admin socket cannot be in the same directory or a subdirectory as that containing the Workload API socket") + if c.workloadAPIEnabled() { + socketPathAbs, err := filepath.Abs(c.SocketPath) + if err != nil { + return nil, fmt.Errorf("failed to get absolute path for socket_path: %w", err) + } + if strings.HasPrefix(adminSocketPathAbs, filepath.Dir(socketPathAbs)+"/") { + return nil, errors.New("admin socket cannot be in the same directory or a subdirectory as that containing the Workload API socket") + } } return &net.UnixAddr{ @@ -56,17 +58,19 @@ func (c *agentConfig) hasAdminAddr() bool { // brokerSocketAddr resolves the UDS branch of broker bind-address selection. // Returns the platform-agnostic TCP branch is handled in brokerBindAddr. func (c *agentConfig) brokerSocketAddr() (net.Addr, error) { - socketPathAbs, err := filepath.Abs(c.SocketPath) - if err != nil { - return nil, fmt.Errorf("failed to get absolute path for socket_path: %w", err) - } brokerSocketPathAbs, err := filepath.Abs(c.Experimental.Broker.SocketPath) if err != nil { return nil, fmt.Errorf("failed to get absolute path for experimental.broker.socket_path: %w", err) } - if strings.HasPrefix(brokerSocketPathAbs, filepath.Dir(socketPathAbs)+"/") { - return nil, errors.New("broker socket cannot be in the same directory or a subdirectory as that containing the Workload API socket") + if c.workloadAPIEnabled() { + socketPathAbs, err := filepath.Abs(c.SocketPath) + if err != nil { + return nil, fmt.Errorf("failed to get absolute path for socket_path: %w", err) + } + if strings.HasPrefix(brokerSocketPathAbs, filepath.Dir(socketPathAbs)+"/") { + return nil, errors.New("broker socket cannot be in the same directory or a subdirectory as that containing the Workload API socket") + } } return &net.UnixAddr{ @@ -87,12 +91,14 @@ func (c *agentConfig) validateOS() error { } func prepareEndpoints(c *agent.Config) error { - // Create uds dir and parents if not exists - dir := filepath.Dir(c.BindAddress.String()) - if _, statErr := os.Stat(dir); os.IsNotExist(statErr) { - c.Log.WithField("dir", dir).Infof("Creating spire agent UDS directory") - if err := os.MkdirAll(dir, 0755); err != nil { - return err + if c.BindAddress != nil { + // Create uds dir and parents if not exists + dir := filepath.Dir(c.BindAddress.String()) + if _, statErr := os.Stat(dir); os.IsNotExist(statErr) { + c.Log.WithField("dir", dir).Infof("Creating spire agent UDS directory") + if err := os.MkdirAll(dir, 0755); err != nil { + return err + } } } diff --git a/cmd/spire-agent/cli/run/run_posix_test.go b/cmd/spire-agent/cli/run/run_posix_test.go index 8767f9a681..b1ba315569 100644 --- a/cmd/spire-agent/cli/run/run_posix_test.go +++ b/cmd/spire-agent/cli/run/run_posix_test.go @@ -362,6 +362,52 @@ func newAgentConfigCasesOS(t *testing.T) []newAgentConfigCase { require.Nil(t, c.AdminBindAddress) }, }, + { + msg: "broker socket cannot share workload API socket directory when Workload API is enabled", + expectError: true, + input: func(c *Config) { + c.Agent.SocketPath = "/tmp/spire-agent/public/api.sock" + c.Agent.Experimental.Broker = &brokerHCLConfig{ + SocketPath: "/tmp/spire-agent/public/broker.sock", + Brokers: []brokerHCLEntry{ + { + ID: "spiffe://example.org/broker", + AllowedReferenceTypes: []brokerAllowedReferenceTypeHCLEntry{ + {TypeURL: "type.googleapis.com/spiffe.broker.WorkloadPIDReference"}, + }, + }, + }, + } + }, + test: func(t *testing.T, c *agent.Config) { + require.Nil(t, c) + }, + }, + { + msg: "broker socket can share disabled Workload API socket directory", + input: func(c *Config) { + enabled := false + c.Agent.WorkloadAPI = &workloadAPIConfig{Enabled: &enabled} + c.Agent.SocketPath = "/tmp/spire-agent/public/api.sock" + c.Agent.Experimental.Broker = &brokerHCLConfig{ + SocketPath: "/tmp/spire-agent/public/broker.sock", + Brokers: []brokerHCLEntry{ + { + ID: "spiffe://example.org/broker", + AllowedReferenceTypes: []brokerAllowedReferenceTypeHCLEntry{ + {TypeURL: "type.googleapis.com/spiffe.broker.WorkloadPIDReference"}, + }, + }, + }, + } + }, + test: func(t *testing.T, c *agent.Config) { + require.Nil(t, c.BindAddress) + require.Len(t, c.Broker.BindAddresses, 1) + require.Equal(t, "/tmp/spire-agent/public/broker.sock", c.Broker.BindAddresses[0].String()) + require.Equal(t, "unix", c.Broker.BindAddresses[0].Network()) + }, + }, { msg: "log_file allows to reopen", input: func(c *Config) { diff --git a/cmd/spire-agent/cli/run/run_test.go b/cmd/spire-agent/cli/run/run_test.go index 45f265e92f..e69c2595b5 100644 --- a/cmd/spire-agent/cli/run/run_test.go +++ b/cmd/spire-agent/cli/run/run_test.go @@ -1053,6 +1053,41 @@ func TestNewAgentConfig(t *testing.T) { }, c.Broker.Brokers[0].AllowedReferenceTypes) }, }, + { + msg: "workload_api can be disabled when broker is configured", + input: func(c *Config) { + enabled := false + c.Agent.WorkloadAPI = &workloadAPIConfig{Enabled: &enabled} + c.Agent.Experimental.Broker = &brokerHCLConfig{ + BindAddress: "127.0.0.1:8443", + Brokers: []brokerHCLEntry{ + { + ID: "spiffe://example.org/broker", + AllowedReferenceTypes: []brokerAllowedReferenceTypeHCLEntry{ + {TypeURL: "type.googleapis.com/spiffe.broker.WorkloadPIDReference"}, + }, + }, + }, + } + }, + test: func(t *testing.T, c *agent.Config) { + require.Nil(t, c.BindAddress) + require.Len(t, c.Broker.BindAddresses, 1) + require.Len(t, c.Broker.Brokers, 1) + }, + }, + { + msg: "workload_api cannot be disabled without broker", + expectError: true, + requireErrorPrefix: "workload_api.enabled=false requires experimental.broker to be configured", + input: func(c *Config) { + enabled := false + c.Agent.WorkloadAPI = &workloadAPIConfig{Enabled: &enabled} + }, + test: func(t *testing.T, c *agent.Config) { + require.Nil(t, c) + }, + }, { msg: "broker allowed_reference_types is required", expectError: true, @@ -1354,6 +1389,9 @@ func TestParseBrokerAllowedReferenceTypes(t *testing.T) { _, err = file.WriteString(` agent { + workload_api { + enabled = false + } experimental { broker { bind_address = "127.0.0.1:8443" @@ -1380,6 +1418,9 @@ agent { c, err := ParseFile(file.Name(), false) require.NoError(t, err) + require.NotNil(t, c.Agent.WorkloadAPI) + require.NotNil(t, c.Agent.WorkloadAPI.Enabled) + require.False(t, *c.Agent.WorkloadAPI.Enabled) require.NotNil(t, c.Agent.Experimental.Broker) require.Equal(t, []brokerAllowedReferenceTypeHCLEntry{ {TypeURL: "type.googleapis.com/spiffe.broker.KubernetesObjectReference", AllowOverTCP: true}, diff --git a/conf/agent/agent_full.conf b/conf/agent/agent_full.conf index 2dd5be64fc..a3820699fb 100644 --- a/conf/agent/agent_full.conf +++ b/conf/agent/agent_full.conf @@ -62,8 +62,16 @@ agent { server_port = "8081" # socket_path: Location to bind the workload API socket. Default: /tmp/spire-agent/public/api.sock. + # Only used when workload_api.enabled is true. socket_path = "/tmp/spire-agent/public/api.sock" + # workload_api: Optional Workload API and SDS endpoint configuration. + # workload_api { + # # enabled: Set to false to run only the SPIFFE Broker API. + # # Requires experimental.broker to be configured. Default: true. + # # enabled = true + # } + # trust_bundle_path: Path to the SPIRE server CA bundle. trust_bundle_path = "./conf/agent/dummy_root_ca.crt" diff --git a/doc/plugin_agent_nodeattestor_k8s_psat.md b/doc/plugin_agent_nodeattestor_k8s_psat.md index 1162b2bcbb..ca395fcaf3 100644 --- a/doc/plugin_agent_nodeattestor_k8s_psat.md +++ b/doc/plugin_agent_nodeattestor_k8s_psat.md @@ -7,12 +7,18 @@ reads and provides the signed projected service account token (PSAT) to the serv In addition to service account data, PSAT embeds the pod name and UID on its claims. This allows SPIRE to create more fine-grained attestation policies for agents. -The [server-side `k8s_psat` plugin](plugin_server_nodeattestor_k8s_psat.md) will generate a SPIFFE ID on behalf of the agent of the form: +The [server-side `k8s_psat` plugin](plugin_server_nodeattestor_k8s_psat.md) will generate a SPIFFE ID on behalf of the agent. By default, the SPIFFE ID has the form: ```xml spiffe:///spire/agent/k8s_psat// ``` +If the server-side cluster configuration enables `use_pod_uid_for_agent_id`, the SPIFFE ID has the form: + +```xml +spiffe:///spire/agent/k8s_psat//pod/ +``` + The main configuration accepts the following values: | Configuration | Description | Default | diff --git a/doc/plugin_server_nodeattestor_k8s_psat.md b/doc/plugin_server_nodeattestor_k8s_psat.md index 85e5219ecb..b5356c406a 100644 --- a/doc/plugin_server_nodeattestor_k8s_psat.md +++ b/doc/plugin_server_nodeattestor_k8s_psat.md @@ -5,12 +5,18 @@ The `k8s_psat` plugin attests nodes running inside of Kubernetes. The server validates the signed projected service account token provided by the agent. This validation is performed using Kubernetes [Token Review API](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#tokenreview-v1-authentication-k8s-io). In addition to validation, this API provides other useful information (namespace, service account name and pod name) that SPIRE server uses to build selectors. -Kubernetes API server is also queried to get extra data like node UID, which is used to generate a SPIFFE ID with the form: +Kubernetes API server is also queried to get extra data like node UID, which is used by default to generate a SPIFFE ID with the form: ```xml spiffe:///spire/agent/k8s_psat// ``` +When `use_pod_uid_for_agent_id` is enabled for a cluster, the agent SPIFFE ID is instead generated from the pod UID: + +```xml +spiffe:///spire/agent/k8s_psat//pod/ +``` + The server does not need to be running in Kubernetes in order to perform node attestation. In fact, the plugin can be configured to attest nodes running in multiple clusters. @@ -29,6 +35,7 @@ Each cluster in the main configuration requires the following configuration: | Configuration | Description | Default | |------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------| | `service_account_allow_list` | A list of service account names, qualified by namespace (for example, "default:blog" or "production:web") to allow for node attestation. Attestation will be rejected for tokens bound to service accounts that aren't in the allow list. | | +| `use_pod_uid_for_agent_id` | Use agent pod UID instead of node UID when generating the agent SPIFFE ID. When enabled, the ID has the form `spiffe:///spire/agent/k8s_psat//pod/`. | false | | `audience` | Audience for token validation. If it is set to an empty array (`[]`), Kubernetes API server audience is used | ["spire-server"] | | `kube_config_file` | Path to a k8s configuration file for API Server authentication. A kubernetes configuration file must be specified if SPIRE server runs outside of the k8s cluster. If empty, SPIRE server is assumed to be running inside the cluster and in-cluster configuration is used. | "" | | `allowed_node_label_keys` | Node label keys considered for selectors | | diff --git a/doc/spire_agent.md b/doc/spire_agent.md index 091ce15c4b..cec20b0704 100644 --- a/doc/spire_agent.md +++ b/doc/spire_agent.md @@ -66,7 +66,8 @@ This may be useful for templating configuration files, for example across differ | `profiling_port` | Port number of the [net/http/pprof](https://pkg.go.dev/net/http/pprof) endpoint. Only used when `profiling_enabled` is `true`. | | | `server_address` | DNS name or IP address of the SPIRE server | | | `server_port` | Port number of the SPIRE server | | -| `socket_path` | Location to bind the SPIRE Agent API socket (Unix only) | /tmp/spire-agent/public/api.sock | +| `socket_path` | Location to bind the SPIRE Agent API socket (Unix only). Only used when the Workload API is enabled. | /tmp/spire-agent/public/api.sock | +| `workload_api` | Optional Workload API and SDS endpoint configuration. Set `enabled = false` only when `experimental.broker` is configured to run the agent with the Broker API and no Workload API. | `enabled = true` | | `sds` | Optional SDS configuration section | | | `trust_bundle_path` | Path to the SPIRE server CA bundle | | | `trust_bundle_url` | URL to download the initial SPIRE server trust bundle | | @@ -88,6 +89,24 @@ This may be useful for templating configuration files, for example across differ | `ratelimit` | Optional per-caller rate limiting for Workload API and SDS methods, enforced after workload attestation. See [Workload API Rate Limiting](#workload-api-rate-limiting) for details. | | | `broker` | Optional SPIFFE Broker API endpoint configuration. See [SPIFFE Broker API](#spiffe-broker-api). | | +### Workload API + +The Workload API and SDS endpoints are enabled by default. To run an agent with only the SPIFFE Broker API, configure `workload_api.enabled = false` and configure `experimental.broker`. When the Workload API is disabled, `socket_path` is not used and the Workload API/SDS endpoints are not served. + +```hcl +agent { + workload_api { + enabled = false + } + + experimental { + broker { + # Broker API configuration... + } + } +} +``` + ### Workload API Rate Limiting The `ratelimit` configuration block enforces per-caller rate limits on Workload API and Envoy SDS methods to protect the agent from noisy-neighbor workloads and reconnection storms. @@ -626,8 +645,7 @@ in the agent config. At least one of `socket_path` (POSIX-only UDS) or `bind_address` (TCP, all platforms) must be set; both may be set simultaneously. -The `socket_path` must not live in the same directory as the Workload -API socket — SPIRE Agent will refuse to start otherwise. +When the Workload API is enabled, the Broker API `socket_path` must not live in the same directory as the Workload API socket; SPIRE Agent will refuse to start otherwise. This restriction does not apply when `workload_api.enabled = false`. Each entry in `brokers` enumerates an authorized broker's SPIFFE ID (cross-trust-domain identities are allowed) and the reference types it @@ -642,6 +660,11 @@ allow any reference type the agent's attestor stack understands. agent { trust_domain = "example.org" ... + # Optional: run only the Broker API. + # workload_api { + # enabled = false + # } + experimental { broker { socket_path = "/run/spire/broker-sockets/broker.sock" # POSIX UDS diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index bd63a086bb..bf56b18db3 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -153,7 +153,7 @@ func (a *Agent) Run(ctx context.Context) error { var as *node_attestor.AttestationResult - readyForHealthChecks := make(chan struct{}) + readyForHealthChecks := make(chan struct{}, 1) go func() { a.startHealthChecks(readyForHealthChecks, taskRunner, healthChecker) }() @@ -274,27 +274,31 @@ func (a *Agent) Run(ctx context.Context) error { Metrics: metrics, }) - agentEndpoints := a.newEndpoints(metrics, mgr, workloadAttestor) - go func() { - agentEndpoints.WaitForListening(readyForHealthChecks) - a.started = true - }() - tasks := []func(context.Context) error{ metrics.ListenAndServe, mgr.Run, storeService.Run, - agentEndpoints.ListenAndServe, catalog.ReconfigureTask(a.c.Log.WithField(telemetry.SubsystemName, "reconfigurer"), cat), } + workloadAPIEnabled := a.c.BindAddress != nil + if workloadAPIEnabled { + agentEndpoints := a.newEndpoints(metrics, mgr, workloadAttestor) + go func() { + agentEndpoints.WaitForListening(readyForHealthChecks) + a.started = true + }() + tasks = append(tasks, agentEndpoints.ListenAndServe) + } + if a.c.AdminBindAddress != nil { adminEndpoints := a.newAdminEndpoints(metrics, mgr, workloadAttestor, a.c.AuthorizedDelegates) tasks = append(tasks, adminEndpoints.ListenAndServe) } + var brokerEndpoints *broker.Endpoints if len(a.c.Broker.BindAddresses) != 0 { - brokerEndpoints, err := broker.New(&broker.Config{ + brokerEndpoints, err = broker.New(&broker.Config{ BindAddrs: a.c.Broker.BindAddresses, Manager: mgr, Log: a.c.Log, @@ -311,6 +315,17 @@ func (a *Agent) Run(ctx context.Context) error { tasks = append(tasks, brokerEndpoints.ListenAndServe) } + if !workloadAPIEnabled { + go func() { + if brokerEndpoints != nil { + brokerEndpoints.WaitForListening(readyForHealthChecks) + } else { + close(readyForHealthChecks) + } + a.started = true + }() + } + if a.c.LogReopener != nil { tasks = append(tasks, a.c.LogReopener) } @@ -540,6 +555,14 @@ func (a *Agent) newAdminEndpoints(metrics telemetry.Metrics, mgr manager.Manager // CheckHealth is used as a top-level health check for the agent. func (a *Agent) CheckHealth() health.State { + if a.c.BindAddress == nil { + return health.State{ + Started: &a.started, + Ready: a.started, + Live: true, + } + } + err := a.checkWorkloadAPI() // Both liveness and readiness checks are done by diff --git a/pkg/agent/agent_test.go b/pkg/agent/agent_test.go new file mode 100644 index 0000000000..74fd3d3d68 --- /dev/null +++ b/pkg/agent/agent_test.go @@ -0,0 +1,28 @@ +package agent + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestCheckHealthWithWorkloadAPIDisabled(t *testing.T) { + a := New(&Config{}) + + state := a.CheckHealth() + require.NotNil(t, state.Started) + require.False(t, *state.Started) + require.False(t, state.Ready) + require.True(t, state.Live) + require.Nil(t, state.ReadyDetails) + require.Nil(t, state.LiveDetails) + + a.started = true + state = a.CheckHealth() + require.NotNil(t, state.Started) + require.True(t, *state.Started) + require.True(t, state.Ready) + require.True(t, state.Live) + require.Nil(t, state.ReadyDetails) + require.Nil(t, state.LiveDetails) +} diff --git a/pkg/agent/broker/endpoints.go b/pkg/agent/broker/endpoints.go index 25714d9172..db24d626a4 100644 --- a/pkg/agent/broker/endpoints.go +++ b/pkg/agent/broker/endpoints.go @@ -85,7 +85,8 @@ type AllowedReferenceType struct { } type Endpoints struct { - c *Config + c *Config + listening chan struct{} } func New(c *Config) (*Endpoints, error) { @@ -106,7 +107,8 @@ func New(c *Config) (*Endpoints, error) { return nil, errors.New("bundle source is required") } return &Endpoints{ - c: c, + c: c, + listening: make(chan struct{}, 1), }, nil } @@ -179,6 +181,8 @@ func (e *Endpoints) ListenAndServe(ctx context.Context) error { }).Info("Starting SPIFFE Broker Endpoint") } + e.triggerListeningHook() + // Fan one gRPC server out across every listener with an errgroup. The // first goroutine to error (or context cancellation) cancels the // errgroup's context, which the watcher goroutine uses to call @@ -207,6 +211,21 @@ func (e *Endpoints) ListenAndServe(ctx context.Context) error { return nil } +func (e *Endpoints) WaitForListening(listening chan struct{}) { + <-e.listening + select { + case listening <- struct{}{}: + default: + } +} + +func (e *Endpoints) triggerListeningHook() { + select { + case e.listening <- struct{}{}: + default: + } +} + func (e *Endpoints) registerBrokerAPI(server *grpc.Server) { service := brokerapi.New(brokerapi.Config{ Manager: e.c.Manager, diff --git a/pkg/agent/config.go b/pkg/agent/config.go index 0188344894..830dc79d42 100644 --- a/pkg/agent/config.go +++ b/pkg/agent/config.go @@ -27,7 +27,7 @@ const ( type WorkloadAPIRateLimitConfig = endpoints.WorkloadAPIRateLimitConfig type Config struct { - // Address to bind the workload api to + // Address to bind the workload API to. Nil disables the Workload API and SDS endpoint. BindAddress net.Addr // Directory to store runtime data diff --git a/pkg/server/plugin/nodeattestor/k8spsat/psat.go b/pkg/server/plugin/nodeattestor/k8spsat/psat.go index 92d4f29612..cfe8706503 100644 --- a/pkg/server/plugin/nodeattestor/k8spsat/psat.go +++ b/pkg/server/plugin/nodeattestor/k8spsat/psat.go @@ -51,6 +51,9 @@ type ClusterConfig struct { // Attestation is denied if coming from a service account that is not in the list ServiceAccountAllowList []string `hcl:"service_account_allow_list"` + // Use the agent pod UID instead of the node UID when generating the agent SPIFFE ID + UsePodUIDForAgentID bool `hcl:"use_pod_uid_for_agent_id"` + // Audience for PSAT token validation // If audience is not configured, defaultAudience will be used // If audience value is set to an empty slice, k8s apiserver audience will be used @@ -74,6 +77,7 @@ type attestorConfig struct { type clusterConfig struct { serviceAccounts map[string]bool + usePodUIDForAgentID bool audience []string client apiserver.Client allowedNodeLabelKeys map[string]bool @@ -125,6 +129,7 @@ func buildConfig(coreConfig catalog.CoreConfig, hclText string, status *pluginco newConfig.clusters[name] = &clusterConfig{ serviceAccounts: serviceAccounts, + usePodUIDForAgentID: hclCluster.UsePodUIDForAgentID, audience: audience, client: apiserver.New(hclCluster.KubeConfigFile), allowedNodeLabelKeys: allowedNodeLabelKeys, @@ -258,11 +263,16 @@ func (p *AttestorPlugin) Attest(stream nodeattestorv1.NodeAttestor_AttestServer) } } + agentIDPathSuffix := nodeUID + if cluster.usePodUIDForAgentID { + agentIDPathSuffix = fmt.Sprintf("pod/%s", podUID) + } + return stream.Send(&nodeattestorv1.AttestResponse{ Response: &nodeattestorv1.AttestResponse_AgentAttributes{ AgentAttributes: &nodeattestorv1.AgentAttributes{ CanReattest: true, - SpiffeId: k8s.AgentID(pluginName, config.trustDomain, attestationData.Cluster, nodeUID), + SpiffeId: k8s.AgentID(pluginName, config.trustDomain, attestationData.Cluster, agentIDPathSuffix), SelectorValues: selectorValues, }, }, diff --git a/pkg/server/plugin/nodeattestor/k8spsat/psat_test.go b/pkg/server/plugin/nodeattestor/k8spsat/psat_test.go index a6a16278b4..1e6254e499 100644 --- a/pkg/server/plugin/nodeattestor/k8spsat/psat_test.go +++ b/pkg/server/plugin/nodeattestor/k8spsat/psat_test.go @@ -323,6 +323,44 @@ func (s *AttestorSuite) TestAttestSuccess() { }, result.Selectors) } +func (s *AttestorSuite) TestAttestSuccessWithPodUIDAgentID() { + attestor := s.loadPluginWithConfig(` + clusters = { + "FOO" = { + service_account_allow_list = ["NS1:SA1"] + kube_config_file = "" + use_pod_uid_for_agent_id = true + } + } + `) + + tokenData := &TokenData{ + namespace: "NS1", + serviceAccountName: "SA1", + podName: "PODNAME-1", + podUID: "PODUID-1", + } + token := s.signToken(s.fooSigner, tokenData) + s.apiServerClient.SetTokenStatus(token, createTokenStatus(tokenData, true, defaultAudience)) + s.apiServerClient.SetPod(createPod("NS1", "PODNAME-1", "NODENAME-1", "172.16.10.1")) + s.apiServerClient.SetNode(createNode("NODENAME-1", "NODEUID-1")) + + result, err := attestor.Attest(context.Background(), makePayload("FOO", token), expectNoChallenge) + s.Require().NoError(err) + s.Require().NotNil(result) + s.Require().Equal(result.AgentID, "spiffe://example.org/spire/agent/k8s_psat/FOO/pod/PODUID-1") + s.RequireProtoListEqual([]*common.Selector{ + {Type: "k8s_psat", Value: "cluster:FOO"}, + {Type: "k8s_psat", Value: "agent_ns:NS1"}, + {Type: "k8s_psat", Value: "agent_sa:SA1"}, + {Type: "k8s_psat", Value: "agent_pod_name:PODNAME-1"}, + {Type: "k8s_psat", Value: "agent_pod_uid:PODUID-1"}, + {Type: "k8s_psat", Value: "agent_node_ip:172.16.10.1"}, + {Type: "k8s_psat", Value: "agent_node_name:NODENAME-1"}, + {Type: "k8s_psat", Value: "agent_node_uid:NODEUID-1"}, + }, result.Selectors) +} + func (s *AttestorSuite) TestConfigure() { doConfig := func(coreConfig catalog.CoreConfig, config string) error { var err error @@ -388,9 +426,7 @@ func (s *AttestorSuite) signToken(signer jose.Signer, tokenData *TokenData) stri } func (s *AttestorSuite) loadPlugin() nodeattestor.NodeAttestor { - attestor := New() - v1 := new(nodeattestor.V1) - plugintest.Load(s.T(), builtin(attestor), v1, plugintest.Configure(` + return s.loadPluginWithConfig(` clusters = { "FOO" = { service_account_allow_list = ["NS1:SA1"] @@ -404,14 +440,23 @@ func (s *AttestorSuite) loadPlugin() nodeattestor.NodeAttestor { audience = ["AUDIENCE"] } } - `), plugintest.CoreConfig(catalog.CoreConfig{ - TrustDomain: spiffeid.RequireTrustDomainFromString("example.org"), - })) + `) +} + +func (s *AttestorSuite) loadPluginWithConfig(config string) nodeattestor.NodeAttestor { + attestor := New() + v1 := new(nodeattestor.V1) + plugintest.Load(s.T(), builtin(attestor), v1, + plugintest.Configure(config), + plugintest.CoreConfig(catalog.CoreConfig{ + TrustDomain: spiffeid.RequireTrustDomainFromString("example.org"), + })) // TODO: provide this client in a cleaner way s.apiServerClient = newFakeAPIServerClient() - attestor.config.clusters["FOO"].client = s.apiServerClient - attestor.config.clusters["BAR"].client = s.apiServerClient + for _, cluster := range attestor.config.clusters { + cluster.client = s.apiServerClient + } return v1 }