Skip to content

Support agent deployments for only Broker API #7118

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
matheuscscp wants to merge 1 commit into from
Closed

Support agent deployments for only Broker API #7118

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 55 additions & 31 deletions cmd/spire-agent/cli/run/run.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,33 +67,34 @@ type Config struct {
}

type agentConfig struct {
DataDir string `hcl:"data_dir"`
AdminSocketPath string `hcl:"admin_socket_path"`
InsecureBootstrap bool `hcl:"insecure_bootstrap"`
RebootstrapMode string `hcl:"rebootstrap_mode"`
RebootstrapDelay string `hcl:"rebootstrap_delay"`
JoinToken string `hcl:"join_token"`
JoinTokenFile string `hcl:"join_token_file"`
LogFile string `hcl:"log_file"`
LogFormat string `hcl:"log_format"`
LogLevel string `hcl:"log_level"`
LogSelectors []string `hcl:"log_selectors"`
LogSourceLocation bool `hcl:"log_source_location"`
SDS sdsConfig `hcl:"sds"`
ServerAddress string `hcl:"server_address"`
ServerPort int `hcl:"server_port"`
SocketPath string `hcl:"socket_path"`
WorkloadX509SVIDKeyType string `hcl:"workload_x509_svid_key_type"`
TrustBundleFormat string `hcl:"trust_bundle_format"`
TrustBundlePath string `hcl:"trust_bundle_path"`
TrustBundleUnixSocket string `hcl:"trust_bundle_unix_socket"`
TrustBundleURL string `hcl:"trust_bundle_url"`
TrustDomain string `hcl:"trust_domain"`
AllowUnauthenticatedVerifiers bool `hcl:"allow_unauthenticated_verifiers"`
AllowedForeignJWTClaims []string `hcl:"allowed_foreign_jwt_claims"`
AvailabilityTarget string `hcl:"availability_target"`
X509SVIDCacheMaxSize int `hcl:"x509_svid_cache_max_size"`
JWTSVIDCacheMaxSize int `hcl:"jwt_svid_cache_max_size"`
DataDir string `hcl:"data_dir"`
AdminSocketPath string `hcl:"admin_socket_path"`
InsecureBootstrap bool `hcl:"insecure_bootstrap"`
RebootstrapMode string `hcl:"rebootstrap_mode"`
RebootstrapDelay string `hcl:"rebootstrap_delay"`
JoinToken string `hcl:"join_token"`
JoinTokenFile string `hcl:"join_token_file"`
LogFile string `hcl:"log_file"`
LogFormat string `hcl:"log_format"`
LogLevel string `hcl:"log_level"`
LogSelectors []string `hcl:"log_selectors"`
LogSourceLocation bool `hcl:"log_source_location"`
SDS sdsConfig `hcl:"sds"`
ServerAddress string `hcl:"server_address"`
ServerPort int `hcl:"server_port"`
SocketPath string `hcl:"socket_path"`
WorkloadX509SVIDKeyType string `hcl:"workload_x509_svid_key_type"`
TrustBundleFormat string `hcl:"trust_bundle_format"`
TrustBundlePath string `hcl:"trust_bundle_path"`
TrustBundleUnixSocket string `hcl:"trust_bundle_unix_socket"`
TrustBundleURL string `hcl:"trust_bundle_url"`
TrustDomain string `hcl:"trust_domain"`
AllowUnauthenticatedVerifiers bool `hcl:"allow_unauthenticated_verifiers"`
AllowedForeignJWTClaims []string `hcl:"allowed_foreign_jwt_claims"`
AvailabilityTarget string `hcl:"availability_target"`
X509SVIDCacheMaxSize int `hcl:"x509_svid_cache_max_size"`
JWTSVIDCacheMaxSize int `hcl:"jwt_svid_cache_max_size"`
WorkloadAPI *workloadAPIConfig `hcl:"workload_api"`

AuthorizedDelegates []string `hcl:"authorized_delegates"`

Expand Down Expand Up @@ -228,6 +229,15 @@ type sdsConfig struct {
DisableSPIFFECertValidation bool `hcl:"disable_spiffe_cert_validation"`
}

type workloadAPIConfig struct {
// Enabled controls whether the agent serves the Workload API and SDS
// endpoint. It defaults to true. Setting it to false requires the Broker
// API endpoint to be configured.
Enabled *bool `hcl:"enabled"`

UnusedKeyPositions map[string][]token.Pos `hcl:",unusedKeyPositions"`
}

type workloadAPIRateLimitConfig struct {
FetchX509SVID *int `hcl:"fetch_x509_svid"`
FetchJWTSVID *int `hcl:"fetch_jwt_svid"`
Expand Down Expand Up @@ -359,6 +369,10 @@ func (c *agentConfig) validate() error {
return errors.New("agent section must be configured")
}

if !c.workloadAPIEnabled() && c.Experimental.Broker == nil {
return errors.New("workload_api.enabled=false requires experimental.broker to be configured")
}

// Validate join token configuration
if c.JoinToken != "" && c.JoinTokenFile != "" {
return errors.New("only one of join_token or join_token_file can be specified, not both")
Expand Down Expand Up @@ -431,6 +445,10 @@ func (c *agentConfig) validate() error {
return c.validateOS()
}

func (c *agentConfig) workloadAPIEnabled() bool {
return c.WorkloadAPI == nil || c.WorkloadAPI.Enabled == nil || *c.WorkloadAPI.Enabled
}

func ParseFile(path string, expandEnv bool) (*Config, error) {
c := &Config{}

Expand Down Expand Up @@ -632,11 +650,13 @@ func NewAgentConfig(c *Config, logOptions []log.Option, allowUnknownConfig bool)
}
ac.TrustDomain = td

addr, err := c.Agent.getAddr()
if err != nil {
return nil, err
if c.Agent.workloadAPIEnabled() {
addr, err := c.Agent.getAddr()
if err != nil {
return nil, err
}
ac.BindAddress = addr
}
ac.BindAddress = addr

if c.Agent.hasAdminAddr() {
adminAddr, err := c.Agent.getAdminAddr()
Expand Down Expand Up @@ -856,6 +876,10 @@ func checkForUnknownConfig(c *Config, l logrus.FieldLogger) (err error) {
detectedUnknown("agent", a.UnusedKeyPositions)
}

if a := c.Agent; a != nil && a.WorkloadAPI != nil && len(a.WorkloadAPI.UnusedKeyPositions) != 0 {
detectedUnknown("agent.workload_api", a.WorkloadAPI.UnusedKeyPositions)
}

if a := c.Agent; a != nil && a.Experimental.Broker != nil {
if len(a.Experimental.Broker.UnusedKeyPositions) != 0 {
detectedUnknown("experimental.broker", a.Experimental.Broker.UnusedKeyPositions)
Expand Down
42 changes: 24 additions & 18 deletions cmd/spire-agent/cli/run/run_posix.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,17 +30,19 @@ func (c *agentConfig) getAddr() (net.Addr, error) {
}

func (c *agentConfig) getAdminAddr() (net.Addr, error) {
socketPathAbs, err := filepath.Abs(c.SocketPath)
if err != nil {
return nil, fmt.Errorf("failed to get absolute path for socket_path: %w", err)
}
adminSocketPathAbs, err := filepath.Abs(c.AdminSocketPath)
if err != nil {
return nil, fmt.Errorf("failed to get absolute path for admin_socket_path: %w", err)
}

if strings.HasPrefix(adminSocketPathAbs, filepath.Dir(socketPathAbs)+"/") {
return nil, errors.New("admin socket cannot be in the same directory or a subdirectory as that containing the Workload API socket")
if c.workloadAPIEnabled() {
socketPathAbs, err := filepath.Abs(c.SocketPath)
if err != nil {
return nil, fmt.Errorf("failed to get absolute path for socket_path: %w", err)
}
if strings.HasPrefix(adminSocketPathAbs, filepath.Dir(socketPathAbs)+"/") {
return nil, errors.New("admin socket cannot be in the same directory or a subdirectory as that containing the Workload API socket")
}
}

return &net.UnixAddr{
Expand All @@ -56,17 +58,19 @@ func (c *agentConfig) hasAdminAddr() bool {
// brokerSocketAddr resolves the UDS branch of broker bind-address selection.
// Returns the platform-agnostic TCP branch is handled in brokerBindAddr.
func (c *agentConfig) brokerSocketAddr() (net.Addr, error) {
socketPathAbs, err := filepath.Abs(c.SocketPath)
if err != nil {
return nil, fmt.Errorf("failed to get absolute path for socket_path: %w", err)
}
brokerSocketPathAbs, err := filepath.Abs(c.Experimental.Broker.SocketPath)
if err != nil {
return nil, fmt.Errorf("failed to get absolute path for experimental.broker.socket_path: %w", err)
}

if strings.HasPrefix(brokerSocketPathAbs, filepath.Dir(socketPathAbs)+"/") {
return nil, errors.New("broker socket cannot be in the same directory or a subdirectory as that containing the Workload API socket")
if c.workloadAPIEnabled() {
socketPathAbs, err := filepath.Abs(c.SocketPath)
if err != nil {
return nil, fmt.Errorf("failed to get absolute path for socket_path: %w", err)
}
if strings.HasPrefix(brokerSocketPathAbs, filepath.Dir(socketPathAbs)+"/") {
return nil, errors.New("broker socket cannot be in the same directory or a subdirectory as that containing the Workload API socket")
}
}

return &net.UnixAddr{
Expand All @@ -87,12 +91,14 @@ func (c *agentConfig) validateOS() error {
}

func prepareEndpoints(c *agent.Config) error {
// Create uds dir and parents if not exists
dir := filepath.Dir(c.BindAddress.String())
if _, statErr := os.Stat(dir); os.IsNotExist(statErr) {
c.Log.WithField("dir", dir).Infof("Creating spire agent UDS directory")
if err := os.MkdirAll(dir, 0755); err != nil {
return err
if c.BindAddress != nil {
// Create uds dir and parents if not exists
dir := filepath.Dir(c.BindAddress.String())
if _, statErr := os.Stat(dir); os.IsNotExist(statErr) {
c.Log.WithField("dir", dir).Infof("Creating spire agent UDS directory")
if err := os.MkdirAll(dir, 0755); err != nil {
return err
}
}
}

Expand Down
46 changes: 46 additions & 0 deletions cmd/spire-agent/cli/run/run_posix_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -362,6 +362,52 @@ func newAgentConfigCasesOS(t *testing.T) []newAgentConfigCase {
require.Nil(t, c.AdminBindAddress)
},
},
{
msg: "broker socket cannot share workload API socket directory when Workload API is enabled",
expectError: true,
input: func(c *Config) {
c.Agent.SocketPath = "/tmp/spire-agent/public/api.sock"
c.Agent.Experimental.Broker = &brokerHCLConfig{
SocketPath: "/tmp/spire-agent/public/broker.sock",
Brokers: []brokerHCLEntry{
{
ID: "spiffe://example.org/broker",
AllowedReferenceTypes: []brokerAllowedReferenceTypeHCLEntry{
{TypeURL: "type.googleapis.com/spiffe.broker.WorkloadPIDReference"},
},
},
},
}
},
test: func(t *testing.T, c *agent.Config) {
require.Nil(t, c)
},
},
{
msg: "broker socket can share disabled Workload API socket directory",
input: func(c *Config) {
enabled := false
c.Agent.WorkloadAPI = &workloadAPIConfig{Enabled: &enabled}
c.Agent.SocketPath = "/tmp/spire-agent/public/api.sock"
c.Agent.Experimental.Broker = &brokerHCLConfig{
SocketPath: "/tmp/spire-agent/public/broker.sock",
Brokers: []brokerHCLEntry{
{
ID: "spiffe://example.org/broker",
AllowedReferenceTypes: []brokerAllowedReferenceTypeHCLEntry{
{TypeURL: "type.googleapis.com/spiffe.broker.WorkloadPIDReference"},
},
},
},
}
},
test: func(t *testing.T, c *agent.Config) {
require.Nil(t, c.BindAddress)
require.Len(t, c.Broker.BindAddresses, 1)
require.Equal(t, "/tmp/spire-agent/public/broker.sock", c.Broker.BindAddresses[0].String())
require.Equal(t, "unix", c.Broker.BindAddresses[0].Network())
},
},
{
msg: "log_file allows to reopen",
input: func(c *Config) {
Expand Down
41 changes: 41 additions & 0 deletions cmd/spire-agent/cli/run/run_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1053,6 +1053,41 @@ func TestNewAgentConfig(t *testing.T) {
}, c.Broker.Brokers[0].AllowedReferenceTypes)
},
},
{
msg: "workload_api can be disabled when broker is configured",
input: func(c *Config) {
enabled := false
c.Agent.WorkloadAPI = &workloadAPIConfig{Enabled: &enabled}
c.Agent.Experimental.Broker = &brokerHCLConfig{
BindAddress: "127.0.0.1:8443",
Brokers: []brokerHCLEntry{
{
ID: "spiffe://example.org/broker",
AllowedReferenceTypes: []brokerAllowedReferenceTypeHCLEntry{
{TypeURL: "type.googleapis.com/spiffe.broker.WorkloadPIDReference"},
},
},
},
}
},
test: func(t *testing.T, c *agent.Config) {
require.Nil(t, c.BindAddress)
require.Len(t, c.Broker.BindAddresses, 1)
require.Len(t, c.Broker.Brokers, 1)
},
},
{
msg: "workload_api cannot be disabled without broker",
expectError: true,
requireErrorPrefix: "workload_api.enabled=false requires experimental.broker to be configured",
input: func(c *Config) {
enabled := false
c.Agent.WorkloadAPI = &workloadAPIConfig{Enabled: &enabled}
},
test: func(t *testing.T, c *agent.Config) {
require.Nil(t, c)
},
},
{
msg: "broker allowed_reference_types is required",
expectError: true,
Expand Down Expand Up @@ -1354,6 +1389,9 @@ func TestParseBrokerAllowedReferenceTypes(t *testing.T) {

_, err = file.WriteString(`
agent {
workload_api {
enabled = false
}
experimental {
broker {
bind_address = "127.0.0.1:8443"
Expand All @@ -1380,6 +1418,9 @@ agent {

c, err := ParseFile(file.Name(), false)
require.NoError(t, err)
require.NotNil(t, c.Agent.WorkloadAPI)
require.NotNil(t, c.Agent.WorkloadAPI.Enabled)
require.False(t, *c.Agent.WorkloadAPI.Enabled)
require.NotNil(t, c.Agent.Experimental.Broker)
require.Equal(t, []brokerAllowedReferenceTypeHCLEntry{
{TypeURL: "type.googleapis.com/spiffe.broker.KubernetesObjectReference", AllowOverTCP: true},
Expand Down
8 changes: 8 additions & 0 deletions conf/agent/agent_full.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,8 +62,16 @@ agent {
server_port = "8081"

# socket_path: Location to bind the workload API socket. Default: /tmp/spire-agent/public/api.sock.
# Only used when workload_api.enabled is true.
socket_path = "/tmp/spire-agent/public/api.sock"

# workload_api: Optional Workload API and SDS endpoint configuration.
# workload_api {
# # enabled: Set to false to run only the SPIFFE Broker API.
# # Requires experimental.broker to be configured. Default: true.
# # enabled = true
# }

# trust_bundle_path: Path to the SPIRE server CA bundle.
trust_bundle_path = "./conf/agent/dummy_root_ca.crt"

Expand Down
8 changes: 7 additions & 1 deletion doc/plugin_agent_nodeattestor_k8s_psat.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,18 @@ reads and provides the signed projected service account token (PSAT) to the serv
In addition to service account data, PSAT embeds the pod name and UID on its claims. This allows
SPIRE to create more fine-grained attestation policies for agents.

The [server-side `k8s_psat` plugin](plugin_server_nodeattestor_k8s_psat.md) will generate a SPIFFE ID on behalf of the agent of the form:
The [server-side `k8s_psat` plugin](plugin_server_nodeattestor_k8s_psat.md) will generate a SPIFFE ID on behalf of the agent. By default, the SPIFFE ID has the form:

```xml
spiffe://<trust_domain>/spire/agent/k8s_psat/<cluster>/<node_UID>
```

If the server-side cluster configuration enables `use_pod_uid_for_agent_id`, the SPIFFE ID has the form:

```xml
spiffe://<trust_domain>/spire/agent/k8s_psat/<cluster>/pod/<pod_UID>
```

The main configuration accepts the following values:

| Configuration | Description | Default |
Expand Down
9 changes: 8 additions & 1 deletion doc/plugin_server_nodeattestor_k8s_psat.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,18 @@
The `k8s_psat` plugin attests nodes running inside of Kubernetes. The server
validates the signed projected service account token provided by the agent.
This validation is performed using Kubernetes [Token Review API](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#tokenreview-v1-authentication-k8s-io). In addition to validation, this API provides other useful information (namespace, service account name and pod name) that SPIRE server uses to build selectors.
Kubernetes API server is also queried to get extra data like node UID, which is used to generate a SPIFFE ID with the form:
Kubernetes API server is also queried to get extra data like node UID, which is used by default to generate a SPIFFE ID with the form:

```xml
spiffe://<trust_domain>/spire/agent/k8s_psat/<cluster>/<node UID>
```

When `use_pod_uid_for_agent_id` is enabled for a cluster, the agent SPIFFE ID is instead generated from the pod UID:

```xml
spiffe://<trust_domain>/spire/agent/k8s_psat/<cluster>/pod/<pod UID>
```

The server does not need to be running in Kubernetes in order to perform node
attestation. In fact, the plugin can be configured to attest nodes running in
multiple clusters.
Expand All @@ -29,6 +35,7 @@ Each cluster in the main configuration requires the following configuration:
| Configuration | Description | Default |
|------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------|
| `service_account_allow_list` | A list of service account names, qualified by namespace (for example, "default:blog" or "production:web") to allow for node attestation. Attestation will be rejected for tokens bound to service accounts that aren't in the allow list. | |
| `use_pod_uid_for_agent_id` | Use agent pod UID instead of node UID when generating the agent SPIFFE ID. When enabled, the ID has the form `spiffe://<trust_domain>/spire/agent/k8s_psat/<cluster>/pod/<pod UID>`. | false |
| `audience` | Audience for token validation. If it is set to an empty array (`[]`), Kubernetes API server audience is used | ["spire-server"] |
| `kube_config_file` | Path to a k8s configuration file for API Server authentication. A kubernetes configuration file must be specified if SPIRE server runs outside of the k8s cluster. If empty, SPIRE server is assumed to be running inside the cluster and in-cluster configuration is used. | "" |
| `allowed_node_label_keys` | Node label keys considered for selectors | |
Expand Down
Loading
Loading