Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions perfkitbenchmarker/benchmark_spec.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@
from perfkitbenchmarker.configs import vm_group_decoders
from perfkitbenchmarker.resources import ai_agent_service
from perfkitbenchmarker.resources import base_job
from perfkitbenchmarker.resources import agent_sandbox
from perfkitbenchmarker.resources import example_resource
from perfkitbenchmarker.resources import managed_ai_model
from perfkitbenchmarker.resources.container_service import container_cluster
Expand Down Expand Up @@ -202,6 +203,7 @@ def __init__(
self.base_job = None
self.edw_service = None
self.edw_compute_resource = None
self.agent_sandbox = None
self.example_resource = None
self.multi_attach_disk = None
self.nfs_service = None
Expand Down Expand Up @@ -337,6 +339,7 @@ def ConstructResources(self):
# Put registry first, as it can be needed by cluster.
self.ConstructContainerRegistry()
self.ConstructContainerCluster()
self.ConstructAgentSandbox()
# dpb service needs to go first, because it adds some vms.
self.ConstructDpbService()
self.ConstructCluster()
Expand Down Expand Up @@ -589,6 +592,19 @@ def ConstructExampleResource(self):
) # pytype: disable=not-instantiable
self.resources.append(self.example_resource)

def ConstructAgentSandbox(self):
"""Create the agent_sandbox object (requires a container_cluster)."""
if self.config.agent_sandbox is None:
return
if self.container_cluster is None:
raise errors.Config.InvalidValue(
'agent_sandbox requires a container_cluster to be configured.')
self.agent_sandbox = agent_sandbox.GetAgentSandbox(
self.config.agent_sandbox, self.container_cluster
)
if self.agent_sandbox:
self.resources.append(self.agent_sandbox)

def ConstructBaseJob(self):
"""Create an instance of the base job.It is also called from pkb.py."""
if self.config.base_job is None:
Expand Down Expand Up @@ -1057,6 +1073,8 @@ def Provision(self):

if self.container_cluster:
self.container_cluster.Create()
if self.agent_sandbox:
self.agent_sandbox.Create()

# do after network setup but before VM created
if self.nfs_service and self.nfs_service.CLOUD != nfs_service.UNMANAGED:
Expand Down Expand Up @@ -1207,6 +1225,8 @@ def Delete(self):
self.edw_service.Delete()
if hasattr(self, 'edw_compute_resource') and self.edw_compute_resource:
self.edw_compute_resource.Delete()
if self.agent_sandbox:
self.agent_sandbox.Delete()
if self.example_resource:
self.example_resource.Delete()
if self.base_job:
Expand Down
5 changes: 5 additions & 0 deletions perfkitbenchmarker/configs/benchmark_config_spec.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
from perfkitbenchmarker.configs import spec
from perfkitbenchmarker.configs import vm_group_decoders
from perfkitbenchmarker.resources import ai_agent_service_spec
from perfkitbenchmarker.resources import agent_sandbox_spec
from perfkitbenchmarker.resources import example_resource_spec
from perfkitbenchmarker.resources import jobs_setter
from perfkitbenchmarker.resources import managed_ai_model_spec
Expand Down Expand Up @@ -1488,6 +1489,10 @@ def _GetOptionDecoderConstructions(cls):
'tpu_groups': (_TpuGroupsDecoder, {'default': {}}),
'edw_compute_resource': (_EdwComputeResourceDecoder, {'default': None}),
'edw_service': (_EdwServiceDecoder, {'default': None}),
'agent_sandbox': (
agent_sandbox_spec.AgentSandboxConfigDecoder,
{'default': None, 'none_ok': True},
),
'example_resource': (_ExampleResourceDecoder, {'default': None}),
'base_job': (_BaseJobDecoder, {'default': None}),
'memory_store': (_MemoryStoreDecoder, {'default': None}),
Expand Down
84 changes: 84 additions & 0 deletions perfkitbenchmarker/linux_benchmarks/agent_sandbox_benchmark.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
# Copyright 2026 PerfKitBenchmarker Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Stub benchmark that provisions the Kubernetes agent sandbox resource.

The agent sandbox is installed via the top-level agent_sandbox config block,
wired through benchmark_spec.ConstructAgentSandbox. The load generator and
metrics land in a follow-up change; Run currently returns no samples.
"""

from absl import flags
from perfkitbenchmarker import configs
# Imported so the K8sAgentSandbox resource and its config spec register.
from perfkitbenchmarker.resources.kubernetes import k8s_agent_sandbox # pylint: disable=unused-import

FLAGS = flags.FLAGS

BENCHMARK_NAME = 'agent_sandbox'
BENCHMARK_CONFIG = """
agent_sandbox:
description: >
Provision the agent-sandbox stack on a Kubernetes cluster. Load generation
and metrics are added in a follow-up change.
container_cluster:
cloud: GCP
type: Kubernetes
vm_count: 1
vm_spec:
GCP:
machine_type: c4-standard-4
zone: us-central1-a
AWS:
machine_type: m8i.xlarge
zone: us-east-1a
nodepools:
sandbox:
vm_count: 4
vm_spec:
GCP:
machine_type: c4-standard-16
zone: us-central1-a
AWS:
machine_type: m8i.4xlarge
zone: us-east-1a
node_labels:
sandbox.gke.io/runtime: runsc
node_taints:
- sandbox.gke.io/runtime=runsc:NoSchedule
agent_sandbox:
type: Kubernetes
"""


def GetConfig(user_config):
"""Loads the benchmark config and merges user overrides."""
config = configs.LoadConfig(BENCHMARK_CONFIG, user_config, BENCHMARK_NAME)
config['container_cluster']['cloud'] = FLAGS.cloud
return config


def Prepare(benchmark_spec):
"""No-op: the agent sandbox is provisioned via benchmark_spec."""
del benchmark_spec


def Run(benchmark_spec):
"""Returns no samples yet. Load generation lands in a follow-up change."""
del benchmark_spec
return [] # TODO(followup): run the load generator and return samples.


def Cleanup(benchmark_spec):
"""No-op: cluster teardown reclaims the agent sandbox stack."""
del benchmark_spec
54 changes: 54 additions & 0 deletions perfkitbenchmarker/resources/agent_sandbox.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
# Copyright 2026 PerfKitBenchmarker Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Base class for an agent sandbox resource installed on a Kubernetes cluster.

An agent sandbox is a stack (CRDs, controller, runtime class, warm pools)
installed onto a Kubernetes cluster. Concrete subclasses provide the install
logic, for example the open-source kubernetes-sigs/agent-sandbox stack or a
cloud-managed offering. The resource is modeled on the kubernetes inference
server pattern and is attached to a cluster as cluster.agent_sandbox.
"""

from __future__ import annotations

from typing import Optional

from perfkitbenchmarker import errors
from perfkitbenchmarker import resource as pkb_resource


class BaseAgentSandbox(pkb_resource.BaseResource):
"""Base class for an agent sandbox resource."""

RESOURCE_TYPE = 'BaseAgentSandbox'
REQUIRED_ATTRS = ['SANDBOX_TYPE']

def __init__(self, spec, cluster):
super().__init__()
self.spec = spec
self.cluster = cluster
if self.cluster is None:
raise errors.Resource.CreationError(
'A kubernetes cluster is required for an agent sandbox resource.'
)


def GetAgentSandbox(spec, cluster) -> Optional[BaseAgentSandbox]:
"""Returns an agent sandbox resource for the given spec, or None."""
if not spec:
return None
agent_sandbox_class: type[BaseAgentSandbox] = pkb_resource.GetResourceClass(
BaseAgentSandbox, SANDBOX_TYPE=spec.type
)
return agent_sandbox_class(spec, cluster)
75 changes: 75 additions & 0 deletions perfkitbenchmarker/resources/agent_sandbox_spec.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
# Copyright 2026 PerfKitBenchmarker Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Spec for an agent sandbox resource."""

from perfkitbenchmarker import errors
from perfkitbenchmarker.configs import option_decoders
from perfkitbenchmarker.configs import spec

DEFAULT_SANDBOX_TYPE = 'Kubernetes'


class BaseAgentSandboxConfigSpec(spec.BaseSpec):
"""Spec for agent sandbox configuration.

Attributes:
type: Type of the agent sandbox (for example Kubernetes or a managed offering).
"""

SPEC_TYPE = 'BaseAgentSandboxConfigSpec'
SPEC_ATTRS = ['SANDBOX_TYPE']

def __init__(self, component_full_name, flag_values=None, **kwargs):
self.type: str
super().__init__(component_full_name, flag_values=flag_values, **kwargs)

@classmethod
def _GetOptionDecoderConstructions(cls):
"""Gets decoder classes and constructor args for each configurable option."""
result = super()._GetOptionDecoderConstructions()
result.update({
'type': (
option_decoders.StringDecoder,
{'default': DEFAULT_SANDBOX_TYPE, 'none_ok': False},
),
})
return result


class AgentSandboxConfigDecoder(option_decoders.TypeVerifier):
"""Decodes an agent sandbox configuration block."""

def __init__(self, **kwargs):
super().__init__((dict,), **kwargs)

def Decode(self, value, component_full_name, flag_values):
super().Decode(value, component_full_name, flag_values)
sandbox_type = value['type'] if 'type' in value else DEFAULT_SANDBOX_TYPE
config_spec_class = GetAgentSandboxConfigSpecClass(sandbox_type)
if config_spec_class is None:
raise errors.Config.UnrecognizedOption(
'Unrecognized agent sandbox type: {}.'.format(sandbox_type)
)
return config_spec_class(
self._GetOptionFullName(component_full_name),
flag_values=flag_values,
**value
)


def GetAgentSandboxConfigSpecClass(sandbox_type):
"""Gets the AgentSandboxConfigSpec class for the given type."""
return spec.GetSpecClass(
BaseAgentSandboxConfigSpec, SANDBOX_TYPE=sandbox_type
)
49 changes: 49 additions & 0 deletions perfkitbenchmarker/resources/kubernetes/k8s_agent_sandbox.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
# Copyright 2026 PerfKitBenchmarker Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Kubernetes implementation of an agent sandbox.

This module models the configuration surface for the open-source
kubernetes-sigs/agent-sandbox stack. The on-cluster installation orchestration
(gVisor, controller, sandbox template, and warm pool) is implemented in the
follow-up benchmark change.
"""

from absl import logging

from perfkitbenchmarker.resources import agent_sandbox
from perfkitbenchmarker.resources import agent_sandbox_spec
from perfkitbenchmarker.resources.kubernetes import k8s_agent_sandbox_spec # pylint: disable=unused-import


class K8sAgentSandbox(agent_sandbox.BaseAgentSandbox):
"""Models the configuration surface for the kubernetes-sigs/agent-sandbox stack."""

SANDBOX_TYPE = agent_sandbox_spec.DEFAULT_SANDBOX_TYPE

def _Create(self) -> None:
"""Registers the agent sandbox; on-cluster installation is deferred.

This resource currently models only the configuration surface. The
on-cluster installation orchestration (gVisor, controller, sandbox
template, and warm pool) and its manifests are implemented in the
follow-up benchmark change.
"""
logging.info(
'Agent sandbox configuration registered; on-cluster installation is '
'deferred to a follow-up change.'
)

def _Delete(self):
"""No-op: the ephemeral cluster teardown reclaims the sandbox stack."""
pass
Loading