diff --git a/perfkitbenchmarker/configs/container_spec.py b/perfkitbenchmarker/configs/container_spec.py index 1f808ad066..32bf8d7771 100644 --- a/perfkitbenchmarker/configs/container_spec.py +++ b/perfkitbenchmarker/configs/container_spec.py @@ -243,6 +243,9 @@ def __init__( self.vm_spec: virtual_machine_spec.BaseVmSpec self.machine_families: list[str] | None self.sandbox_config: SandboxSpec | None + self.max_pods_per_node: int | None + self.node_labels: dict[str, str] | None + self.node_taints: list[str] | None @classmethod def _GetOptionDecoderConstructions(cls): @@ -273,6 +276,18 @@ def _GetOptionDecoderConstructions(cls): ), 'vm_spec': (spec.PerCloudConfigDecoder, {}), 'sandbox_config': (_SandboxDecoder, {'default': None}), + 'max_pods_per_node': ( + option_decoders.IntDecoder, + {'default': None, 'none_ok': True, 'min': 1}, + ), + 'node_labels': ( + option_decoders.TypeVerifier, + {'valid_types': (dict,), 'default': None, 'none_ok': True}, + ), + 'node_taints': ( + option_decoders.TypeVerifier, + {'valid_types': (list,), 'default': None, 'none_ok': True}, + ), }) return result diff --git a/perfkitbenchmarker/providers/gcp/flags.py b/perfkitbenchmarker/providers/gcp/flags.py index a56fe72b99..f20eec5aca 100644 --- a/perfkitbenchmarker/providers/gcp/flags.py +++ b/perfkitbenchmarker/providers/gcp/flags.py @@ -67,9 +67,7 @@ 'Whether or not we create a Confidential VM Instance', ) GCE_CONFIDENTIAL_COMPUTE_TYPE = flags.DEFINE_string( - 'gce_confidential_compute_type', - 'sev', - 'Type of Confidential VM Instance' + 'gce_confidential_compute_type', 'sev', 'Type of Confidential VM Instance' ) GCE_NETWORK_NAMES = flags.DEFINE_list( 'gce_network_name', @@ -561,6 +559,97 @@ False, 'Whether to enable shielded nodes.', ) +GKE_ENABLE_PRIVATE_NODES = flags.DEFINE_boolean( + 'gke_enable_private_nodes', + False, + 'Whether to create the cluster with private nodes (nodes have only internal' + ' IPs).', +) +GKE_ENABLE_DNS_ACCESS = flags.DEFINE_boolean( + 'gke_enable_dns_access', + False, + 'Whether to enable DNS-based control plane access (replaces the' + ' public/private IP endpoint model).', +) +GKE_ENABLE_IP_ACCESS = flags.DEFINE_boolean( + 'gke_enable_ip_access', + True, + 'Whether to enable IP-based control plane access. Disabling requires DNS' + ' access and is mutually exclusive with public clusters (nodes with public' + ' IPs).', +) +GKE_MASTER_IPV4_CIDR = flags.DEFINE_string( + 'gke_master_ipv4_cidr', + None, + 'CIDR range to use for the hosted master network. Required when private' + ' nodes are enabled without DNS access.', +) +GKE_ENABLE_DATAPLANE_V2 = flags.DEFINE_boolean( + 'gke_enable_dataplane_v2', + False, + 'Whether to enable GKE Dataplane V2 (eBPF-based datapath, Cilium under the' + ' hood). Requires cluster recreation; cannot be toggled on an existing' + ' cluster.', +) +GKE_ENABLE_MANAGED_PROMETHEUS = flags.DEFINE_boolean( + 'gke_enable_managed_prometheus', + False, + 'Whether to enable Google Cloud Managed Service for Prometheus on the' + ' cluster.', +) +GKE_ENABLE_COST_ALLOCATION = flags.DEFINE_boolean( + 'gke_enable_cost_allocation', + False, + 'Whether to enable GKE cost allocation tracking.', +) +GKE_MONITORING_COMPONENTS = flags.DEFINE_string( + 'gke_monitoring_components', + 'SYSTEM,API_SERVER,SCHEDULER,CONTROLLER_MANAGER', + 'Comma-separated list of GKE monitoring components to enable ' + '(e.g. SYSTEM,API_SERVER,SCHEDULER,CONTROLLER_MANAGER,POD,DEPLOYMENT,' + 'STATEFULSET,DAEMONSET,HPA,STORAGE,CADVISOR,KUBELET).', +) +GKE_ENABLE_AGENT_SANDBOX = flags.DEFINE_boolean( + 'gke_enable_agent_sandbox', + False, + 'Whether to enable the GKE Agent Sandbox controller on the cluster. ' + 'Installs the managed agent-sandbox controller and CRDs, enabling ' + 'SandboxClaim/Sandbox/SandboxWarmPool reconciliation by GKE. This is ' + 'separate from the gvisor sandbox runtime (--sandbox=type=gvisor on a ' + 'node pool). Requires GKE 1.35.2-gke.1269000 or later. See ' + 'https://docs.cloud.google.com/kubernetes-engine/docs/how-to/agent-sandbox.', +) + + +def _ValidateGkePrivateNodeFlags(flags_dict): + if ( + not flags_dict['gke_enable_ip_access'] + and not flags_dict['gke_enable_dns_access'] + ): + raise flags.ValidationError( + '--no-gke_enable_ip_access requires --gke_enable_dns_access.' + ) + if ( + flags_dict['gke_enable_private_nodes'] + and not flags_dict['gke_enable_dns_access'] + and not flags_dict['gke_master_ipv4_cidr'] + ): + raise flags.ValidationError( + '--gke_enable_private_nodes without --gke_enable_dns_access requires' + ' --gke_master_ipv4_cidr.' + ) + return True + + +flags.register_multi_flags_validator( + [ + 'gke_enable_ip_access', + 'gke_enable_dns_access', + 'gke_enable_private_nodes', + 'gke_master_ipv4_cidr', + ], + _ValidateGkePrivateNodeFlags, +) GKE_ADDONS = flags.DEFINE_string( 'gke_addons', '', diff --git a/perfkitbenchmarker/providers/gcp/gce_network.py b/perfkitbenchmarker/providers/gcp/gce_network.py index c4c666d478..6ccd004d4a 100644 --- a/perfkitbenchmarker/providers/gcp/gce_network.py +++ b/perfkitbenchmarker/providers/gcp/gce_network.py @@ -914,6 +914,7 @@ class GceNetwork(network.BaseNetwork): def __init__(self, network_spec: GceNetworkSpec): super().__init__(network_spec) self.project: str | None = network_spec.project + self._zone: str = network_spec.zone self.vpn_gateway: Dict[str, GceVpnGateway] = {} # Figuring out the type of network here. @@ -1231,6 +1232,60 @@ def _GetNumberVms(self) -> int: for group_spec in benchmark_spec.config.vm_groups.values() ) + def _CreateCloudNat(self): + """Provision a Cloud Router + NAT so private resources can egress. + + Called during network provisioning so NAT has time to fully propagate + before any cluster lifecycle code starts. Shared across all resources + in the network. + """ + region = util.GetRegionFromZone(self._zone) + router_name = f'{self.primary_subnet_name}-router' + nat_name = f'{self.primary_subnet_name}-nat' + + router_cmd = util.GcloudCommand( + self, 'compute', 'routers', 'create', router_name + ) + router_cmd.flags['network'] = self.primary_subnet_name + router_cmd.flags['region'] = region + router_cmd.flags.pop('zone', None) + _, stderr, retcode = router_cmd.Issue(raise_on_failure=False) + if retcode and 'already exists' not in stderr: + logging.warning('Cloud Router create failed: %s', stderr) + + nat_cmd = util.GcloudCommand( + self, 'compute', 'routers', 'nats', 'create', nat_name + ) + nat_cmd.flags['router'] = router_name + nat_cmd.flags['region'] = region + nat_cmd.flags.pop('zone', None) + nat_cmd.args.append('--auto-allocate-nat-external-ips') + nat_cmd.args.append('--nat-all-subnet-ip-ranges') + _, stderr, retcode = nat_cmd.Issue(raise_on_failure=False) + if retcode and 'already exists' not in stderr: + logging.warning('Cloud NAT create failed: %s', stderr) + + def _DeleteCloudNat(self): + """Best-effort teardown of the NAT and router this network created.""" + region = util.GetRegionFromZone(self._zone) + router_name = f'{self.primary_subnet_name}-router' + nat_name = f'{self.primary_subnet_name}-nat' + + nat_cmd = util.GcloudCommand( + self, 'compute', 'routers', 'nats', 'delete', nat_name + ) + nat_cmd.flags['router'] = router_name + nat_cmd.flags['region'] = region + nat_cmd.flags.pop('zone', None) + nat_cmd.Issue(raise_on_failure=False) + + router_cmd = util.GcloudCommand( + self, 'compute', 'routers', 'delete', router_name + ) + router_cmd.flags['region'] = region + router_cmd.flags.pop('zone', None) + router_cmd.Issue(raise_on_failure=False) + def Create(self): """Creates the actual network.""" if not self.is_existing_network: @@ -1244,6 +1299,8 @@ def Create(self): lambda rule: self.external_nets_rules[rule].Create(), list(self.external_nets_rules.keys()), ) + if gcp_flags.GKE_ENABLE_PRIVATE_NODES.value: + self._CreateCloudNat() if getattr(self, 'vpn_gateway', False): background_tasks.RunThreaded( lambda gateway: self.vpn_gateway[gateway].Create(), @@ -1257,6 +1314,12 @@ def Delete(self): if self.placement_group: self.placement_group.Delete() if not self.is_existing_network: + # Always attempt NAT+router cleanup: both gcloud calls use + # raise_on_failure=False so this is a no-op when none was created. + # Checking the live flag here is wrong because teardown runs from a + # restored pickle and the flag may default to False even when a NAT + # was created at provision time. + self._DeleteCloudNat() if getattr(self, 'vpn_gateway', False): background_tasks.RunThreaded( lambda gateway: self.vpn_gateway[gateway].Delete(), diff --git a/perfkitbenchmarker/providers/gcp/google_kubernetes_engine.py b/perfkitbenchmarker/providers/gcp/google_kubernetes_engine.py index f943a53ff1..9868d8cca9 100644 --- a/perfkitbenchmarker/providers/gcp/google_kubernetes_engine.py +++ b/perfkitbenchmarker/providers/gcp/google_kubernetes_engine.py @@ -168,10 +168,37 @@ def _RunClusterCreateCommand(self, cmd: util.GcloudCommand): ) cmd.flags['release-channel'] = self.release_channel + if gcp_flags.GKE_ENABLE_PRIVATE_NODES.value: + cmd.args.append('--enable-private-nodes') + # GKE requires VPC-native (alias IPs) when private nodes are enabled. + # Without this gcloud rejects the create with: + # Cannot specify --enable-private-nodes without --enable-ip-alias. + cmd.args.append('--enable-ip-alias') + else: + cmd.args.append('--no-enable-private-nodes') + if gcp_flags.GKE_ENABLE_DNS_ACCESS.value: + cmd.args.append('--enable-dns-access') + else: + cmd.args.append('--no-enable-dns-access') + if gcp_flags.GKE_ENABLE_IP_ACCESS.value: + cmd.args.append('--enable-ip-access') + else: + cmd.args.append('--no-enable-ip-access') + if gcp_flags.GKE_ENABLE_DATAPLANE_V2.value: + cmd.args.append('--enable-dataplane-v2') + if gcp_flags.GKE_ENABLE_AGENT_SANDBOX.value: + cmd.args.append('--enable-agent-sandbox') + if gcp_flags.GKE_MASTER_IPV4_CIDR.value: + cmd.flags['master-ipv4-cidr'] = gcp_flags.GKE_MASTER_IPV4_CIDR.value + if FLAGS.gke_enable_alpha: cmd.args.append('--enable-kubernetes-alpha') cmd.args.append('--no-enable-autorepair') - cmd.flags['monitoring'] = 'SYSTEM,API_SERVER,SCHEDULER,CONTROLLER_MANAGER' + cmd.flags['monitoring'] = gcp_flags.GKE_MONITORING_COMPONENTS.value + if gcp_flags.GKE_ENABLE_MANAGED_PROMETHEUS.value: + cmd.args.append('--enable-managed-prometheus') + if gcp_flags.GKE_ENABLE_COST_ALLOCATION.value: + cmd.args.append('--enable-cost-allocation') user = util.GetDefaultUser() if FLAGS.gcp_service_account: @@ -209,6 +236,10 @@ def _GetKubeconfig(self): cmd = self._GcloudCommand( 'container', 'clusters', 'get-credentials', self.name ) + if gcp_flags.GKE_ENABLE_DNS_ACCESS.value: + # Private-node clusters are unreachable via the IP endpoint; use the + # DNS-based control plane endpoint instead. + cmd.args.append('--dns-endpoint') env = os.environ.copy() env['KUBECONFIG'] = FLAGS.kubeconfig cmd.IssueRetryable(env=env) @@ -377,6 +408,10 @@ def _Create(self): cmd = self._GcloudCommand('container', 'clusters', 'create', self.name) if self.default_nodepool.network: cmd.flags['network'] = self.default_nodepool.network.network_resource.name + if self.default_nodepool.network.subnet_resource: + cmd.flags['subnetwork'] = ( + self.default_nodepool.network.subnet_resource.name + ) if gcp_flags.GKE_ENABLE_SHIELDED_NODES.value: cmd.args.append('--enable-shielded-nodes') @@ -576,10 +611,19 @@ def _AddNodeParamsToCmd( if nodepool_config.sandbox_config is not None: cmd.flags['sandbox'] = nodepool_config.sandbox_config.ToSandboxFlag() + if nodepool_config.max_pods_per_node is not None: + cmd.flags['max-pods-per-node'] = nodepool_config.max_pods_per_node + if self.image_type: cmd.flags['image-type'] = self.image_type - cmd.flags['node-labels'] = f'pkb_nodepool={nodepool_config.name}' + labels = {} + if nodepool_config.node_labels: + labels.update(nodepool_config.node_labels) + labels['pkb_nodepool'] = nodepool_config.name + cmd.flags['node-labels'] = ','.join(f'{k}={v}' for k, v in labels.items()) + if nodepool_config.node_taints: + cmd.flags['node-taints'] = ','.join(nodepool_config.node_taints) if nodepool_config.min_nodes != nodepool_config.max_nodes: cmd.args.append('--enable-autoscaling') cmd.flags['min-nodes'] = nodepool_config.min_nodes @@ -673,6 +717,10 @@ def _Create(self): ) if self.default_nodepool.network: cmd.flags['network'] = self.default_nodepool.network.network_resource.name + if self.default_nodepool.network.subnet_resource: + cmd.flags['subnetwork'] = ( + self.default_nodepool.network.subnet_resource.name + ) cmd.flags['labels'] = util.MakeFormattedDefaultTags() if self.enable_aam: diff --git a/perfkitbenchmarker/resources/container_service/container.py b/perfkitbenchmarker/resources/container_service/container.py index 3e05a1ec2b..e6b1e9b02f 100644 --- a/perfkitbenchmarker/resources/container_service/container.py +++ b/perfkitbenchmarker/resources/container_service/container.py @@ -187,6 +187,9 @@ def __init__( # Defined by GceVirtualMachineConfig. Used by google_kubernetes_engine # pylint: disable=g-missing-from-attributes self.sandbox_config: container_spec_lib.SandboxSpec | None = None + self.node_labels: dict[str, str] | None = None + self.node_taints: list[str] | None = None + self.max_pods_per_node: int | None = None self.max_local_disks: int | None self.ssd_interface: str | None self.threads_per_core: int diff --git a/perfkitbenchmarker/resources/container_service/container_cluster.py b/perfkitbenchmarker/resources/container_service/container_cluster.py index 9458662c98..0a99034b48 100644 --- a/perfkitbenchmarker/resources/container_service/container_cluster.py +++ b/perfkitbenchmarker/resources/container_service/container_cluster.py @@ -116,6 +116,9 @@ def _InitializeNodePool( nodepool_spec.machine_families, ) nodepool_config.sandbox_config = nodepool_spec.sandbox_config + nodepool_config.node_labels = nodepool_spec.node_labels + nodepool_config.node_taints = nodepool_spec.node_taints + nodepool_config.max_pods_per_node = nodepool_spec.max_pods_per_node nodepool_config.zone = zone nodepool_config.num_nodes = nodepool_spec.vm_count if nodepool_spec.min_vm_count is None: diff --git a/tests/providers/gcp/google_kubernetes_engine_test.py b/tests/providers/gcp/google_kubernetes_engine_test.py index dbf8232f5e..63810d9287 100644 --- a/tests/providers/gcp/google_kubernetes_engine_test.py +++ b/tests/providers/gcp/google_kubernetes_engine_test.py @@ -887,9 +887,7 @@ def testGetMachineTypeFromNodeName(self): spec = self.create_kubernetes_engine_spec() with self.patch_critical_objects(): cluster = google_kubernetes_engine.GkeAutopilotCluster(spec) - self.MockIssueCommand( - {'get node': [('ek-standard-16', '', 0)]} - ) + self.MockIssueCommand({'get node': [('ek-standard-16', '', 0)]}) self.assertEqual( cluster.GetMachineTypeFromNodeName( 'gke-pkb-cluster-default-pool-node-1' @@ -949,5 +947,88 @@ def testCreateWithPerNodepoolAutoscaling(self): self.assertIn('--max-nodes 10', nodepool_cmd) +class GoogleKubernetesEngineNodeLabelsAndTaintsTestCase(PatchedObjectsTestCase): + + def testNodepoolWithLabelsAndTaintsIncludesBothInGcloudCmd(self): + kubernetes_engine_spec = container_spec.ContainerClusterSpec( + 'NAME', + **{ + 'cloud': 'GCP', + 'vm_spec': { + 'GCP': { + 'machine_type': 'fake-machine-type', + 'zone': 'us-central1-a', + }, + }, + 'vm_count': 1, + 'nodepools': { + 'sandbox': { + 'vm_spec': { + 'GCP': { + 'machine_type': 'n2-standard-8', + }, + }, + 'vm_count': 3, + 'node_labels': {'sandbox.gke.io/runtime': 'runsc'}, + 'node_taints': ['sandbox.gke.io/runtime=runsc:NoSchedule'], + }, + }, + 'poll_for_events': False, + }, + ) + with self.patch_critical_objects() as issue_command: + cluster = google_kubernetes_engine.GkeCluster(kubernetes_engine_spec) + cluster._Create() + + nodepool_cmd = issue_command.GetCommandWithSubstring( + 'node-pools create sandbox' + ) + # pkb_nodepool label must always be present. + self.assertIn('pkb_nodepool=sandbox', nodepool_cmd) + # Custom label must also be present in the same --node-labels value. + self.assertIn('sandbox.gke.io/runtime=runsc', nodepool_cmd) + # Both appear under a single --node-labels flag. + self.assertIn('--node-labels', nodepool_cmd) + # Taint must be present. + self.assertIn( + '--node-taints sandbox.gke.io/runtime=runsc:NoSchedule', nodepool_cmd + ) + + def testNodepoolWithoutLabelsOrTaintsOnlyEmitsPkbNodepoolLabel(self): + kubernetes_engine_spec = container_spec.ContainerClusterSpec( + 'NAME', + **{ + 'cloud': 'GCP', + 'vm_spec': { + 'GCP': { + 'machine_type': 'fake-machine-type', + 'zone': 'us-central1-a', + }, + }, + 'vm_count': 1, + 'nodepools': { + 'default-pool': { + 'vm_spec': { + 'GCP': { + 'machine_type': 'n2-standard-4', + }, + }, + 'vm_count': 2, + }, + }, + 'poll_for_events': False, + }, + ) + with self.patch_critical_objects() as issue_command: + cluster = google_kubernetes_engine.GkeCluster(kubernetes_engine_spec) + cluster._Create() + + nodepool_cmd = issue_command.GetCommandWithSubstring( + 'node-pools create default-pool' + ) + self.assertIn('--node-labels pkb_nodepool=default-pool', nodepool_cmd) + self.assertNotIn('--node-taints', nodepool_cmd) + + if __name__ == '__main__': unittest.main()