Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
96 changes: 96 additions & 0 deletions perfkitbenchmarker/configs/container_spec.py
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,7 @@ def __init__(
self.vm_spec: virtual_machine_spec.BaseVmSpec
self.machine_families: list[str] | None
self.sandbox_config: SandboxSpec | None
self.swap_config: SwapConfigSpec | None

@classmethod
def _GetOptionDecoderConstructions(cls):
Expand Down Expand Up @@ -273,6 +274,7 @@ def _GetOptionDecoderConstructions(cls):
),
'vm_spec': (spec.PerCloudConfigDecoder, {}),
'sandbox_config': (_SandboxDecoder, {'default': None}),
'swap_config': (_SwapConfigDecoder, {'default': None}),
})
return result

Expand Down Expand Up @@ -333,6 +335,100 @@ def Decode(self, value, component_full_name, flag_values):
return result


class SwapConfigSpec(spec.BaseSpec):
"""Configurable swap options for a GKE/EKS nodepool.

Declared in BENCHMARK_CONFIG under nodepools.<name>.swap_config.
Consumed by the cloud provider's _AddNodeParamsToCmd() / equivalent to
apply the cloud-specific swap configuration during nodepool creation.

Attributes:
enabled: Whether to enable swap on the nodepool (default True).
swappiness: vm.swappiness sysctl value (0-200, default 100).
min_free_kbytes: vm.min_free_kbytes sysctl (default 200).
watermark_scale_factor: vm.watermark_scale_factor sysctl (default 500).
lssd: True if the nodepool uses local NVMe SSDs for the swap device.
lssd_count: Number of local NVMe SSDs (GKE dedicatedLocalSsdProfile).
boot_disk_iops: Provisioned IOPS for hyperdisk-balanced (0 = not set).
boot_disk_throughput: Provisioned throughput MiB/s for hyperdisk-balanced.
"""

def __init__(self, *args, **kwargs):
self.enabled: bool = True
self.swappiness: int = 100
self.min_free_kbytes: int = 200
self.watermark_scale_factor: int = 500
self.lssd: bool = False
self.lssd_count: int = 0
self.boot_disk_iops: int = 0
self.boot_disk_throughput: int = 0
super().__init__(*args, **kwargs)

@classmethod
def _GetOptionDecoderConstructions(cls):
result = super()._GetOptionDecoderConstructions()
result.update({
'enabled': (
option_decoders.BooleanDecoder,
{'default': True},
),
'swappiness': (
option_decoders.IntDecoder,
{'default': 100, 'min': 0, 'max': 200},
),
'min_free_kbytes': (
option_decoders.IntDecoder,
{'default': 200, 'min': 0},
),
'watermark_scale_factor': (
option_decoders.IntDecoder,
{'default': 500, 'min': 0},
),
'lssd': (
option_decoders.BooleanDecoder,
{'default': False},
),
'lssd_count': (
option_decoders.IntDecoder,
{'default': 0, 'min': 0},
),
'boot_disk_iops': (
option_decoders.IntDecoder,
{'default': 0, 'min': 0},
),
'boot_disk_throughput': (
option_decoders.IntDecoder,
{'default': 0, 'min': 0},
),
})
return result


class _SwapConfigDecoder(option_decoders.TypeVerifier):
"""Decodes the swap_config option of a NodepoolSpec."""

def Decode(self, value, component_full_name, flag_values):
"""Decodes the swap_config dictionary into a SwapConfigSpec.

Args:
value: dict. Keys match SwapConfigSpec._GetOptionDecoderConstructions.
component_full_name: str. Fully qualified name of the parent component.
flag_values: flags.FlagValues. Runtime flags propagated to BaseSpec.

Returns:
SwapConfigSpec instance.

Raises:
errors.Config.InvalidValue upon invalid input value.
"""
super().Decode(value, component_full_name, flag_values)
return SwapConfigSpec(
self._GetOptionFullName(component_full_name),
flag_values=flag_values,
**value,
)


class SandboxSpec(spec.BaseSpec):
"""Configurable options for sandboxed node pools."""

Expand Down
120 changes: 120 additions & 0 deletions perfkitbenchmarker/data/cluster/swap_encryption_daemonset.yaml.j2
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: {{ ds_name }}
namespace: {{ ds_namespace }}
labels:
app: {{ ds_label }}
spec:
selector:
matchLabels:
app: {{ ds_label }}
template:
metadata:
labels:
app: {{ ds_label }}
spec:
hostPID: true
hostNetwork: true
# Pin to the benchmark nodepool — never schedule on the dummy default pool.
nodeSelector:
pkb_nodepool: {{ benchmark_nodepool }}
tolerations:
- operator: Exists
containers:
- name: benchmark
image: {{ image }}
command:
- bash
- -c
- |
echo "[pkb] Installing measurement tools..."
# Only the tools needed for Phase 1 (raw-device fio) and Phase 2
# (CPU/I/O overhead) are installed here. Workload benchmarks
# (redis, opensearch, kernel-build) run in separate pods via
# existing PKB benchmark modules and are NOT installed here.
PKB_APT_OK=0
for _attempt in 1 2 3; do
apt-get update -qq 2>&1 || true
DEBIAN_FRONTEND=noninteractive apt-get install -y -qq \
fio \
cryptsetup \
mdadm \
sysstat \
nvme-cli \
2>&1 && PKB_APT_OK=1 && break
echo "[pkb] apt-get attempt $_attempt failed, retrying in 15s..." >&2
sleep 15
done
if [ "$PKB_APT_OK" != "1" ] || ! command -v fio >/dev/null 2>&1; then
echo "[pkb] FATAL: fio not installed after 3 attempts" >&2
exit 1
fi
echo "[pkb] fio installed: $(fio --version 2>&1 | head -1)"
echo "[pkb] Verifying swap device is active..."
PKB_SWAP_FOUND=0
for _attempt in $(seq 1 30); do
if awk 'NR>1{found=1} END{exit !found}' /proc/swaps 2>/dev/null; then
PKB_SWAP_DEV=$(awk 'NR==2{print $1}' /proc/swaps)
echo "[pkb] Swap device active: $PKB_SWAP_DEV"
PKB_SWAP_FOUND=1
break
fi
echo "[pkb] Waiting for swap device (attempt $_attempt/30)..." >&2
sleep 5
done
if [ "$PKB_SWAP_FOUND" != "1" ]; then
echo "[pkb] WARNING: no active swap device after 150s — " \
"check linuxConfig.swapConfig / kubelet swap config." >&2
fi
echo "[pkb] Measurement tools ready. Writing ready sentinel."
touch /tmp/pkb_ready
sleep infinity
securityContext:
privileged: true
capabilities:
add: ["SYS_ADMIN", "IPC_LOCK"]
resources:
requests:
memory: "512Mi"
env:
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
volumeMounts:
- name: dev
mountPath: /dev
- name: sys
mountPath: /sys
- name: run
mountPath: /run
- name: proc-host
mountPath: /proc-host
readOnly: true
- name: stateful-partition
mountPath: /mnt/stateful_partition
- name: lib-modules
mountPath: /lib/modules
readOnly: true
volumes:
- name: dev
hostPath:
path: /dev
- name: sys
hostPath:
path: /sys
- name: run
hostPath:
path: /run
- name: proc-host
hostPath:
path: /proc
- name: stateful-partition
hostPath:
path: /mnt/stateful_partition
type: DirectoryOrCreate
- name: lib-modules
hostPath:
path: /lib/modules
type: Directory
Loading