Skip to content

Commit a69d6be

Browse files
committed
fix: allow RHEL host subscriptions with custom driver repo config
Signed-off-by: Rahul Sharma <rahulsharm@nvidia.com>
1 parent 3fc63e2 commit a69d6be

18 files changed

Lines changed: 563 additions & 29 deletions

api/nvidia/v1/clusterpolicy_types.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1280,6 +1280,12 @@ type DriverRepoConfigSpec struct {
12801280
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.displayName="ConfigMap Name"
12811281
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.x-descriptors="urn:alm:descriptor:com.tectonic.ui:text"
12821282
ConfigMapName string `json:"configMapName,omitempty"`
1283+
1284+
// UseHostSubscription mounts the host RHEL subscription configuration with this custom repo configuration.
1285+
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true
1286+
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.displayName="Use Host RHEL Subscription"
1287+
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.x-descriptors="urn:alm:descriptor:com.tectonic.ui:booleanSwitch"
1288+
UseHostSubscription bool `json:"useHostSubscription,omitempty"`
12831289
}
12841290

12851291
// DriverCertConfigSpec defines custom certificates configuration for NVIDIA Driver container

api/nvidia/v1alpha1/nvidiadriver_types.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -453,6 +453,12 @@ type DriverRepoConfigSpec struct {
453453
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.displayName="ConfigMap Name"
454454
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.x-descriptors="urn:alm:descriptor:com.tectonic.ui:text"
455455
Name string `json:"name,omitempty"`
456+
457+
// UseHostSubscription mounts the host RHEL subscription configuration with this custom repo configuration.
458+
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true
459+
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.displayName="Use Host RHEL Subscription"
460+
// +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.x-descriptors="urn:alm:descriptor:com.tectonic.ui:booleanSwitch"
461+
UseHostSubscription bool `json:"useHostSubscription,omitempty"`
456462
}
457463

458464
// DriverLicensingConfigSpec defines licensing server configuration for NVIDIA Driver container

bundle/manifests/gpu-operator-certified.clusterserviceversion.yaml

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -346,6 +346,11 @@ spec:
346346
path: imagePullPolicy
347347
x-descriptors:
348348
- 'urn:alm:descriptor:com.tectonic.ui:imagePullPolicy'
349+
- displayName: Use Host RHEL Subscription
350+
description: Mount the host RHEL subscription configuration with this custom repo configuration.
351+
path: repoConfig.useHostSubscription
352+
x-descriptors:
353+
- 'urn:alm:descriptor:com.tectonic.ui:booleanSwitch'
349354
- displayName: UseOpenKernelModules
350355
description: 'Enable use of open GPU kernel modules'
351356
path: useOpenKernelModules
@@ -488,6 +493,11 @@ spec:
488493
path: driver.imagePullPolicy
489494
x-descriptors:
490495
- 'urn:alm:descriptor:com.tectonic.ui:imagePullPolicy'
496+
- displayName: Use Host RHEL Subscription
497+
description: Mount the host RHEL subscription configuration with this custom repo configuration.
498+
path: driver.repoConfig.useHostSubscription
499+
x-descriptors:
500+
- 'urn:alm:descriptor:com.tectonic.ui:booleanSwitch'
491501
- displayName: UseOpenKernelModules
492502
description: 'Enable use of open GPU kernel modules'
493503
path: driver.useOpenKernelModules

bundle/manifests/nvidia.com_clusterpolicies.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1137,6 +1137,10 @@ spec:
11371137
properties:
11381138
configMapName:
11391139
type: string
1140+
useHostSubscription:
1141+
description: UseHostSubscription mounts the host RHEL subscription
1142+
configuration with this custom repo configuration.
1143+
type: boolean
11401144
type: object
11411145
repository:
11421146
description: NVIDIA Driver image repository

bundle/manifests/nvidia.com_nvidiadrivers.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -834,6 +834,10 @@ spec:
834834
properties:
835835
name:
836836
type: string
837+
useHostSubscription:
838+
description: UseHostSubscription mounts the host RHEL subscription
839+
configuration with this custom repo configuration.
840+
type: boolean
837841
type: object
838842
repository:
839843
description: NVIDIA Driver repository

config/crd/bases/nvidia.com_clusterpolicies.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1137,6 +1137,10 @@ spec:
11371137
properties:
11381138
configMapName:
11391139
type: string
1140+
useHostSubscription:
1141+
description: UseHostSubscription mounts the host RHEL subscription
1142+
configuration with this custom repo configuration.
1143+
type: boolean
11401144
type: object
11411145
repository:
11421146
description: NVIDIA Driver image repository

config/crd/bases/nvidia.com_nvidiadrivers.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -834,6 +834,10 @@ spec:
834834
properties:
835835
name:
836836
type: string
837+
useHostSubscription:
838+
description: UseHostSubscription mounts the host RHEL subscription
839+
configuration with this custom repo configuration.
840+
type: boolean
837841
type: object
838842
repository:
839843
description: NVIDIA Driver repository

config/manifests/bases/gpu-operator.clusterserviceversion.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -201,6 +201,11 @@ spec:
201201
path: driver.imagePullPolicy
202202
x-descriptors:
203203
- 'urn:alm:descriptor:com.tectonic.ui:imagePullPolicy'
204+
- displayName: Use Host RHEL Subscription
205+
description: Mount the host RHEL subscription configuration with this custom repo configuration.
206+
path: driver.repoConfig.useHostSubscription
207+
x-descriptors:
208+
- 'urn:alm:descriptor:com.tectonic.ui:booleanSwitch'
204209
- description: Group Feature Discovery Plugin config
205210
displayName: Group Feature Discovery Plugin config
206211
path: gfd

controllers/object_controls.go

Lines changed: 73 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424
"path"
2525
"path/filepath"
2626
"regexp"
27+
"slices"
2728
"sort"
2829
"strconv"
2930
"strings"
@@ -3563,6 +3564,10 @@ func applyLicensingConfig(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpec
35633564

35643565
func transformDriverContainer(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicySpec, n ClusterPolicyController) error {
35653566
podSpec := &obj.Spec.Template.Spec
3567+
if config.Driver.RepoConfig != nil && config.Driver.RepoConfig.ConfigMapName != "" &&
3568+
config.Driver.CertConfig != nil && config.Driver.CertConfig.Name == config.Driver.RepoConfig.ConfigMapName {
3569+
return fmt.Errorf("repoConfig and certConfig must reference different ConfigMaps")
3570+
}
35663571
driverContainer := findContainerByName(podSpec.Containers, "nvidia-driver-ctr")
35673572
if driverContainer == nil {
35683573
return fmt.Errorf("driver container (nvidia-driver-ctr) is missing from the driver daemonset manifest")
@@ -3720,6 +3725,7 @@ func transformDriverContainer(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicy
37203725
if err != nil {
37213726
return fmt.Errorf("ERROR: failed to create ConfigMap VolumeMounts for custom repo config: %v", err)
37223727
}
3728+
removeConfigMapVolumeAndMounts(podSpec, config.Driver.RepoConfig.ConfigMapName)
37233729
driverContainer.VolumeMounts = append(driverContainer.VolumeMounts, volumeMounts...)
37243730
podSpec.Volumes = append(podSpec.Volumes, createConfigMapVolume(config.Driver.RepoConfig.ConfigMapName, itemsToInclude))
37253731
}
@@ -3734,6 +3740,7 @@ func transformDriverContainer(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicy
37343740
if err != nil {
37353741
return fmt.Errorf("ERROR: failed to create ConfigMap VolumeMounts for custom certs: %w", err)
37363742
}
3743+
removeConfigMapVolumeAndMounts(podSpec, config.Driver.CertConfig.Name)
37373744
driverContainer.VolumeMounts = append(driverContainer.VolumeMounts, volumeMounts...)
37383745
podSpec.Volumes = append(podSpec.Volumes, createConfigMapVolume(config.Driver.CertConfig.Name, itemsToInclude))
37393746
}
@@ -3749,8 +3756,9 @@ func transformDriverContainer(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicy
37493756
// set up subscription entitlements for RHEL(using K8s with a non-CRIO runtime) and SLES
37503757
if (osID == "rhel" && n.openshift == "" && n.runtime != gpuv1.CRIO) || osID == "sles" || osID == "sl-micro" {
37513758
pathToVolumeSource := MountPathToVolumeSource{}
3752-
if config.Driver.RepoConfig != nil && config.Driver.RepoConfig.ConfigMapName != "" && osID == "rhel" {
3753-
n.logger.Info("Skipping host subscription mounts because repoConfig is enabled", "OS", osID)
3759+
if config.Driver.RepoConfig != nil && config.Driver.RepoConfig.ConfigMapName != "" &&
3760+
!config.Driver.RepoConfig.UseHostSubscription && osID == "rhel" {
3761+
n.logger.Info("Skipping host subscription mounts because repoConfig is enabled and useHostSubscription is false", "OS", osID)
37543762
} else {
37553763
n.logger.Info("Mounting subscriptions into the driver container", "OS", osID)
37563764
pathToVolumeSource, err = n.getSubscriptionPathsToVolumeSources()
@@ -3766,8 +3774,15 @@ func transformDriverContainer(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicy
37663774
}
37673775
sort.Strings(mountPaths)
37683776

3769-
for num, mountPath := range mountPaths {
3770-
volMountSubscriptionName := fmt.Sprintf("subscription-config-%d", num)
3777+
removeSubscriptionMountsAndVolumes(driverContainer, podSpec)
3778+
3779+
usedVolumeNames := make(map[string]struct{}, len(podSpec.Volumes)+len(mountPaths))
3780+
for _, volume := range podSpec.Volumes {
3781+
usedVolumeNames[volume.Name] = struct{}{}
3782+
}
3783+
volumeIndex := 0
3784+
for _, mountPath := range mountPaths {
3785+
volMountSubscriptionName := nextSubscriptionVolumeName(usedVolumeNames, &volumeIndex)
37713786

37723787
volMountSubscription := corev1.VolumeMount{
37733788
Name: volMountSubscriptionName,
@@ -3795,6 +3810,60 @@ func transformDriverContainer(obj *appsv1.DaemonSet, config *gpuv1.ClusterPolicy
37953810
return nil
37963811
}
37973812

3813+
// removeConfigMapVolumeAndMounts removes an operator-managed ConfigMap volume and its mounts before rebuilding it.
3814+
func removeConfigMapVolumeAndMounts(podSpec *corev1.PodSpec, configMapName string) {
3815+
configMapVolumeNames := map[string]struct{}{}
3816+
podSpec.Volumes = slices.DeleteFunc(podSpec.Volumes, func(volume corev1.Volume) bool {
3817+
if volume.Name != configMapName || volume.ConfigMap == nil || volume.ConfigMap.Name != configMapName {
3818+
return false
3819+
}
3820+
configMapVolumeNames[volume.Name] = struct{}{}
3821+
return true
3822+
})
3823+
for containerIndex := range podSpec.Containers {
3824+
podSpec.Containers[containerIndex].VolumeMounts = slices.DeleteFunc(podSpec.Containers[containerIndex].VolumeMounts, func(volumeMount corev1.VolumeMount) bool {
3825+
_, found := configMapVolumeNames[volumeMount.Name]
3826+
return found
3827+
})
3828+
}
3829+
}
3830+
3831+
// nextSubscriptionVolumeName returns an unused subscription volume name and records it as used.
3832+
func nextSubscriptionVolumeName(usedVolumeNames map[string]struct{}, volumeIndex *int) string {
3833+
for {
3834+
volumeName := fmt.Sprintf("%s%d", consts.SubscriptionVolumeNamePrefix, *volumeIndex)
3835+
*volumeIndex += 1
3836+
if _, found := usedVolumeNames[volumeName]; found {
3837+
continue
3838+
}
3839+
usedVolumeNames[volumeName] = struct{}{}
3840+
return volumeName
3841+
}
3842+
}
3843+
3844+
// removeSubscriptionMountsAndVolumes removes host subscription volumes and their mounts before rebuilding them.
3845+
// It removes only volumes with the subscription prefix that are backed by hostPath, then removes only
3846+
// mounts that reference those volumes. ConfigMap-backed volumes with the same prefix are preserved.
3847+
func removeSubscriptionMountsAndVolumes(driverContainer *corev1.Container, podSpec *corev1.PodSpec) {
3848+
subscriptionVolumeNames := map[string]struct{}{}
3849+
podSpec.Volumes = slices.DeleteFunc(podSpec.Volumes, func(volume corev1.Volume) bool {
3850+
if !isHostSubscriptionVolume(volume) {
3851+
return false
3852+
}
3853+
subscriptionVolumeNames[volume.Name] = struct{}{}
3854+
return true
3855+
})
3856+
driverContainer.VolumeMounts = slices.DeleteFunc(driverContainer.VolumeMounts, func(volumeMount corev1.VolumeMount) bool {
3857+
_, found := subscriptionVolumeNames[volumeMount.Name]
3858+
return found
3859+
})
3860+
}
3861+
3862+
// isHostSubscriptionVolume reports whether a volume is managed for host subscriptions.
3863+
func isHostSubscriptionVolume(volume corev1.Volume) bool {
3864+
return strings.HasPrefix(volume.Name, consts.SubscriptionVolumeNamePrefix) && volume.HostPath != nil
3865+
}
3866+
37983867
func createSecretEnvReference(ctx context.Context, ctrlClient client.Client, secretName string,
37993868
namespace string, container *corev1.Container) error {
38003869
envFrom := container.EnvFrom

0 commit comments

Comments
 (0)