feat: Route workloads to city locations via distributed scheduling

.github/workflows/publish.yaml

@@ -18,6 +18,7 @@ jobs:
     secrets: inherit
 
   publish-kustomize-bundles:
+    needs: publish-container-image
     permissions:
       id-token: write
       contents: read
@@ -26,4 +27,6 @@ jobs:
     with:
       bundle-name: ghcr.io/datum-cloud/compute-kustomize
       bundle-path: config
+      image-name: ghcr.io/datum-cloud/compute
+      image-overlays: config/base/manager
     secrets: inherit

.gitignore

@@ -25,3 +25,6 @@ go.work.sum
 .env
 
 bin/
+
+# Local e2e environment artefacts (Kind kubeconfigs, etc.)
+tmp/

api/v1alpha/workloaddeployment_types.go

@@ -2,8 +2,6 @@ package v1alpha
 
 import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
-	networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha"
 )
 
 // WorkloadDeploymentSpec defines the desired state of WorkloadDeployment
@@ -37,11 +35,6 @@ type WorkloadDeploymentSpec struct {
 
 // WorkloadDeploymentStatus defines the observed state of WorkloadDeployment
 type WorkloadDeploymentStatus struct {
-	// The location which the deployment has been scheduled to
-	//
-	// +kubebuilder:validation:Optional
-	Location *networkingv1alpha.LocationReference `json:"location,omitempty"`
-
 	// Represents the observations of a deployment's current state.
 	// Known condition types are: "Available", "Progressing"
 	Conditions []metav1.Condition `json:"conditions,omitempty"`
@@ -80,8 +73,6 @@ const (
 // +kubebuilder:printcolumn:name="Ready",type=string,JSONPath=`.status.readyReplicas`
 // +kubebuilder:printcolumn:name="Desired",type=string,JSONPath=`.status.desiredReplicas`
 // +kubebuilder:printcolumn:name="Up-to-date",type=string,JSONPath=`.status.currentReplicas`
-// +kubebuilder:printcolumn:name="Location Namespace",type=string,JSONPath=`.status.location.namespace`,priority=1
-// +kubebuilder:printcolumn:name="Location Name",type=string,JSONPath=`.status.location.name`,priority=1
 type WorkloadDeployment struct {
 	metav1.TypeMeta   `json:",inline"`
 	metav1.ObjectMeta `json:"metadata,omitempty"`

cmd/main.go

@@ -18,17 +18,22 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/serializer"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/cluster"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
+	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
 	mcmanager "sigs.k8s.io/multicluster-runtime/pkg/manager"
 	"sigs.k8s.io/multicluster-runtime/pkg/multicluster"
 	mcsingle "sigs.k8s.io/multicluster-runtime/providers/single"
 
+	karmadaclusterv1alpha1 "github.com/karmada-io/api/cluster/v1alpha1"
+	karmadapolicyv1alpha1 "github.com/karmada-io/api/policy/v1alpha1"
 	computev1alpha "go.datum.net/compute/api/v1alpha"
 	"go.datum.net/compute/internal/config"
 	"go.datum.net/compute/internal/controller"
@@ -51,6 +56,11 @@ var (
 	gitCommit    = "unknown"
 	gitTreeState = "unknown"
 	buildDate    = "unknown"
+
+	// downstreamRestConfig holds the REST config for the downstream control plane.
+	// It is populated from --downstream-kubeconfig when set, and is nil when the
+	// flag is omitted (e.g. in non-federation deployments).
+	downstreamRestConfig *rest.Config
 )
 
 func init() {
@@ -61,6 +71,8 @@ func init() {
 	utilruntime.Must(computev1alpha.AddToScheme(scheme))
 	utilruntime.Must(networkingv1alpha.AddToScheme(scheme))
 	utilruntime.Must(quotav1alpha1.AddToScheme(scheme))
+	utilruntime.Must(karmadapolicyv1alpha1.Install(scheme))
+	utilruntime.Must(karmadaclusterv1alpha1.Install(scheme))
 
 	// +kubebuilder:scaffold:scheme
 }
@@ -71,12 +83,27 @@ func main() {
 	var leaderElectionNamespace string
 	var probeAddr string
 	var serverConfigFile string
+	var downstreamKubeconfig string
+	var downstreamContext string
+	var enableManagementControllers bool
+	var enableCellControllers bool
 
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
 		"Enable leader election for controller manager. "+
 			"Enabling this will ensure there is only one active controller manager.")
 	flag.StringVar(&leaderElectionNamespace, "leader-elect-namespace", "", "The namespace to use for leader election.")
+	flag.StringVar(&downstreamKubeconfig, "downstream-kubeconfig", "",
+		"Path to the kubeconfig file for the downstream control plane. "+
+			"When omitted, downstream federation features are disabled.")
+	flag.StringVar(&downstreamContext, "downstream-context", "",
+		"Context to use from the downstream kubeconfig. When omitted, the current context is used.")
+	flag.BoolVar(&enableManagementControllers, "enable-management-controllers", true,
+		"Enable management-plane controllers (WorkloadDeploymentFederator, InstanceProjector). "+
+			"Disable when running a cell-only operator instance.")
+	flag.BoolVar(&enableCellControllers, "enable-cell-controllers", true,
+		"Enable cell controllers (WorkloadDeploymentReconciler, InstanceReconciler). "+
+			"Disable when running a management-only operator instance.")
 
 	opts := zap.Options{
 		Development: true,
@@ -89,6 +116,23 @@ func main() {
 
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
 
+	// Load the downstream REST config when --downstream-kubeconfig is provided.
+	// When the flag is omitted, downstreamRestConfig remains nil and federation
+	// features will be skipped at controller setup time.
+	if downstreamKubeconfig != "" {
+		loader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
+			&clientcmd.ClientConfigLoadingRules{ExplicitPath: downstreamKubeconfig},
+			&clientcmd.ConfigOverrides{CurrentContext: downstreamContext},
+		)
+		var err error
+		downstreamRestConfig, err = loader.ClientConfig()
+		if err != nil {
+			setupLog.Error(err, "unable to load downstream kubeconfig", "path", downstreamKubeconfig)
+			os.Exit(1)
+		}
+		setupLog.Info("downstream kubeconfig loaded", "path", downstreamKubeconfig)
+	}
+
 	setupLog.Info("starting compute",
 		"version", version,
 		"gitCommit", gitCommit,
@@ -180,17 +224,63 @@ func main() {
 		setupLog.Error(err, "unable to create controller", "controller", "Workload")
 		os.Exit(1)
 	}
-	if err = (&controller.WorkloadDeploymentReconciler{}).SetupWithManager(mgr); err != nil {
-		setupLog.Error(err, "unable to create controller", "controller", "WorkloadDeployment")
-		os.Exit(1)
+
+	// Build a single downstream client shared across all controllers that need
+	// to read or write to the downstream control plane. Nil when federation is disabled.
+	var downstreamClient client.Client
+	if downstreamRestConfig != nil {
+		downstreamClient, err = client.New(downstreamRestConfig, client.Options{Scheme: scheme})
+		if err != nil {
+			setupLog.Error(err, "unable to create downstream client")
+			os.Exit(1)
+		}
 	}
-	if err = (&controller.WorkloadDeploymentScheduler{}).SetupWithManager(mgr); err != nil {
-		setupLog.Error(err, "unable to create controller", "controller", "WorkloadDeploymentScheduler")
-		os.Exit(1)
+
+	if enableCellControllers {
+		if err = (&controller.WorkloadDeploymentReconciler{}).SetupWithManager(mgr); err != nil {
+			setupLog.Error(err, "unable to create controller", "controller", "WorkloadDeployment")
+			os.Exit(1)
+		}
 	}
-	if err = (&controller.InstanceReconciler{}).SetupWithManager(mgr, deploymentCluster); err != nil {
-		setupLog.Error(err, "unable to create controller", "controller", "Instance")
-		os.Exit(1)
+
+	if enableCellControllers {
+		instanceReconciler := &controller.InstanceReconciler{DownstreamClient: downstreamClient}
+		if err = instanceReconciler.SetupWithManager(mgr, deploymentCluster); err != nil {
+			setupLog.Error(err, "unable to create controller", "controller", "Instance")
+			os.Exit(1)
+		}
+	}
+
+	// WorkloadDeploymentFederator and InstanceProjector are management-plane
+	// controllers that run on the control-plane cluster. They require a downstream
+	// control plane to be configured (--downstream-kubeconfig provided).
+	if enableManagementControllers && downstreamRestConfig != nil {
+		federator := &controller.WorkloadDeploymentFederator{DownstreamClient: downstreamClient}
+		if err = federator.SetupWithManager(mgr); err != nil {
+			setupLog.Error(err, "unable to create controller", "controller", "WorkloadDeploymentFederator")
+			os.Exit(1)
+		}
+
+		// InstanceProjector: runs in the Control Plane Cell, watches Instances
+		// written back to the downstream control plane by POP-cell operators, and
+		// projects them into the corresponding project namespaces via the
+		// multicluster manager.
+		downstreamMgr, err := manager.New(downstreamRestConfig, manager.Options{
+			Scheme:  scheme,
+			Metrics: metricsserver.Options{BindAddress: "0"},
+		})
+		if err != nil {
+			setupLog.Error(err, "unable to create downstream manager for InstanceProjector")
+			os.Exit(1)
+		}
+		if err = (&controller.InstanceProjector{
+			DownstreamClient: downstreamClient,
+			MCManager:        mgr,
+		}).SetupWithManager(downstreamMgr); err != nil {
+			setupLog.Error(err, "unable to create controller", "controller", "InstanceProjector")
+			os.Exit(1)
+		}
+		runnables = append(runnables, downstreamMgr)
 	}
 
 	if serverConfig.WebhookServer != nil {
@@ -284,6 +374,7 @@ func initializeClusterDiscovery(
 		}
 
 		discoveryManager, err := manager.New(discoveryRestConfig, manager.Options{
+			Metrics: metricsserver.Options{BindAddress: "0"},
 			Client: client.Options{
 				Cache: &client.CacheOptions{
 					Unstructured: true,

config/base/downstream-rbac/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - rbac.yaml

config/base/downstream-rbac/rbac.yaml

@@ -0,0 +1,32 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: compute-manager
+rules:
+  - apiGroups: ["compute.datumapis.com"]
+    resources: ["workloaddeployments", "workloaddeployments/status", "instances", "instances/status"]
+    verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+  - apiGroups: ["policy.karmada.io"]
+    resources: ["propagationpolicies", "clusterpropagationpolicies"]
+    verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+  - apiGroups: ["cluster.karmada.io"]
+    resources: ["clusters"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["work.karmada.io"]
+    resources: ["resourcebindings", "clusterresourcebindings"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["config.karmada.io"]
+    resources: ["resourceinterpreterwebhookconfigurations", "resourceinterpretercustomizations"]
+    verbs: ["get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: compute-manager
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: compute-manager
+subjects:
+  - kind: User
+    name: system:serviceaccount:compute-system:compute-manager

config/base/federation/kustomization.yaml

@@ -0,0 +1,10 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - ../crd/bases/compute.datumapis.com_instances.yaml
+  - ../crd/bases/compute.datumapis.com_workloaddeployments.yaml
+  - ../crd/bases/compute.datumapis.com_workloads.yaml
+
+components:
+  - ../../components/federation

config/base/manager/manager.yaml

@@ -26,14 +26,30 @@ spec:
         seccompProfile:
           type: RuntimeDefault
       containers:
-      - command:
+      - name: manager
+        command:
         - /manager
         args:
-          - --leader-elect
-          - --health-probe-bind-address=:8081
-          - --server-config=/config/config.yaml
+          - --leader-elect=$(LEADER_ELECT)
+          - --health-probe-bind-address=$(HEALTH_PROBE_BIND_ADDRESS)
+          - --server-config=$(SERVER_CONFIG)
+          - --downstream-kubeconfig=$(DOWNSTREAM_KUBECONFIG)
+          - --enable-management-controllers=$(ENABLE_MANAGEMENT_CONTROLLERS)
+          - --enable-cell-controllers=$(ENABLE_CELL_CONTROLLERS)
+        env:
+          - name: LEADER_ELECT
+            value: "true"
+          - name: HEALTH_PROBE_BIND_ADDRESS
+            value: ":8081"
+          - name: SERVER_CONFIG
+            value: /config/config.yaml
+          - name: DOWNSTREAM_KUBECONFIG
+            value: ""
+          - name: ENABLE_MANAGEMENT_CONTROLLERS
+            value: "false"
+          - name: ENABLE_CELL_CONTROLLERS
+            value: "false"
         image: ghcr.io/datum-cloud/compute:latest
-        name: manager
         ports:
         - containerPort: 9443
           name: webhook-server
@@ -69,7 +85,7 @@ spec:
         - name: webhook-cert
           mountPath: /tmp/k8s-webhook-server/serving-certs
           readOnly: true
-      serviceAccountName: compute
+      serviceAccountName: compute-manager
       terminationGracePeriodSeconds: 10
       volumes:
       - name: config

config/base/manager/service_account.yaml

@@ -4,4 +4,4 @@ metadata:
   labels:
     app.kubernetes.io/name: compute
     app.kubernetes.io/managed-by: kustomize
-  name: compute
+  name: compute-manager

config/components/cell-controllers/kustomization.yaml

@@ -0,0 +1,20 @@
+apiVersion: kustomize.config.k8s.io/v1alpha1
+kind: Component
+
+patches:
+  - target:
+      kind: Deployment
+      name: compute-manager
+    patch: |-
+      apiVersion: apps/v1
+      kind: Deployment
+      metadata:
+        name: compute-manager
+      spec:
+        template:
+          spec:
+            containers:
+              - name: manager
+                env:
+                  - name: ENABLE_CELL_CONTROLLERS
+                    value: "true"

config/components/controller_rbac/metrics_auth_role_binding.yaml

@@ -8,4 +8,4 @@ roleRef:
   name: compute-metrics-auth-role
 subjects:
 - kind: ServiceAccount
-  name: compute
+  name: compute-manager

config/components/controller_rbac/role_binding.yaml

@@ -11,4 +11,4 @@ roleRef:
   name: compute
 subjects:
 - kind: ServiceAccount
-  name: compute
+  name: compute-manager

config/components/federation/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1alpha1
+kind: Component
+
+resources:
+  - workloaddeployment-interpreter.yaml