[WIP] Spellbooks for Xelon

workflows/cloudscale/decommission-steps.yml

@@ -79,6 +79,11 @@ spells:
     HIERADATA_REPO_TOKEN=${INPUT_gitlab_api_token}
     EOF
 
+    if [[ ! -d catalog ]]
+    then
+      commodore catalog compile "$INPUT_commodore_cluster_id"
+    fi
+
     tf_image=$(\
       yq eval ".parameters.openshift4_terraform.images.terraform.image" \
       dependencies/openshift4-terraform/class/defaults.yml)
@@ -195,7 +200,10 @@ spells:
       "-backend-config=unlock_method=DELETE" \
       "-backend-config=retry_wait_min=5"
 
-    terraform state rm "module.cluster.module.lb.module.hiera[0].gitfile_checkout.appuio_hieradata"
+    if terraform state list | grep "gitfile_checkout.appuio_hieradata" > /dev/null
+    then
+      terraform state rm "module.cluster.module.lb.module.hiera[0].gitfile_checkout.appuio_hieradata"
+    fi
 
     # Suppress errors on the first run; it is expected to fail
     terraform destroy --auto-approve || true

workflows/cloudscale/image-steps.yml

@@ -115,7 +115,7 @@ spells:
       "$(echo "$INPUT_bucket_user" | jq -r '.keys[0].access_key')" \
       "$(echo "$INPUT_bucket_user" | jq -r '.keys[0].secret_key')"
 
-    echo "Uploading Red Hat CoreOS image '$INPUT_image_path' to S3 bucket '${INPUT_commodore_cluster_id}-image-registry'..."
+    echo "Uploading Red Hat CoreOS image '$INPUT_image_path' to S3 bucket '${INPUT_commodore_cluster_id}-bootstrap-ignition'..."
     mc cp "rhcos-${INPUT_image_major}.${INPUT_image_minor}.qcow2" "${INPUT_commodore_cluster_id}/${INPUT_commodore_cluster_id}-bootstrap-ignition/"
 
     echo "Upload completed."

workflows/cloudscale/init-steps.yml

@@ -61,10 +61,12 @@ spells:
     if [[ $( curl -sH "Authorization: Bearer ${INPUT_cloudscale_token}" https://api.cloudscale.ch/v1/flavors -o /dev/null -w"%{http_code}" ) != 200 ]]
     then
       echo "Cloudscale token not valid!"
+      exit 1
     fi
     if [[ $( curl -sH "Authorization: Bearer ${INPUT_cloudscale_token_floaty}" https://api.cloudscale.ch/v1/flavors -o /dev/null -w"%{http_code}" ) != 200 ]]
     then
       echo "Cloudscale Floaty token not valid!"
+      exit 1
     fi
 - match: Then I set secrets in Vault
   description: |-

workflows/exoscale-decommission.workflow

@@ -8,6 +8,7 @@ Then I confirm cluster deletion
 Then I disable the OpsGenie heartbeat
 And I disable Project Syn
 And I delete all Load Balancer services
+And I delete all persistent volumes
 Then I save the loadbalancer metadata
 And I downtime the loadbalancers in icinga
 And I decommission Terraform resources

workflows/exoscale/decommission-steps.yml

@@ -42,6 +42,11 @@ spells:
     HIERADATA_REPO_TOKEN=${INPUT_gitlab_api_token}
     EOF
 
+    if [[ ! -d catalog ]]
+    then
+      commodore catalog compile "$INPUT_commodore_cluster_id"
+    fi
+
     tf_image=$(\
       yq eval ".parameters.openshift4_terraform.images.terraform.image" \
       dependencies/openshift4-terraform/class/defaults.yml)
@@ -165,7 +170,10 @@ spells:
       "-backend-config=unlock_method=DELETE" \
       "-backend-config=retry_wait_min=5"
 
-    terraform state rm "module.cluster.module.lb.module.hiera[0].gitfile_checkout.appuio_hieradata"
+    if terraform state list | grep "gitfile_checkout.appuio_hieradata" > /dev/null
+    then
+      terraform state rm "module.cluster.module.lb.module.hiera[0].gitfile_checkout.appuio_hieradata"
+    fi
 
     # Suppress errors on the first run; it is expected to fail
     terraform destroy --auto-approve || true

workflows/shared/decommission-steps.yml

@@ -9,6 +9,7 @@ spells:
 
       Usually of the form `<CLUSTER_ID>.<BASE_DOMAIN>`
   - name: passbolt_passphrase
+    type: local-sensitive
     description: |-
       Your password for Passbolt.
 
@@ -90,7 +91,7 @@ spells:
     kubectl delete machineautoscaler -A --all
 - match: And I delete all persistent volumes
   description: |-
-    This step deletes all persistent volumes on the cluster, so that the corresponding Cloudscale resources can be decommissioned by the controller.
+    This step deletes all persistent volumes on the cluster, so that the corresponding CSP resources can be decommissioned by the controller.
 
     By cordoning all non-master nodes and deleting all their pods (except the csi driver pods) we ensure that no new PVs are created, while the existing ones can be cleaned up. Deleting all pods has the additional benefit that we don’t have to deal with PDBs when deleting the machinesets in the next step.
   inputs:
@@ -154,20 +155,13 @@ spells:
   inputs:
   - name: lb_fqdn_1
   - name: lb_fqdn_2
+  - name: control_vshn_api_token
   run: |
     set -euo pipefail
-    echo '###################################################################################'
-    echo '#                                                                                 #'
-    echo "#  Please manually delete the cluster's LoadBalancer servers before proceeding.   #"
-    echo '#                                                                                 #'
-    echo '###################################################################################'
-    echo
-    echo You can go to:
-    echo https://control.vshn.net/servers/definitions/appuio/${INPUT_lb_fqdn_1}/delete
-    echo https://control.vshn.net/servers/definitions/appuio/${INPUT_lb_fqdn_2}/delete
-    sleep 2
-    # NOTE(aa): This step is currently annoying to automate, but once ticket PORTAL-253 is resolved,
-    # it should be easy.
+    echo "Deleting server definitions ..."
+    curl -XDELETE -H"$AUTH" https://control.vshn.net/api/servers/1/appuio/${INPUT_lb_fqdn_1}
+    curl -XDELETE -H"$AUTH" https://control.vshn.net/api/servers/1/appuio/${INPUT_lb_fqdn_2}
+    echo "Server definitions deleted."
 
 - match: And I decommission the LoadBalancers
   description: |-

workflows/shared/image-steps.yml

@@ -36,5 +36,47 @@ spells:
 
     echo Downloading OpenShift image "$MATCH_image_name" to "$image_path"
 
+    echo curl -L "https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/${MAJOR}.${MINOR}/${MATCH_image_name}/rhcos-${MATCH_image_name}-x86_64-openstack.x86_64.qcow2.gz"
     curl -L "https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/${MAJOR}.${MINOR}/${MATCH_image_name}/rhcos-${MATCH_image_name}-x86_64-openstack.x86_64.qcow2.gz" | gzip -d > "$image_path"
     env -i "image_path=$image_path" >> "$OUTPUT"
+
+spells:
+- match: Then I download the OpenShift OVA image for version "(?P<image_name>[^"]+)"
+  description: |-
+    This step downloads the OpenShift OVA image for the version specified by in the step.
+
+    If the image already exists locally, it skips the download.
+  outputs:
+  - name: image_path
+  - name: image_major
+  - name: image_minor
+  - name: image_patch
+  run: |
+    set -euo pipefail
+
+    . "$GANDALF_SPELLBOOK_DIR"/scripts/semver.sh
+
+    MAJOR=0
+    MINOR=0
+    PATCH=0
+    SPECIAL=""
+    semverParseInto "$MATCH_image_name" MAJOR MINOR PATCH SPECIAL
+
+    image_path="rhcos-$MAJOR.$MINOR.ova"
+
+    env -i "image_major=$MAJOR" >> "$OUTPUT"
+    env -i "image_minor=$MINOR" >> "$OUTPUT"
+    env -i "image_patch=$PATCH" >> "$OUTPUT"
+
+    echo "Image is $image_path"
+
+    if [ -f "$image_path" ]; then
+      echo "Image $image_path already exists, skipping download."
+      env -i "image_path=$image_path" >> "$OUTPUT"
+      exit 0
+    fi
+
+    echo Downloading OpenShift image "$MATCH_image_name" to "$image_path"
+
+    curl -L "https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/${MAJOR}.${MINOR}/${MATCH_image_name}/rhcos-${MATCH_image_name}-x86_64-vmware.x86_64.ova" > "$image_path"
+    env -i "image_path=$image_path" >> "$OUTPUT"

workflows/xelon-decommission.workflow

@@ -0,0 +1,4 @@
+Given I have all prerequisites installed
+Then I destroy Terraform resources
+And I delete all S3 buckets
+And I delete the cluster's API tokens

workflows/xelon.workflow

@@ -0,0 +1,13 @@
+Given I have all prerequisites installed
+And I download the `openshift-install` binary for version "4.21"
+And Cloudscale API tokens
+And Xelon API tokens
+And a personal VSHN GitLab access token
+Then I download the OpenShift OVA image for version "4.21.0"
+And I import the image into Xelon
+And I set up required S3 buckets
+Then I configure the OpenShift installer
+Then I prepare for terraform
+Then I create the bootstrap node
+Then I fix the load balancer
+Then I create the control plane nodes

workflows/xelon/cilium/apiextensions.k8s.io_v1_customresourcedefinition_ciliumconfigs.cilium.io.yaml

@@ -0,0 +1,116 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.16.5
+  name: ciliumconfigs.cilium.io
+spec:
+  group: cilium.io
+  names:
+    categories:
+      - all
+      - cilium
+    kind: CiliumConfig
+    listKind: CiliumConfigList
+    plural: ciliumconfigs
+    shortNames:
+      - cconf
+      - cconfs
+    singular: ciliumconfig
+  scope: Cluster
+  versions:
+    - name: v1alpha1
+      schema:
+        openAPIV3Schema:
+          description: CiliumConfig defines the configuration of Isovalent Networking
+            for Kubernetes and all its components
+          properties:
+            apiVersion:
+              description: |-
+                APIVersion defines the versioned schema of this representation of an object.
+                Servers should convert recognized schemas to the latest internal value, and
+                may reject unrecognized values.
+                More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+              type: string
+            kind:
+              description: |-
+                Kind is a string value representing the REST resource this object represents.
+                Servers may infer this from the endpoint the client submits requests to.
+                Cannot be updated.
+                In CamelCase.
+                More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+              type: string
+            metadata:
+              type: object
+            spec:
+              description: CiliumConfigSpec defines the desired state of CiliumConfig
+              type: object
+              x-kubernetes-preserve-unknown-fields: true
+            status:
+              description: CiliumConfigStatus defines the observed state of CiliumConfig
+              properties:
+                conditions:
+                  description: Conditions provides details on the state of the component
+                  items:
+                    description: Condition contains details for one aspect of the
+                      current state of this API Resource.
+                    properties:
+                      lastTransitionTime:
+                        description: |-
+                          lastTransitionTime is the last time the condition transitioned from one status to another.
+                          This should be when the underlying condition changed.  If that is not known, then using the time when the API field changed is acceptable.
+                        format: date-time
+                        type: string
+                      message:
+                        description: |-
+                          message is a human readable message indicating details about the transition.
+                          This may be an empty string.
+                        maxLength: 32768
+                        type: string
+                      observedGeneration:
+                        description: |-
+                          observedGeneration represents the .metadata.generation that the condition was set based upon.
+                          For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
+                          with respect to the current state of the instance.
+                        format: int64
+                        minimum: 0
+                        type: integer
+                      reason:
+                        description: |-
+                          reason contains a programmatic identifier indicating the reason for the condition's last transition.
+                          Producers of specific condition types may define expected values and meanings for this field,
+                          and whether the values are considered a guaranteed API.
+                          The value should be a CamelCase string.
+                          This field may not be empty.
+                        maxLength: 1024
+                        minLength: 1
+                        pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
+                        type: string
+                      status:
+                        description: status of the condition, one of True, False,
+                          Unknown.
+                        enum:
+                          - 'True'
+                          - 'False'
+                          - Unknown
+                        type: string
+                      type:
+                        description: type of condition in CamelCase or in foo.example.com/CamelCase.
+                        maxLength: 316
+                        pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
+                        type: string
+                    required:
+                      - lastTransitionTime
+                      - message
+                      - reason
+                      - status
+                      - type
+                    type: object
+                  type: array
+                  x-kubernetes-list-type: atomic
+              type: object
+          type: object
+      served: true
+      storage: true
+      subresources:
+        status: {}

workflows/xelon/cilium/ciliumconfig.yaml

@@ -0,0 +1,82 @@
+apiVersion: cilium.io/v1alpha1
+kind: CiliumConfig
+metadata:
+  labels:
+    app.kubernetes.io/name: clife
+  name: ciliumconfig
+spec:
+  bgpControlPlane:
+    enabled: false
+    secretsNamespace:
+      name: cilium
+  bpf:
+    masquerade: true
+  clustermesh:
+    apiserver:
+      metrics:
+        serviceMonitor:
+          enabled: true
+  cni:
+    binPath: /var/lib/cni/bin
+    confPath: /var/run/multus/cni/net.d
+  egressGateway:
+    enabled: true
+  endpointRoutes:
+    enabled: true
+  enterprise:
+    egressGatewayHA:
+      enabled: true
+    featureGate:
+      approved:
+        - BGPControlPlane
+        - EgressGatewayHA
+        - EgressGatewayIPv4
+        - EncryptionWireguard
+        - EnterpriseBGPControlPlane
+        - L2Announcement
+        - LocalRedirectPolicy
+  envoy:
+    enabled: true
+  hubble:
+    metrics:
+      enabled:
+        - dns:sourceContext=workload|namespace|reserved-identity;destinationContext=workload|namespace|reserved-identity
+        - drop:sourceContext=workload|namespace|reserved-identity;destinationContext=workload|namespace|reserved-identity
+        - httpV2:sourceContext=workload|namespace|reserved-identity;destinationContext=workload|namespace|reserved-identity
+      serviceMonitor:
+        enabled: true
+    relay:
+      enabled: true
+    tls:
+      enabled: true
+  ipam:
+    mode: cluster-pool
+    operator:
+      clusterPoolIPv4MaskSize: 24
+      clusterPoolIPv4PodCIDRList:
+        - 10.0.0.0/8
+  k8sClientRateLimit:
+    burst: 30
+    qps: 15
+  k8sServiceHost: ${APIINT}
+  k8sServicePort: '6443'
+  kubeProxyReplacement: 'true'
+  l2announcements:
+    enabled: false
+  l7Proxy: true
+  localRedirectPolicy: true
+  operator:
+    prometheus:
+      enabled: false
+      serviceMonitor:
+        enabled: true
+    resources:
+      limits:
+        memory: 500Mi
+      requests:
+        cpu: 100m
+        memory: 300Mi
+  prometheus:
+    enabled: true
+    serviceMonitor:
+      enabled: true

workflows/xelon/cilium/operatorgroup.yaml

@@ -0,0 +1,6 @@
+apiVersion: operators.coreos.com/v1alpha2
+kind: OperatorGroup
+metadata:
+  name: clife
+  namespace: cilium
+spec: null