diff --git a/kinder/ci/kinder-run.sh b/kinder/ci/kinder-run.sh index 72ff54e0..1c7fdca9 100755 --- a/kinder/ci/kinder-run.sh +++ b/kinder/ci/kinder-run.sh @@ -45,7 +45,7 @@ popd # add the kinder ROOT directory to PATH # this is needed so that the workflow runner can easy call "kinder ..." commands. -export PATH="${PATH}:${ROOT_PATH}" +export PATH="${ROOT_PATH}:${PATH}" # set an optional default name for the cluster created by kinder export KINDER_CLUSTER_NAME="kinder-test" diff --git a/kinder/ci/tools/update-workflows/config.yaml b/kinder/ci/tools/update-workflows/config.yaml index ef042370..d7e2b6e6 100644 --- a/kinder/ci/tools/update-workflows/config.yaml +++ b/kinder/ci/tools/update-workflows/config.yaml @@ -171,6 +171,18 @@ jobGroups: jobs: - kubernetesVersion: latest +- name: upgrade-etcd-rollback + testInfraJobSpec: + targetFile: kubeadm-kinder-upgrade-etcd-rollback.yaml + template: ./templates/testinfra/kubeadm-kinder-upgrade-etcd-rollback.yaml + kinderWorkflowSpec: + targetFile: upgrade-etcd-rollback-{{ .KubernetesVersion }}.yaml + template: ./templates/workflows/upgrade-etcd-rollback.yaml + additionalFiles: + - ./templates/workflows/upgrade-etcd-rollback-tasks.yaml + jobs: + - kubernetesVersion: latest + - name: rootless testInfraJobSpec: targetFile: kubeadm-kinder-rootless.yaml @@ -226,3 +238,13 @@ jobGroups: - ./templates/workflows/encryption-algorithm-tasks.yaml jobs: - kubernetesVersion: latest + +- name: extended + testInfraJobSpec: + targetFile: kubeadm-kinder-extended.yaml + template: ./templates/testinfra/kubeadm-kinder-extended.yaml + kinderWorkflowSpec: + targetFile: extended-{{ .KubernetesVersion }}.yaml + template: ./templates/workflows/extended.yaml + jobs: + - kubernetesVersion: latest diff --git a/kinder/ci/tools/update-workflows/templates/testinfra/kubeadm-kinder-extended.yaml b/kinder/ci/tools/update-workflows/templates/testinfra/kubeadm-kinder-extended.yaml new file mode 100644 index 00000000..d58695d7 --- /dev/null +++ b/kinder/ci/tools/update-workflows/templates/testinfra/kubeadm-kinder-extended.yaml @@ -0,0 +1,43 @@ +- name: ci-kubernetes-e2e-kubeadm-kinder-extended-{{ dashVer .KubernetesVersion }} + cluster: eks-prow-build-cluster + interval: 12h + decorate: true + labels: + preset-dind-enabled: "true" + preset-kind-volume-mounts: "true" + annotations: + testgrid-dashboards: sig-cluster-lifecycle-kubeadm + testgrid-tab-name: kubeadm-kinder-extended-{{ dashVer .KubernetesVersion }} + testgrid-alert-email: sig-cluster-lifecycle-kubeadm-alerts@kubernetes.io + description: "OWNER: sig-cluster-lifecycle (kinder); Runs independent kubeadm feature and regression workflows and reports all scenario failures" + testgrid-num-columns-recent: "20" + testgrid-num-failures-to-alert: "4" + testgrid-alert-stale-results-hours: "48" + decoration_config: + timeout: 180m + extra_refs: + - org: kubernetes + repo: kubernetes + base_ref: {{ branchFor .KubernetesVersion }} + path_alias: k8s.io/kubernetes + - org: kubernetes + repo: kubeadm + base_ref: main + path_alias: k8s.io/kubeadm + spec: + containers: + - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:{{ .TestInfraImage }}-{{ imageVer .KubernetesVersion }} + command: + - runner.sh + - "../kubeadm/kinder/ci/kinder-run.sh" + args: + - {{ .WorkflowFile }} + securityContext: + privileged: true + resources: + limits: + memory: "9000Mi" + cpu: 2000m + requests: + memory: "9000Mi" + cpu: 2000m diff --git a/kinder/ci/tools/update-workflows/templates/testinfra/kubeadm-kinder-upgrade-etcd-rollback.yaml b/kinder/ci/tools/update-workflows/templates/testinfra/kubeadm-kinder-upgrade-etcd-rollback.yaml new file mode 100644 index 00000000..1e4b0d85 --- /dev/null +++ b/kinder/ci/tools/update-workflows/templates/testinfra/kubeadm-kinder-upgrade-etcd-rollback.yaml @@ -0,0 +1,42 @@ +- name: ci-kubernetes-e2e-kubeadm-kinder-upgrade-etcd-rollback-{{ dashVer .KubernetesVersion }} + cluster: eks-prow-build-cluster + interval: {{ .JobInterval }} + decorate: true + labels: + preset-dind-enabled: "true" + preset-kind-volume-mounts: "true" + annotations: + testgrid-dashboards: sig-cluster-lifecycle-kubeadm + testgrid-tab-name: kubeadm-kinder-upgrade-etcd-rollback-{{ dashVer .KubernetesVersion }} + testgrid-alert-email: sig-cluster-lifecycle-kubeadm-alerts@kubernetes.io + description: "OWNER: sig-cluster-lifecycle (kinder); Verifies etcd data rollback after a failed kubeadm upgrade" + testgrid-num-columns-recent: "20" +{{ .AlertAnnotations }} + decoration_config: + timeout: 60m + extra_refs: + - org: kubernetes + repo: kubernetes + base_ref: {{ branchFor .KubernetesVersion }} + path_alias: k8s.io/kubernetes + - org: kubernetes + repo: kubeadm + base_ref: main + path_alias: k8s.io/kubeadm + spec: + containers: + - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:{{ .TestInfraImage }}-{{ imageVer .KubernetesVersion }} + command: + - runner.sh + - "../kubeadm/kinder/ci/kinder-run.sh" + args: + - {{ .WorkflowFile }} + securityContext: + privileged: true + resources: + limits: + memory: "9000Mi" + cpu: 2000m + requests: + memory: "9000Mi" + cpu: 2000m diff --git a/kinder/ci/tools/update-workflows/templates/workflows/extended.yaml b/kinder/ci/tools/update-workflows/templates/workflows/extended.yaml new file mode 100644 index 00000000..a2e3acb4 --- /dev/null +++ b/kinder/ci/tools/update-workflows/templates/workflows/extended.yaml @@ -0,0 +1,13 @@ +version: 1 +summary: | + Run independent kubeadm feature and regression workflows using native Kinder workflow groups. + Cover functionality beyond regular upgrade and downgrade testing. + A failed scenario is reported without skipping the remaining scenarios. + See https://github.com/kubernetes/kubeadm/issues/3331. +workflows: +- name: upgrade-etcd-rollback + workflow: upgrade-etcd-rollback-{{ .KubernetesVersion }}.yaml + timeout: 70m +- name: upgrade-no-addon-config-maps + workflow: upgrade-no-addon-config-maps-{{ .KubernetesVersion }}.yaml + timeout: 70m diff --git a/kinder/ci/tools/update-workflows/templates/workflows/upgrade-etcd-rollback-tasks.yaml b/kinder/ci/tools/update-workflows/templates/workflows/upgrade-etcd-rollback-tasks.yaml new file mode 100644 index 00000000..b1d4e6eb --- /dev/null +++ b/kinder/ci/tools/update-workflows/templates/workflows/upgrade-etcd-rollback-tasks.yaml @@ -0,0 +1,253 @@ +# IMPORTANT! This workflow is imported by upgrade-etcd-rollback-* workflows. +version: 1 +summary: | + Verify etcd data rollback after a failed kubeadm upgrade. + Regression test for https://github.com/kubernetes/kubeadm/issues/3330. +vars: + kubernetesVersion: ci/latest + baseImage: kindest/base:v20260214-ea8e5717 + image: kindest/node:etcd-rollback + clusterName: kinder-upgrade-etcd-rollback + kubeadmVerbosity: 6 +tasks: + - name: pull-base-image + cmd: docker + args: + - pull + - '{{ .vars.baseImage }}' + - name: add-kubernetes-version + cmd: kinder + args: + - build + - node-image-variant + - --base-image={{ .vars.baseImage }} + - --image={{ .vars.image }} + - --with-init-artifacts={{ .vars.kubernetesVersion }} + - --loglevel=debug + timeout: 15m + - name: create-cluster + cmd: kinder + args: + - create + - cluster + - --name={{ .vars.clusterName }} + - --image={{ .vars.image }} + - --control-plane-nodes=1 + - --worker-nodes=0 + - --loglevel=debug + timeout: 5m + - name: init + cmd: kinder + args: + - do + - kubeadm-init + - --name={{ .vars.clusterName }} + - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }} + - --loglevel=debug + timeout: 5m + - name: prepare-rollback-test + description: Save an API object and the original etcd manifest/data marker, then prepare a failing etcd Pod patch. + cmd: docker + args: + - exec + - '{{ .vars.clusterName }}-control-plane-1' + - bash + - -ceu + - | + set -euo pipefail + export KUBECONFIG=/etc/kubernetes/admin.conf + state=/kinder/rollback + data=/var/lib/etcd + mkdir -p "${state}/patches" + + # Pull the fixture before taking etcd down; image pull failure must fail the job. + crictl pull registry.k8s.io/e2e-test-images/busybox:1.37.0-2 + kubectl create configmap rollback-test --from-literal=value=before-upgrade + kubectl get configmap rollback-test -o jsonpath='{.metadata.uid}:{.data.value}' > "${state}/object-before" + cp /etc/kubernetes/manifests/etcd.yaml "${state}/etcd-before.yaml" + printf 'before-upgrade\n' > "${state}/marker-before" + cp "${state}/marker-before" "${data}/rollback-marker" + touch "${state}/armed" + + # Kubeadm installs this patch after backup. Kubelet terminates the original + # static Pod before starting its replacement, so the init container owns the + # data directory exclusively. Preserve all DB/WAL bytes rather than truncating + # a database file; the regular file at "member" makes old etcd require restore. + cat > "${state}/patches/etcd+strategic.yaml" <<'PATCH' + spec: + initContainers: + - name: fail-etcd-upgrade + image: registry.k8s.io/e2e-test-images/busybox:1.37.0-2 + command: + - /bin/sh + - -ec + - | + test -f /rollback-state/armed + test ! -e /rollback-state/fault-injected + cmp /rollback-state/marker-before /var/lib/etcd/rollback-marker + test -d /var/lib/etcd/member + test ! -e /var/lib/etcd/member.failed-upgrade + mv /var/lib/etcd/member /var/lib/etcd/member.failed-upgrade + printf 'injected upgrade failure\n' > /var/lib/etcd/member + printf 'after-backup\n' > /var/lib/etcd/rollback-marker + printf 'created-after-backup\n' > /var/lib/etcd/rollback-extra + printf 'injected\n' > /rollback-state/fault-injected + exit 1 + volumeMounts: + - name: etcd-data + mountPath: /var/lib/etcd + - name: rollback-state + mountPath: /rollback-state + volumes: + - name: rollback-state + hostPath: + path: /kinder/rollback + type: Directory + PATCH + + version=$(kubeadm version -o short) + printf '%s\n' "${version}" > "${state}/version" + cat > "${state}/upgrade.yaml" < "${state}/upgrade.log" 2>&1 || result=$? + printf '%s\n' "${result}" > "${state}/upgrade-exit-code" + cat "${state}/upgrade.log" + if [[ "${result}" -eq 0 ]]; then + echo 'FAIL: upgrade unexpectedly succeeded' >&2 + exit 1 + fi + # This acknowledgment is outside the data backup and is written only after + # injection completes. A preflight error or an image-pull failure cannot pass. + if [[ ! -s "${state}/fault-injected" ]]; then + echo 'FAIL: upgrade failed before the fault injection completed' >&2 + exit 1 + fi + timeout: 15m + - name: verify-etcd-data-rollback + description: Require the original manifest/data marker and removal of files created after backup. + cmd: docker + args: + - exec + - '{{ .vars.clusterName }}-control-plane-1' + - bash + - -ceu + - | + set -euo pipefail + export KUBECONFIG=/etc/kubernetes/admin.conf + state=/kinder/rollback + data=/var/lib/etcd + failed=0 + if ! cmp -s "${state}/etcd-before.yaml" /etc/kubernetes/manifests/etcd.yaml; then + echo 'FAIL: original etcd manifest was not restored' >&2 + failed=1 + fi + if ! cmp -s "${state}/marker-before" "${data}/rollback-marker"; then + echo 'FAIL: original etcd data was not restored' >&2 + failed=1 + fi + if [[ ! -d "${data}/member" || -e "${data}/member.failed-upgrade" || -e "${data}/rollback-extra" ]]; then + echo 'FAIL: failed-upgrade files survived rollback' >&2 + failed=1 + fi + if [[ -n "$(find "${data}" -mindepth 1 -maxdepth 1 -name 'kubeadm-backup-etcd-*')" ]]; then + echo 'FAIL: backup was nested inside the live etcd directory' >&2 + failed=1 + fi + # Fail on a broken restore without adding a second API-readiness timeout. + [[ "${failed}" -eq 0 ]] || exit 1 + timeout: 1m + - name: verify-api-recovery + description: Wait for API recovery and verify that the original ConfigMap UID and value survived rollback. + cmd: docker + args: + - exec + - '{{ .vars.clusterName }}-control-plane-1' + - bash + - -ceu + - | + set -euo pipefail + export KUBECONFIG=/etc/kubernetes/admin.conf + state=/kinder/rollback + data=/var/lib/etcd + deadline=$((SECONDS + 180)) + until kubectl --request-timeout=5s get --raw=/readyz > /dev/null 2>&1; do + if (( SECONDS >= deadline )); then + echo 'FAIL: API server did not recover after rollback' >&2 + exit 1 + fi + sleep 2 + done + kubectl --request-timeout=10s get configmap rollback-test -o jsonpath='{.metadata.uid}:{.data.value}' > "${state}/object-after" + cmp "${state}/object-before" "${state}/object-after" + timeout: 4m + - name: collect-rollback-diagnostics + cmd: docker + args: + - exec + - '{{ .vars.clusterName }}-control-plane-1' + - bash + - -ceu + - | + mkdir -p /kinder/rollback + find /var/lib/etcd -maxdepth 6 -type f -printf '%p %s bytes\n' > /kinder/rollback/data-files.txt + find /var/lib/etcd -maxdepth 1 -printf '%f %y\n' > /kinder/rollback/data-entries.txt + crictl ps -a > /kinder/rollback/containers.txt + force: true + ignoreError: true + - name: export-rollback-diagnostics + cmd: docker + args: + - cp + - '{{ .vars.clusterName }}-control-plane-1:/kinder/rollback' + - '{{ .env.ARTIFACTS }}/etcd-rollback' + force: true + ignoreError: true + - name: get-logs + cmd: kinder + args: + - export + - logs + - --name={{ .vars.clusterName }} + - --loglevel=debug + - '{{ .env.ARTIFACTS }}' + force: true + timeout: 5m + ignoreError: true + - name: delete + cmd: kinder + args: + - delete + - cluster + - --name={{ .vars.clusterName }} + - --loglevel=debug + force: true diff --git a/kinder/ci/tools/update-workflows/templates/workflows/upgrade-etcd-rollback.yaml b/kinder/ci/tools/update-workflows/templates/workflows/upgrade-etcd-rollback.yaml new file mode 100644 index 00000000..22dcf4ab --- /dev/null +++ b/kinder/ci/tools/update-workflows/templates/workflows/upgrade-etcd-rollback.yaml @@ -0,0 +1,9 @@ +version: 1 +summary: | + Verify etcd data rollback after a failed kubeadm upgrade. + test grid > https://testgrid.k8s.io/sig-cluster-lifecycle-kubeadm#kubeadm-kinder-upgrade-etcd-rollback-{{ dashVer .KubernetesVersion }} + config > https://git.k8s.io/test-infra/config/jobs/kubernetes/sig-cluster-lifecycle/{{ .TargetFile }} +vars: + kubernetesVersion: "\{\{ resolve `ci/{{ ciLabelFor .KubernetesVersion }}` \}\}" +tasks: +- import: upgrade-etcd-rollback-tasks.yaml diff --git a/kinder/ci/workflows/extended-latest.yaml b/kinder/ci/workflows/extended-latest.yaml new file mode 100644 index 00000000..c3b810ee --- /dev/null +++ b/kinder/ci/workflows/extended-latest.yaml @@ -0,0 +1,14 @@ +# AUTOGENERATED by https://git.k8s.io/kubeadm/kinder/ci/tools/update-workflows +version: 1 +summary: | + Run independent kubeadm feature and regression workflows using native Kinder workflow groups. + Cover functionality beyond regular upgrade and downgrade testing. + A failed scenario is reported without skipping the remaining scenarios. + See https://github.com/kubernetes/kubeadm/issues/3331. +workflows: +- name: upgrade-etcd-rollback + workflow: upgrade-etcd-rollback-latest.yaml + timeout: 70m +- name: upgrade-no-addon-config-maps + workflow: upgrade-no-addon-config-maps-latest.yaml + timeout: 70m diff --git a/kinder/ci/workflows/upgrade-etcd-rollback-latest.yaml b/kinder/ci/workflows/upgrade-etcd-rollback-latest.yaml new file mode 100644 index 00000000..28c4ef4a --- /dev/null +++ b/kinder/ci/workflows/upgrade-etcd-rollback-latest.yaml @@ -0,0 +1,10 @@ +# AUTOGENERATED by https://git.k8s.io/kubeadm/kinder/ci/tools/update-workflows +version: 1 +summary: | + Verify etcd data rollback after a failed kubeadm upgrade. + test grid > https://testgrid.k8s.io/sig-cluster-lifecycle-kubeadm#kubeadm-kinder-upgrade-etcd-rollback-latest + config > https://git.k8s.io/test-infra/config/jobs/kubernetes/sig-cluster-lifecycle/kubeadm-kinder-upgrade-etcd-rollback.yaml +vars: + kubernetesVersion: "{{ resolve `ci/latest` }}" +tasks: +- import: upgrade-etcd-rollback-tasks.yaml diff --git a/kinder/ci/workflows/upgrade-etcd-rollback-tasks.yaml b/kinder/ci/workflows/upgrade-etcd-rollback-tasks.yaml new file mode 100644 index 00000000..10ae551d --- /dev/null +++ b/kinder/ci/workflows/upgrade-etcd-rollback-tasks.yaml @@ -0,0 +1,254 @@ +# AUTOGENERATED by https://git.k8s.io/kubeadm/kinder/ci/tools/update-workflows +# IMPORTANT! This workflow is imported by upgrade-etcd-rollback-* workflows. +version: 1 +summary: | + Verify etcd data rollback after a failed kubeadm upgrade. + Regression test for https://github.com/kubernetes/kubeadm/issues/3330. +vars: + kubernetesVersion: ci/latest + baseImage: kindest/base:v20260214-ea8e5717 + image: kindest/node:etcd-rollback + clusterName: kinder-upgrade-etcd-rollback + kubeadmVerbosity: 6 +tasks: + - name: pull-base-image + cmd: docker + args: + - pull + - '{{ .vars.baseImage }}' + - name: add-kubernetes-version + cmd: kinder + args: + - build + - node-image-variant + - --base-image={{ .vars.baseImage }} + - --image={{ .vars.image }} + - --with-init-artifacts={{ .vars.kubernetesVersion }} + - --loglevel=debug + timeout: 15m + - name: create-cluster + cmd: kinder + args: + - create + - cluster + - --name={{ .vars.clusterName }} + - --image={{ .vars.image }} + - --control-plane-nodes=1 + - --worker-nodes=0 + - --loglevel=debug + timeout: 5m + - name: init + cmd: kinder + args: + - do + - kubeadm-init + - --name={{ .vars.clusterName }} + - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }} + - --loglevel=debug + timeout: 5m + - name: prepare-rollback-test + description: Save an API object and the original etcd manifest/data marker, then prepare a failing etcd Pod patch. + cmd: docker + args: + - exec + - '{{ .vars.clusterName }}-control-plane-1' + - bash + - -ceu + - | + set -euo pipefail + export KUBECONFIG=/etc/kubernetes/admin.conf + state=/kinder/rollback + data=/var/lib/etcd + mkdir -p "${state}/patches" + + # Pull the fixture before taking etcd down; image pull failure must fail the job. + crictl pull registry.k8s.io/e2e-test-images/busybox:1.37.0-2 + kubectl create configmap rollback-test --from-literal=value=before-upgrade + kubectl get configmap rollback-test -o jsonpath='{.metadata.uid}:{.data.value}' > "${state}/object-before" + cp /etc/kubernetes/manifests/etcd.yaml "${state}/etcd-before.yaml" + printf 'before-upgrade\n' > "${state}/marker-before" + cp "${state}/marker-before" "${data}/rollback-marker" + touch "${state}/armed" + + # Kubeadm installs this patch after backup. Kubelet terminates the original + # static Pod before starting its replacement, so the init container owns the + # data directory exclusively. Preserve all DB/WAL bytes rather than truncating + # a database file; the regular file at "member" makes old etcd require restore. + cat > "${state}/patches/etcd+strategic.yaml" <<'PATCH' + spec: + initContainers: + - name: fail-etcd-upgrade + image: registry.k8s.io/e2e-test-images/busybox:1.37.0-2 + command: + - /bin/sh + - -ec + - | + test -f /rollback-state/armed + test ! -e /rollback-state/fault-injected + cmp /rollback-state/marker-before /var/lib/etcd/rollback-marker + test -d /var/lib/etcd/member + test ! -e /var/lib/etcd/member.failed-upgrade + mv /var/lib/etcd/member /var/lib/etcd/member.failed-upgrade + printf 'injected upgrade failure\n' > /var/lib/etcd/member + printf 'after-backup\n' > /var/lib/etcd/rollback-marker + printf 'created-after-backup\n' > /var/lib/etcd/rollback-extra + printf 'injected\n' > /rollback-state/fault-injected + exit 1 + volumeMounts: + - name: etcd-data + mountPath: /var/lib/etcd + - name: rollback-state + mountPath: /rollback-state + volumes: + - name: rollback-state + hostPath: + path: /kinder/rollback + type: Directory + PATCH + + version=$(kubeadm version -o short) + printf '%s\n' "${version}" > "${state}/version" + cat > "${state}/upgrade.yaml" < "${state}/upgrade.log" 2>&1 || result=$? + printf '%s\n' "${result}" > "${state}/upgrade-exit-code" + cat "${state}/upgrade.log" + if [[ "${result}" -eq 0 ]]; then + echo 'FAIL: upgrade unexpectedly succeeded' >&2 + exit 1 + fi + # This acknowledgment is outside the data backup and is written only after + # injection completes. A preflight error or an image-pull failure cannot pass. + if [[ ! -s "${state}/fault-injected" ]]; then + echo 'FAIL: upgrade failed before the fault injection completed' >&2 + exit 1 + fi + timeout: 15m + - name: verify-etcd-data-rollback + description: Require the original manifest/data marker and removal of files created after backup. + cmd: docker + args: + - exec + - '{{ .vars.clusterName }}-control-plane-1' + - bash + - -ceu + - | + set -euo pipefail + export KUBECONFIG=/etc/kubernetes/admin.conf + state=/kinder/rollback + data=/var/lib/etcd + failed=0 + if ! cmp -s "${state}/etcd-before.yaml" /etc/kubernetes/manifests/etcd.yaml; then + echo 'FAIL: original etcd manifest was not restored' >&2 + failed=1 + fi + if ! cmp -s "${state}/marker-before" "${data}/rollback-marker"; then + echo 'FAIL: original etcd data was not restored' >&2 + failed=1 + fi + if [[ ! -d "${data}/member" || -e "${data}/member.failed-upgrade" || -e "${data}/rollback-extra" ]]; then + echo 'FAIL: failed-upgrade files survived rollback' >&2 + failed=1 + fi + if [[ -n "$(find "${data}" -mindepth 1 -maxdepth 1 -name 'kubeadm-backup-etcd-*')" ]]; then + echo 'FAIL: backup was nested inside the live etcd directory' >&2 + failed=1 + fi + # Fail on a broken restore without adding a second API-readiness timeout. + [[ "${failed}" -eq 0 ]] || exit 1 + timeout: 1m + - name: verify-api-recovery + description: Wait for API recovery and verify that the original ConfigMap UID and value survived rollback. + cmd: docker + args: + - exec + - '{{ .vars.clusterName }}-control-plane-1' + - bash + - -ceu + - | + set -euo pipefail + export KUBECONFIG=/etc/kubernetes/admin.conf + state=/kinder/rollback + data=/var/lib/etcd + deadline=$((SECONDS + 180)) + until kubectl --request-timeout=5s get --raw=/readyz > /dev/null 2>&1; do + if (( SECONDS >= deadline )); then + echo 'FAIL: API server did not recover after rollback' >&2 + exit 1 + fi + sleep 2 + done + kubectl --request-timeout=10s get configmap rollback-test -o jsonpath='{.metadata.uid}:{.data.value}' > "${state}/object-after" + cmp "${state}/object-before" "${state}/object-after" + timeout: 4m + - name: collect-rollback-diagnostics + cmd: docker + args: + - exec + - '{{ .vars.clusterName }}-control-plane-1' + - bash + - -ceu + - | + mkdir -p /kinder/rollback + find /var/lib/etcd -maxdepth 6 -type f -printf '%p %s bytes\n' > /kinder/rollback/data-files.txt + find /var/lib/etcd -maxdepth 1 -printf '%f %y\n' > /kinder/rollback/data-entries.txt + crictl ps -a > /kinder/rollback/containers.txt + force: true + ignoreError: true + - name: export-rollback-diagnostics + cmd: docker + args: + - cp + - '{{ .vars.clusterName }}-control-plane-1:/kinder/rollback' + - '{{ .env.ARTIFACTS }}/etcd-rollback' + force: true + ignoreError: true + - name: get-logs + cmd: kinder + args: + - export + - logs + - --name={{ .vars.clusterName }} + - --loglevel=debug + - '{{ .env.ARTIFACTS }}' + force: true + timeout: 5m + ignoreError: true + - name: delete + cmd: kinder + args: + - delete + - cluster + - --name={{ .vars.clusterName }} + - --loglevel=debug + force: true diff --git a/kinder/cmd/kinder/test/workflow/workflow.go b/kinder/cmd/kinder/test/workflow/workflow.go index 5aa75365..be6548cf 100644 --- a/kinder/cmd/kinder/test/workflow/workflow.go +++ b/kinder/cmd/kinder/test/workflow/workflow.go @@ -59,7 +59,7 @@ func NewCommand() *cobra.Command { cmd.Flags().BoolVar( &flags.ExitOnError, "exit-on-task-error", false, - "exit after first task failed", + "stop after the first failure, running forced cleanup tasks and writing reports", ) return cmd } diff --git a/kinder/pkg/test/workflow/extended_test.go b/kinder/pkg/test/workflow/extended_test.go new file mode 100644 index 00000000..aec33ba3 --- /dev/null +++ b/kinder/pkg/test/workflow/extended_test.go @@ -0,0 +1,551 @@ +/* +Copyright 2026 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package workflow + +import ( + "bytes" + "context" + "encoding/xml" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "syscall" + "testing" + "time" +) + +func writeWorkflowFile(t *testing.T, dir, name, content string) string { + t.Helper() + path := filepath.Join(dir, name) + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0644); err != nil { + t.Fatal(err) + } + return path +} + +func loadTestWorkflow(t *testing.T, file string) *Workflow { + t.Helper() + w, err := NewWorkflow(file) + if err != nil { + t.Fatal(err) + } + return w +} + +func readReport(t *testing.T, artifacts string) junitTestSuite { + t.Helper() + data, err := os.ReadFile(filepath.Join(artifacts, "junit_runner.xml")) + if err != nil { + t.Fatal(err) + } + var suite junitTestSuite + if err := xml.Unmarshal(data, &suite); err != nil { + t.Fatal(err) + } + return suite +} + +func requireFile(t *testing.T, path, content string) { + t.Helper() + data, err := os.ReadFile(path) + if err != nil || string(data) != content { + t.Fatalf("%s: got %q, error %v, want %q", path, data, err, content) + } +} + +func requireAbsent(t *testing.T, path string) { + t.Helper() + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Fatalf("unexpected file %s (stat error %v)", path, err) + } +} + +// Execute real child workflows through the generated group's native references. +// No mock kinder executable, Docker, or network is involved. +func TestExtendedWorkflow(t *testing.T) { + source, err := os.ReadFile("../../../ci/workflows/extended-latest.yaml") + if err != nil { + t.Fatal(err) + } + for _, failed := range []string{"", "upgrade-etcd-rollback", "upgrade-no-addon-config-maps", "both"} { + t.Run("failed="+failed, func(t *testing.T) { + dir := t.TempDir() + file := writeWorkflowFile(t, dir, "extended.yaml", string(source)) + t.Setenv("WORKFLOW_TEST_EQUALS", "a=b=c") + for _, name := range []string{"upgrade-etcd-rollback", "upgrade-no-addon-config-maps"} { + code := 0 + if failed == name || failed == "both" { + code = 1 + } + writeWorkflowFile(t, dir, name+"-latest.yaml", fmt.Sprintf(`version: 1 +vars: + marker: %s +env: + SCENARIO: '{{ .vars.marker }}' +tasks: +- name: run + cmd: sh + args: + - -ec + - | + test "$WORKFLOW_TEST_EQUALS" = 'a=b=c' + printf '%%s' "$SCENARIO" > "$ARTIFACTS/ran" + exit %d +- name: dependent + cmd: sh + args: [-ec, 'touch "$ARTIFACTS/dependent"'] +- name: cleanup + cmd: sh + args: [-ec, 'touch "$ARTIFACTS/cleaned"'] + force: true +`, name, code)) + } + w := loadTestWorkflow(t, file) + // Dry-running must not mutate templates or create artifacts. + dryArtifacts := filepath.Join(dir, "dry") + if err := w.Run(io.Discard, true, false, false, dryArtifacts); err != nil { + t.Fatal(err) + } + requireAbsent(t, dryArtifacts) + // Run the same loaded object twice with distinct artifact roots. + for i := 0; i < 2; i++ { + artifacts := filepath.Join(dir, fmt.Sprintf("artifacts %d", i)) + err := w.Run(io.Discard, false, false, false, artifacts) + if (err != nil) != (failed != "") { + t.Fatalf("Run() error = %v", err) + } + report := readReport(t, artifacts) + wantFailures := 0 + for _, tc := range report.Cases { + childFailed := failed == tc.Name || failed == "both" + if childFailed { + wantFailures++ + } + if (tc.Failure != "") != childFailed || tc.Skipped != "" { + t.Fatalf("unexpected scenario result: %+v", tc) + } + childArtifacts := filepath.Join(artifacts, tc.Name) + requireFile(t, filepath.Join(childArtifacts, "ran"), tc.Name) + requireFile(t, filepath.Join(childArtifacts, "cleaned"), "") + child := readReport(t, childArtifacts) + for _, task := range child.Cases { + if task.ClassName != "kinder.test.workflow."+tc.Name { + t.Fatalf("missing scenario namespace: %+v", task) + } + } + if child.Tests != 3 || len(child.Cases) != 3 { + t.Fatalf("unexpected child report: %+v", child) + } + if childFailed { + requireAbsent(t, filepath.Join(childArtifacts, "dependent")) + if child.Failures != 1 || child.Cases[1].Skipped == "" { + t.Fatalf("lost child failure: %+v", child) + } + } else { + requireFile(t, filepath.Join(childArtifacts, "dependent"), "") + if child.Failures != 0 { + t.Fatalf("unexpected child failure: %+v", child) + } + } + } + if report.Tests != 2 || len(report.Cases) != 2 || report.Failures != wantFailures { + t.Fatalf("unexpected group report: %+v", report) + } + } + }) + } +} + +func TestWorkflowGroupValidation(t *testing.T) { + for _, tc := range []struct{ name, body string }{ + {"mixed", "tasks: [{cmd: 'true'}]\nworkflows: [{name: child, workflow: leaf.yaml}]"}, + {"empty", "tasks: []"}, + {"null-child", "workflows: [null]"}, + {"null-task", "tasks: [null]"}, + {"duplicate", "workflows: [{name: child, workflow: leaf.yaml}, {name: child, workflow: leaf.yaml}]"}, + {"unsafe-name", "workflows: [{name: ../escape, workflow: leaf.yaml}]"}, + {"missing-file", "workflows: [{name: child, workflow: missing.yaml}]"}, + {"missing-path", "workflows: [{name: child}]"}, + {"negative-timeout", "workflows: [{name: child, workflow: leaf.yaml, timeout: -1s}]"}, + {"group-env", "env: {A: b}\nworkflows: [{name: child, workflow: leaf.yaml}]"}, + {"group-vars", "vars: {A: b}\nworkflows: [{name: child, workflow: leaf.yaml}]"}, + {"force-scenario", "workflows: [{name: child, workflow: leaf.yaml, force: true}]"}, + {"cycle", "workflows: [{name: child, workflow: group.yaml}]"}, + {"import-cycle", "tasks: [{import: group.yaml}]"}, + {"mixed-cycle", "workflows: [{name: child, workflow: import-parent.yaml}]"}, + {"import-group", "tasks: [{import: other-group.yaml}]"}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + writeWorkflowFile(t, dir, "leaf.yaml", "version: 1\ntasks: [{cmd: 'true'}]\n") + writeWorkflowFile(t, dir, "import-parent.yaml", "version: 1\ntasks: [{import: group.yaml}]\n") + writeWorkflowFile(t, dir, "other-group.yaml", "version: 1\nworkflows: [{name: child, workflow: leaf.yaml}]\n") + file := writeWorkflowFile(t, dir, "group.yaml", "version: 1\n"+tc.body+"\n") + if _, err := NewWorkflow(file); err == nil { + t.Fatal("invalid configuration accepted") + } + }) + } + t.Run("symlink-cycle", func(t *testing.T) { + dir := t.TempDir() + file := writeWorkflowFile(t, dir, "group.yaml", "version: 1\nworkflows: [{name: child, workflow: alias.yaml}]\n") + if err := os.Symlink(file, filepath.Join(dir, "alias.yaml")); err != nil { + t.Fatal(err) + } + if _, err := NewWorkflow(file); err == nil { + t.Fatal("symlink cycle accepted") + } + }) +} + +func TestWorkflowGroupPreflight(t *testing.T) { + dir := t.TempDir() + writeWorkflowFile(t, dir, "first.yaml", `version: 1 +tasks: +- cmd: sh + args: [-ec, 'touch "$ARTIFACTS/ran"'] +`) + writeWorkflowFile(t, dir, "bad.yaml", "version: 1\ntasks: [{cmd: '{{ .vars.missing }}'}]\n") + w := loadTestWorkflow(t, writeWorkflowFile(t, dir, "group.yaml", `version: 1 +workflows: +- name: first + workflow: first.yaml +- name: bad + workflow: bad.yaml +`)) + for _, dry := range []bool{true, false} { + artifacts := t.TempDir() + if err := w.Run(io.Discard, dry, false, false, artifacts); err == nil { + t.Fatal("invalid child template accepted") + } + requireAbsent(t, filepath.Join(artifacts, "first", "ran")) + } +} + +func TestWorkflowNestedGroupsAndImports(t *testing.T) { + dir := t.TempDir() + writeWorkflowFile(t, dir, "nested/tasks.yaml", `version: 1 +vars: {value: imported} +tasks: +- cmd: sh + args: [-ec, 'printf "%s" "{{ .vars.value }}" > "$ARTIFACTS/value"'] +`) + writeWorkflowFile(t, dir, "nested/leaf.yaml", "version: 1\nvars: {value: overridden}\ntasks: [{import: tasks.yaml}]\n") + writeWorkflowFile(t, dir, "nested/group.yaml", "version: 1\nworkflows: [{name: inner, workflow: leaf.yaml}]\n") + // Referencing the same workflow twice is allowed: only active cycles fail. + w := loadTestWorkflow(t, writeWorkflowFile(t, dir, "group.yaml", "version: 1\nworkflows: [{name: a, workflow: nested/group.yaml}, {name: b, workflow: nested/group.yaml}]\n")) + artifacts := t.TempDir() + if err := w.Run(io.Discard, false, false, false, artifacts); err != nil { + t.Fatal(err) + } + for _, name := range []string{"a", "b"} { + requireFile(t, filepath.Join(artifacts, name, "inner", "value"), "overridden") + if r := readReport(t, filepath.Join(artifacts, name)); r.Tests != 1 || r.Failures != 0 { + t.Fatalf("nested report: %+v", r) + } + } +} + +// Lifecycle fixture: a task starts a real long-lived process in its task group. +func lifecycleWorkflow(t *testing.T, mode string) (*Workflow, string) { + t.Helper() + dir := t.TempDir() + taskTimeout, scenarioTimeout := "30s", "30s" + command := `sleep 30 & + child=$! + printf '%s' "$child" > "$ARTIFACTS/pid" + touch "$ARTIFACTS/started" + wait "$child"` + if mode == "task-timeout" { + taskTimeout = "200ms" + } + if mode == "scenario-timeout" { + scenarioTimeout = "200ms" + } + if mode == "fail-fast" { + command = "exit 1" + } + writeWorkflowFile(t, dir, "first.yaml", fmt.Sprintf(`version: 1 +tasks: +- name: active + cmd: sh + args: + - -ec + - | + %s + timeout: %s +- name: dependent + cmd: sh + args: [-ec, 'touch "$ARTIFACTS/dependent"'] +- name: cleanup + cmd: sh + args: [-ec, 'touch "$ARTIFACTS/cleaned"'] + force: true + timeout: 2s +`, command, taskTimeout)) + writeWorkflowFile(t, dir, "second.yaml", `version: 1 +tasks: +- cmd: sh + args: [-ec, 'touch "$ARTIFACTS/ran"'] +`) + file := writeWorkflowFile(t, dir, "group.yaml", fmt.Sprintf(`version: 1 +workflows: +- name: first + workflow: first.yaml + timeout: %s +- name: second + workflow: second.yaml +`, scenarioTimeout)) + return loadTestWorkflow(t, file), file +} + +func waitForFile(t *testing.T, file string) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for time.Now().Before(deadline) { + if _, err := os.Stat(file); err == nil { + return + } + time.Sleep(10 * time.Millisecond) + } + t.Fatalf("timed out waiting for %s", file) +} + +func requireStoppedProcess(t *testing.T, pidFile string) { + t.Helper() + data, err := os.ReadFile(pidFile) + if err != nil { + t.Fatal(err) + } + pid, err := strconv.Atoi(string(data)) + if err != nil { + t.Fatal(err) + } + // An orphan may briefly be a zombie pending reaping by init on Linux. + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + state, err := exec.Command("ps", "-p", strconv.Itoa(pid), "-o", "stat=").Output() + if err != nil || strings.HasPrefix(strings.TrimSpace(string(state)), "Z") { + return + } + time.Sleep(10 * time.Millisecond) + } + _ = syscall.Kill(pid, syscall.SIGKILL) + t.Fatalf("task descendant %d survived cancellation", pid) +} + +func checkLifecycle(t *testing.T, artifacts string, secondSkipped bool) { + t.Helper() + requireFile(t, filepath.Join(artifacts, "first", "cleaned"), "") + requireAbsent(t, filepath.Join(artifacts, "first", "dependent")) + parent := readReport(t, artifacts) + if parent.Tests != 2 || parent.Failures != 1 || len(parent.Cases) != 2 || (parent.Cases[1].Skipped != "") != secondSkipped { + t.Fatalf("parent report: %+v", parent) + } + child := readReport(t, filepath.Join(artifacts, "first")) + if child.Tests != 3 || child.Failures != 1 || len(child.Cases) != 3 || child.Cases[1].Skipped == "" || child.Cases[2].Failure != "" { + t.Fatalf("child report: %+v", child) + } + if secondSkipped { + requireAbsent(t, filepath.Join(artifacts, "second", "ran")) + } else { + requireFile(t, filepath.Join(artifacts, "second", "ran"), "") + } +} + +func TestWorkflowGroupLifecycle(t *testing.T) { + for _, mode := range []string{"task-timeout", "scenario-timeout", "cancel", "fail-fast"} { + t.Run(mode, func(t *testing.T) { + w, _ := lifecycleWorkflow(t, mode) + artifacts := t.TempDir() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + result := make(chan error, 1) + go func() { result <- w.runContext(ctx, io.Discard, false, false, mode == "fail-fast", artifacts) }() + if mode == "cancel" { + waitForFile(t, filepath.Join(artifacts, "first", "started")) + cancel() + } + select { + case err := <-result: + if err == nil { + t.Fatal("expected failure") + } + case <-time.After(10 * time.Second): + cancel() + t.Fatal("workflow did not finish") + } + checkLifecycle(t, artifacts, mode == "cancel" || mode == "fail-fast") + if mode != "fail-fast" { + requireStoppedProcess(t, filepath.Join(artifacts, "first", "pid")) + } + }) + } +} + +// Exercise actual SIGINT/SIGTERM delivery without signaling the go test process. +func TestWorkflowGroupSignals(t *testing.T) { + for _, sig := range []syscall.Signal{syscall.SIGINT, syscall.SIGTERM} { + t.Run(sig.String(), func(t *testing.T) { + _, file := lifecycleWorkflow(t, "cancel") + artifacts := t.TempDir() + cmd := exec.Command(os.Args[0], "-test.run=^TestWorkflowSignalHelper$") + cmd.Env = append(os.Environ(), "KINDER_SIGNAL_TEST="+file, "KINDER_SIGNAL_ARTIFACTS="+artifacts) + var output bytes.Buffer + cmd.Stdout, cmd.Stderr = &output, &output + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = cmd.Process.Kill() }) + waitForFile(t, filepath.Join(artifacts, "first", "started")) + if err := cmd.Process.Signal(sig); err != nil { + t.Fatal(err) + } + result := make(chan error, 1) + go func() { result <- cmd.Wait() }() + select { + case err := <-result: + if err == nil { + t.Fatal("canceled workflow returned success") + } + case <-time.After(10 * time.Second): + t.Fatal("canceled workflow hung") + } + checkLifecycle(t, artifacts, true) + requireStoppedProcess(t, filepath.Join(artifacts, "first", "pid")) + }) + } +} + +func TestWorkflowSignalHelper(t *testing.T) { + file := os.Getenv("KINDER_SIGNAL_TEST") + if file == "" { + return + } + w, err := NewWorkflow(file) + if err == nil { + err = w.Run(io.Discard, false, false, false, os.Getenv("KINDER_SIGNAL_ARTIFACTS")) + } + if err != nil { + os.Exit(1) + } + os.Exit(0) +} + +func TestWorkflowGroupCanceledBeforeStart(t *testing.T) { + w, _ := lifecycleWorkflow(t, "cancel") + artifacts := t.TempDir() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := w.runContext(ctx, io.Discard, false, false, false, artifacts); err != context.Canceled { + t.Fatalf("expected cancellation, got %v", err) + } + report := readReport(t, artifacts) + if report.Tests != 2 || len(report.Cases) != 2 { + t.Fatalf("report: %+v", report) + } + for _, tc := range report.Cases { + if tc.Skipped == "" { + t.Fatalf("scenario was not skipped: %+v", tc) + } + requireAbsent(t, filepath.Join(artifacts, tc.Name, "junit_runner.xml")) + } + requireAbsent(t, filepath.Join(artifacts, "first", "started")) + requireAbsent(t, filepath.Join(artifacts, "first", "cleaned")) + requireAbsent(t, filepath.Join(artifacts, "second", "ran")) +} + +func TestWorkflowCleanupTimeout(t *testing.T) { + dir := t.TempDir() + writeWorkflowFile(t, dir, "leaf.yaml", `version: 1 +tasks: +- cmd: sh + args: [-ec, 'exit 1'] +- name: stuck-cleanup + cmd: sh + args: [-ec, 'exec sleep 30'] + timeout: 100ms + force: true +- name: final-cleanup + cmd: sh + args: [-ec, 'touch "$ARTIFACTS/cleaned"'] + force: true +`) + w := loadTestWorkflow(t, writeWorkflowFile(t, dir, "group.yaml", "version: 1\nworkflows: [{name: first, workflow: leaf.yaml}, {name: second, workflow: leaf.yaml}]\n")) + artifacts := t.TempDir() + if err := w.Run(io.Discard, false, false, false, artifacts); err == nil { + t.Fatal("expected failure") + } + for _, name := range []string{"first", "second"} { + requireFile(t, filepath.Join(artifacts, name, "cleaned"), "") + report := readReport(t, filepath.Join(artifacts, name)) + if report.Failures != 2 || report.Tests != 3 { + t.Fatalf("report: %+v", report) + } + } +} + +func TestWorkflowTaskCompatibility(t *testing.T) { + dir := t.TempDir() + file := writeWorkflowFile(t, dir, "tasks.yaml", `version: 1 +tasks: +- cmd: sh + args: [-ec, 'exit 1'] + ignoreError: true +- cmd: sh + args: [-ec, 'touch "$ARTIFACTS/after-ignored"'] +- cmd: kinder-intentionally-nonexistent-command +- cmd: sh + args: [-ec, 'touch "$ARTIFACTS/dependent"'] +- cmd: sh + args: [-ec, 'touch "$ARTIFACTS/cleaned"'] + force: true +`) + w := loadTestWorkflow(t, file) + artifacts := t.TempDir() + if err := w.Run(io.Discard, false, false, true, artifacts); err == nil { + t.Fatal("expected start failure") + } + requireFile(t, filepath.Join(artifacts, "after-ignored"), "") + requireFile(t, filepath.Join(artifacts, "cleaned"), "") + requireAbsent(t, filepath.Join(artifacts, "dependent")) + report := readReport(t, artifacts) + if report.Tests != 5 || report.Failures != 1 || len(report.Cases) != 5 || report.Cases[3].Skipped == "" { + t.Fatalf("report: %+v", report) + } +} + +func TestExistingCIWorkflowConfigurations(t *testing.T) { + files, err := filepath.Glob("../../../ci/workflows/*.yaml") + if err != nil || len(files) == 0 { + t.Fatalf("workflow discovery: files=%d error=%v", len(files), err) + } + for _, file := range files { + t.Run(filepath.Base(file), func(t *testing.T) { + if _, err := NewWorkflow(file); err != nil { + t.Fatal(err) + } + }) + } +} diff --git a/kinder/pkg/test/workflow/run.go b/kinder/pkg/test/workflow/run.go new file mode 100644 index 00000000..1feb7f32 --- /dev/null +++ b/kinder/pkg/test/workflow/run.go @@ -0,0 +1,185 @@ +/* +Copyright 2026 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package workflow + +import ( + "context" + "fmt" + "io" + "os" + "os/signal" + "path/filepath" + "syscall" + "time" + + "github.com/pkg/errors" +) + +// Run executes a workflow or scenario group. One cancellation context covers the +// entire tree, including the intervals between commands and child workflows. +func (w *Workflow) Run(out io.Writer, dryRun, verbose, exitOnError bool, artifacts string) error { + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + return w.runContext(ctx, out, dryRun, verbose, exitOnError, artifacts) +} + +func (w *Workflow) runContext(ctx context.Context, out io.Writer, dryRun, verbose, exitOnError bool, artifacts string) error { + plan, err := w.prepare(artifacts, dryRun, "") + if err != nil { + return err + } + return plan.run(ctx, out, dryRun, verbose, exitOnError) +} + +// preparedWorkflow validates the whole tree before any command is executed. +// Each leaf builds its own environment; parent Vars/Env are never merged into it. +type preparedWorkflow struct { + artifacts string + scope string + commands []*taskCmd + scenarios []preparedScenario +} + +type preparedScenario struct { + name string + timeout time.Duration + workflow *preparedWorkflow +} + +func (w *Workflow) prepare(artifacts string, dryRun bool, scope string) (*preparedWorkflow, error) { + builder, err := newTaskCmdBuilder(w) + if err != nil { + return nil, err + } + if artifacts == "" { + artifacts = builder.env["ARTIFACTS"] + } + if artifacts == "" { + if dryRun { + artifacts = "" + } else { + artifacts, err = os.MkdirTemp(".", "kinder-test-workflow") + if err != nil { + return nil, err + } + } + } + // Commands with a different working directory still need an absolute path. + if !dryRun { + artifacts, err = filepath.Abs(artifacts) + if err != nil { + return nil, err + } + if err := os.MkdirAll(artifacts, 0755); err != nil { + return nil, err + } + } + builder.env["ARTIFACTS"] = artifacts + plan := &preparedWorkflow{artifacts: artifacts, scope: scope} + for _, task := range w.Tasks { + cmd, err := builder.build(task, false) + if err != nil { + return nil, err + } + plan.commands = append(plan.commands, cmd) + } + for _, scenario := range w.Workflows { + childScope := scenario.Name + if scope != "" { + childScope = scope + "." + childScope + } + child, err := scenario.child.prepare(filepath.Join(artifacts, scenario.Name), dryRun, childScope) + if err != nil { + return nil, errors.Wrapf(err, "preparing scenario %q", scenario.Name) + } + plan.scenarios = append(plan.scenarios, preparedScenario{scenario.Name, scenario.Timeout.Duration, child}) + } + return plan, nil +} + +func (p *preparedWorkflow) run(ctx context.Context, out io.Writer, dryRun, verbose, exitOnError bool) error { + runner := newTaskCmdRunner() + if p.scope != "" { + runner.suite.Name += "." + p.scope + } + foundError := false + for _, scenario := range p.scenarios { + fmt.Fprintf(out, "# workflow %s\n", scenario.name) + if !dryRun && (ctx.Err() != nil || (foundError && exitOnError)) { + reason := "skipping because a predecessor scenario failed" + if ctx.Err() != nil { + reason = "skipping because workflow was canceled or its deadline expired" + } + _ = runner.registerTestCase(scenario.name, withSkipped(reason)) + foundError = true + continue + } + childCtx := ctx + cancel := func() {} + if !dryRun && scenario.timeout > 0 { + childCtx, cancel = context.WithTimeout(ctx, scenario.timeout) + } + start := time.Now() + err := scenario.workflow.run(childCtx, out, dryRun, verbose, exitOnError) + // A deadline that expires during forced cleanup must still fail the scenario. + if err == nil { + err = childCtx.Err() + } + cancel() + if dryRun { + if err != nil { + return err + } + continue + } + options := []testCaseOption{withDuration(time.Since(start))} + if err != nil { + options = append(options, withFailure(err.Error())) + foundError = true + } + _ = runner.registerTestCase(scenario.name, options...) + } + for _, cmd := range p.commands { + fmt.Fprintf(out, "# %s\n%s\n\n", cmd.Name, cmd.CmdText) + if dryRun { + continue + } + // Forced tasks are cleanup: they may run after failure or cancellation, + // with their own finite timeout. No new scenario starts after cancellation. + if err := runner.Run(ctx, cmd, p.artifacts, verbose); err != nil { + foundError = true + fmt.Fprintf(out, " %v\n\n", err) + } else { + fmt.Fprint(out, " completed!\n\n") + } + } + if dryRun { + return nil + } + if err := runner.DumpJUnitRunner(p.artifacts); err != nil { + return err + } + runner.ReportSummary() + fmt.Fprintf(out, "see %s and task logs for more details\n\n", filepath.Join(p.artifacts, "junit_runner.xml")) + if ctx.Err() != nil { + return ctx.Err() + } + if foundError { + return errors.New("failed executing the workflow") + } + return nil +} diff --git a/kinder/pkg/test/workflow/taskCmdBuilder.go b/kinder/pkg/test/workflow/taskCmdBuilder.go index 0f631fcf..89702a52 100644 --- a/kinder/pkg/test/workflow/taskCmdBuilder.go +++ b/kinder/pkg/test/workflow/taskCmdBuilder.go @@ -55,7 +55,7 @@ func newTaskCmdBuilder(w *Workflow) (c *taskCmdBuilder, err error) { // loads OS environment variables into the taskCmdBuilder context for _, e := range os.Environ() { - pair := strings.Split(e, "=") + pair := strings.SplitN(e, "=", 2) name := pair[0] value := pair[1] c.env[name] = value @@ -109,6 +109,11 @@ func (c *taskCmdBuilder) expand(text string) (string, error) { // build creates a taskCmd func (c *taskCmdBuilder) build(t *Task, verbose bool) (tcmd *taskCmd, err error) { + // Do not mutate the source workflow; it may be run or dry-run more than once. + copyTask := *t + copyTask.Args = append([]string(nil), t.Args...) + t = ©Task + // expand golang templates that might exists in the cmd and/or into the args t.Cmd, err = c.expand(t.Cmd) if err != nil { diff --git a/kinder/pkg/test/workflow/taskCmdRunner.go b/kinder/pkg/test/workflow/taskCmdRunner.go index b172a8a8..a66b1c30 100644 --- a/kinder/pkg/test/workflow/taskCmdRunner.go +++ b/kinder/pkg/test/workflow/taskCmdRunner.go @@ -17,12 +17,12 @@ limitations under the License. package workflow import ( + "context" "encoding/xml" "fmt" "io" "os" "os/exec" - "os/signal" "path/filepath" "strings" "syscall" @@ -44,11 +44,12 @@ type taskCmdRunner struct { // junitTestSuite implements junit TestSuite standard object type junitTestSuite struct { - XMLName xml.Name `xml:"testsuite"` - Failures int `xml:"failures,attr"` - Tests int `xml:"tests,attr"` - Time float64 `xml:"time,attr"` - Cases []junitTestCase + Name string `xml:"name,attr"` + XMLName xml.Name `xml:"testsuite"` + Failures int `xml:"failures,attr"` + Tests int `xml:"tests,attr"` + Time float64 `xml:"time,attr"` + Cases []junitTestCase `xml:"testcase"` } // junitTestCase implements junit TestCase standard object @@ -65,13 +66,20 @@ type junitTestCase struct { func newTaskCmdRunner() *taskCmdRunner { return &taskCmdRunner{ start: time.Now(), - suite: junitTestSuite{}, + suite: junitTestSuite{Name: "kinder.test.workflow"}, } } // Run a taskCmd -func (c *taskCmdRunner) Run(t *taskCmd, artifacts string, verbose bool) error { +func (c *taskCmdRunner) Run(ctx context.Context, t *taskCmd, artifacts string, verbose bool) error { start := time.Now() + if ctx.Err() != nil { + c.canceled = true + if t.Force { + // Cleanup still has the task's own timeout; ignore the expired parent. + ctx = context.WithoutCancel(ctx) + } + } // unless the cmd execution is forced, check if the taskCmd should be skipped because one of // the previous taskCmd failed, timedOut or was canceled. @@ -84,22 +92,20 @@ func (c *taskCmdRunner) Run(t *taskCmd, artifacts string, verbose bool) error { return c.registerTestCase(t.Name, withSkipped("skipping because a predecessor task timed-out")) } if c.canceled { - return c.registerTestCase(t.Name, withSkipped("skipping because task workflow was canceled by the user")) + return c.registerTestCase(t.Name, withSkipped("skipping because workflow was canceled or its deadline expired")) } } - // creates a channel for handling command cancellation - cancel := make(chan os.Signal, 1) - signal.Notify(cancel, syscall.SIGINT, syscall.SIGTERM) - // sets Stdout and Stderr for the command. // please note that the command output will go on files by default, // and it will be echoed on video only if specifically requested taskLog := filepath.Join(artifacts, fmt.Sprintf("%s-log.txt", t.Name)) writer, err := os.Create(taskLog) if err != nil { - return errors.Wrapf(err, "error creating %q log file", taskLog) + c.failed = true + return c.registerTestCase(t.Name, withFailure(errors.Wrapf(err, "error creating %q log file", taskLog).Error())) } + defer writer.Close() t.Cmd.Stdout = writer t.Cmd.Stderr = writer @@ -135,6 +141,9 @@ func (c *taskCmdRunner) Run(t *taskCmd, artifacts string, verbose bool) error { result <- t.Cmd.Wait() }() + timer := time.NewTimer(t.Timeout.Duration) + defer timer.Stop() + // Wait for one of: // - the command completes // - the command is canceled @@ -160,25 +169,27 @@ func (c *taskCmdRunner) Run(t *taskCmd, artifacts string, verbose bool) error { withDuration(time.Since(start)), ) - case <-cancel: + case <-ctx.Done(): // keeps track of this failure type to block execution of following TestCmd c.canceled = true // cleanup command process and its child, if any cleanup(t.Cmd) + <-result // record test case cancellation and exits with error return c.registerTestCase(t.Name, - withFailure("task was canceled by the user"), + withFailure(ctx.Err().Error()), withDuration(time.Since(start)), ) - case <-time.After(t.Timeout.Duration): + case <-timer.C: // keeps track of this failure type to block execution of following TestCmd c.timedOut = true // cleanup command process and its child, if any cleanup(t.Cmd) + <-result // record test case timeout and exits with error return c.registerTestCase(t.Name, @@ -202,7 +213,7 @@ func (c *taskCmdRunner) ReportSummary() { passed := run - failures fmt.Printf("Ran %d of %d tasks in %.3f seconds\n", run, total, c.suite.Time) - if failures > 0 { + if failures > 0 || skipped > 0 { fmt.Printf("FAIL! -- %d tasks Passed | %d Failed | %d Skipped\n\n", passed, failures, skipped) return } @@ -257,7 +268,7 @@ func withSkipped(message string) testCaseOption { // registerTestCase register task output as a test case result func (c *taskCmdRunner) registerTestCase(name string, options ...testCaseOption) error { tc := &junitTestCase{ - ClassName: "kinder.test.workflow", + ClassName: c.suite.Name, Name: name, } @@ -279,37 +290,14 @@ func (c *taskCmdRunner) registerTestCase(name string, options ...testCaseOption) return nil } -// cleanup tries to ensure a cmdtask is properly closed +// cleanup kills the process group created by taskCmdBuilder, including children +// remaining after the group leader exits. Tasks must not detach from this group. func cleanup(cmd *exec.Cmd) { - defer func() { - if r := recover(); r != nil { - fmt.Println("Recovered in f", r) - } - }() - - // if the process doesn't exist we can skip the cleanup - // https://man7.org/linux/man-pages/man2/kill.2.html - if err := syscall.Kill(cmd.Process.Pid, syscall.Signal(0)); err != nil { + if cmd.Process == nil { return } - - // obtain the process ground ID - pgid, err := syscall.Getpgid(cmd.Process.Pid) - if err != nil { - fmt.Printf("error: failed obtaining the pgid for pid: %v, %v\n", cmd.Process.Pid, err) - goto kill_process - } - - // kill all processes in the process group - if err := syscall.Kill(-pgid, syscall.SIGKILL); err != nil { - fmt.Printf("error: failed to kill pgid: %v, %v\n", pgid, err) - goto kill_process - } - return - -kill_process: - fmt.Println("falling back to killing the parent process only...") - if err := cmd.Process.Kill(); err != nil { - fmt.Printf("error: failed killing process with pid: %v, %v\n", cmd.Process.Pid, err) + if err := syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL); err != nil && err != syscall.ESRCH { + fmt.Printf("error: failed killing task process group %d: %v\n", cmd.Process.Pid, err) + _ = cmd.Process.Kill() } } diff --git a/kinder/pkg/test/workflow/workflow.go b/kinder/pkg/test/workflow/workflow.go index a3d3e25c..e5ae2584 100644 --- a/kinder/pkg/test/workflow/workflow.go +++ b/kinder/pkg/test/workflow/workflow.go @@ -17,8 +17,10 @@ limitations under the License. /* Package workflow implements a simple workflow manager aimed at automating kinder test workflows. -Workflows must be defined in a yaml file containing list of tasks; each task -can be any command, thus including also any kinder commands invoked via CLI. +Workflows are YAML files containing either tasks or a group of child workflows. +Each task can be any command, including kinder commands invoked via CLI. +Child workflows execute sequentially with isolated variables and failure state; +a failed child does not prevent its siblings from running, but cancellation does. Tasks will be executed in order; in case of errors the workflow will stop and the remaining tasks will be skipped with the only exception of tasks specifically marked to be executed in any case @@ -29,7 +31,6 @@ package workflow import ( "encoding/json" "fmt" - "io" "os" "path/filepath" "regexp" @@ -37,10 +38,11 @@ import ( "github.com/pkg/errors" log "github.com/sirupsen/logrus" + "sigs.k8s.io/yaml" ) -// Workflow represents a list of tasks to be executed during test workflow and related context +// Workflow defines either a sequence of tasks and its context, or a group of independent workflows. type Workflow struct { // Version of the workflow file // NB. We are enforcing version to be set only for future improvements of the config API @@ -67,6 +69,22 @@ type Workflow struct { // Tasks defines the list of tasks to be executed during test workflow Tasks Tasks + + // Workflows defines a sequential group of independent scenarios, mutually + // exclusive with Tasks. A scenario failure does not skip its siblings. + Workflows []*Scenario +} + +// Scenario invokes a workflow in the same process with independent variables, +// task state and artifacts. Paths are relative to the containing workflow file. +type Scenario struct { + Name string + Workflow string + // Timeout bounds scenario execution. Forced cleanup tasks may run afterwards, + // each bounded by its own task timeout. Zero means no scenario deadline. + Timeout Duration + + child *Workflow } // Tasks represents a list of tasks to be executed during test workflow. @@ -142,10 +160,25 @@ func (d *Duration) UnmarshalJSON(b []byte) error { // NewWorkflow creates a new workflow as defined in a workflow file func NewWorkflow(file string) (*Workflow, error) { - // Checks if the workflow file exists - if _, err := os.Stat(file); err != nil { - return nil, errors.Errorf("invalid workflow file: %s does not exist", file) + return loadWorkflow(file, map[string]bool{}) +} + +var scenarioNamePattern = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9_-]*$`) + +func loadWorkflow(file string, active map[string]bool) (*Workflow, error) { + canonical, err := filepath.EvalSymlinks(file) + if err != nil { + return nil, errors.Wrapf(err, "invalid workflow file %s", file) } + canonical, err = filepath.Abs(canonical) + if err != nil { + return nil, err + } + if active[canonical] { + return nil, errors.Errorf("cyclic workflow reference: %s", file) + } + active[canonical] = true + defer delete(active, canonical) // Loads and unmarshal it data, err := os.ReadFile(file) @@ -160,23 +193,59 @@ func NewWorkflow(file string) (*Workflow, error) { // checks minimum requirements // - version is set and well know - // - at least one task exists + // - either tasks or child workflows exist if w.Version != 1 { return nil, errors.Errorf("invalid taskfile %s: version does not contain a supported value", file) } - if len(w.Tasks) == 0 { - return nil, errors.Errorf("invalid taskfile %s: at least one task should be defined", file) + if (len(w.Tasks) == 0) == (len(w.Workflows) == 0) { + return nil, errors.Errorf("invalid workflow %s: define either tasks or workflows", file) + } + if len(w.Workflows) > 0 { + if len(w.Vars) > 0 || len(w.Env) > 0 { + return nil, errors.Errorf("workflow group %s: define vars and env in each child workflow", file) + } + names := map[string]bool{} + for _, scenario := range w.Workflows { + if scenario == nil || !scenarioNamePattern.MatchString(scenario.Name) { + return nil, errors.Errorf("workflow group %s: each scenario needs a safe, nonempty name (letters, digits, '_' or '-')", file) + } + if names[scenario.Name] { + return nil, errors.Errorf("workflow group %s: duplicate scenario %q", file, scenario.Name) + } + names[scenario.Name] = true + if scenario.Workflow == "" || scenario.Timeout.Duration < 0 { + return nil, errors.Errorf("workflow group %s: scenario %q needs a workflow path and nonnegative timeout", file, scenario.Name) + } + childPath := scenario.Workflow + if !filepath.IsAbs(childPath) { + childPath = filepath.Join(filepath.Dir(file), childPath) + } + scenario.child, err = loadWorkflow(childPath, active) + if err != nil { + return nil, errors.Wrapf(err, "loading scenario %q", scenario.Name) + } + } + return &w, nil + } + if w.Vars == nil { + w.Vars = map[string]string{} + } + if w.Env == nil { + w.Env = map[string]string{} } // Detect and resolve imports by expanding imported workflows into the top level workflow - if err := w.expandImports(file); err != nil { + if err := w.expandImports(file, active); err != nil { return nil, err } // For each task for i, t := range w.Tasks { + if t == nil { + return nil, errors.Errorf("workflow %s: task #%d is null", file, i+1) + } // if a task name is not defined, assign a default task name // otherwise prepend a prefix in order to get task logs ordered if t.Name == "" { @@ -191,6 +260,10 @@ func NewWorkflow(file string) (*Workflow, error) { t.Timeout.Duration = time.Duration(5 * time.Minute) } + if t.Timeout.Duration < 0 { + return nil, errors.Errorf("task %q has a negative timeout", t.Name) + } + // check if the task defines a cmd if t.Cmd == "" { return nil, errors.Errorf("invalid taskfile %s: task %q does not define a cmd", file, t.Name) @@ -201,10 +274,13 @@ func NewWorkflow(file string) (*Workflow, error) { } // expandImports imports a secondary workflow into the top level Workflow -func (w *Workflow) expandImports(file string) error { +func (w *Workflow) expandImports(file string, active map[string]bool) error { tasks := w.Tasks w.Tasks = Tasks{} for i, t := range tasks { + if t == nil { + return errors.Errorf("workflow %s: task #%d is null", file, i+1) + } // check if the task does not defines an import, preserve it as it is if t.Import == "" { w.Tasks = append(w.Tasks, t) @@ -234,17 +310,20 @@ func (w *Workflow) expandImports(file string) error { // reads the Import file // if path are relative, consider as a base path the folder where the importing file is located. - // TODO: implement a check for avoiding circular imports path := t.Import if !filepath.IsAbs(path) { base := filepath.Dir(file) path = filepath.Join(base, path) } - wx, err := NewWorkflow(path) + wx, err := loadWorkflow(path, active) if err != nil { return errors.Wrapf(err, "error importing workflow file %s", path) } + if len(wx.Workflows) > 0 { + return errors.Errorf("cannot import workflow group %s into tasks; use workflows instead", path) + } + // merge the vars from the import file into the parent file // in case of conflicts, vars in the parent file will shadow vars in the import file for k, v := range wx.Vars { @@ -275,107 +354,3 @@ func (w *Workflow) expandImports(file string) error { return nil } - -// Run executes a workflow -func (w *Workflow) Run(out io.Writer, dryRun, verbose, exitOnError bool, artifacts string) (err error) { - - // get a new taskCmdBuilder, responsible for creating taskCmd commands - taskCmdBuilder, err := newTaskCmdBuilder(w) - if err != nil { - return err - } - - // if artifact folder is not provided as input argument check - // 1. ARTIFACTS env var from the workflow file - // 2. ARTIFACTS OS env var - // Otherwise generate an artifact folder (or dummy placeholder in case of dry running) - - if artifacts == "" { - artifacts = taskCmdBuilder.env["ARTIFACTS"] - } - - if artifacts == "" { - artifacts = os.Getenv("ARTIFACTS") - } - - if artifacts == "" { - if !dryRun { - dir, err := os.Getwd() - if err != nil { - return errors.Wrapf(err, "error getting current directory") - } - - artifacts, err = os.MkdirTemp(dir, "kinder-test-workflow") - if err != nil { - return errors.Wrapf(err, "error creating artifact folder") - } - } else { - artifacts = "" - } - } - - //TODO: ensure artifact folder exist and can be written - - // adds a new env variable indicating where test artifacts should be stored - // to make this value available for cmd and args expansion - taskCmdBuilder.env["ARTIFACTS"] = artifacts - - // Gets a taskCmdRunner, responsible for executing taskCmd, - // handling failure, cancellation, timeouts and for generating or collecting - // all the workflow artifacts (junit_runner.xml, task logs, etc) - taskCmdRunner := newTaskCmdRunner() - - // Process all tasks, exploding golang templates for cmd and args - // and create the corresponding taskCmd - // Nb. we are splitting this step from actual execution of task for ensuring - // that all the formal error are detected before starting any real activity - var tcmds []*taskCmd - for _, t := range w.Tasks { - - tcmd, err := taskCmdBuilder.build(t, verbose) - if err != nil { - return err - } - - tcmds = append(tcmds, tcmd) - } - - foundError := false - // Executes taskCmds - for _, tcmd := range tcmds { - fmt.Fprintf(out, "# %s\n", tcmd.Name) - fmt.Fprintf(out, "%s\n\n", tcmd.CmdText) - - if !dryRun { - err := taskCmdRunner.Run(tcmd, artifacts, verbose) - if err != nil { - foundError = true - fmt.Fprintf(out, " %v\n\n", err) - - if exitOnError { - return err - } - - continue - } - - fmt.Fprintf(out, " completed!\n\n") - } - } - - // If not dry running, prints task summary and dumps the junit_runner.xml file - if !dryRun { - taskCmdRunner.ReportSummary() - - if err := taskCmdRunner.DumpJUnitRunner(artifacts); err != nil { - fmt.Fprintf(out, "%v\n", err) - return err - } - fmt.Fprintf(out, "see junit-runner.xml and task logs files for more details\n\n") - } - - if foundError { - return errors.New("failed executing the workflow") - } - return nil -}