USHIFT-6979: Add PCP metrics collection to test scenarios

test/bin/scenario.sh

@@ -22,6 +22,7 @@ PULL_SECRET_CONTENT="$(jq -c . "${PULL_SECRET}")"
 VM_BOOT_TIMEOUT=1200 # Overall total boot times are around 15m
 VM_GREENBOOT_TIMEOUT=1800 # Greenboot readiness may take up to 15-30m depending on the load
 SKIP_SOS=${SKIP_SOS:-false}  # may be overridden in global settings file
+SKIP_PCP=${SKIP_PCP:-false}  # may be overridden in global settings file
 SKIP_GREENBOOT=${SKIP_GREENBOOT:-false}  # may be overridden in scenario file
 GREENBOOT_TIMEOUT=${GREENBOOT_TIMEOUT:-600}  # may be overridden in scenario file
 # Container image signature verification should be disabled by default in the
@@ -292,6 +293,85 @@ sos_report_for_vm_offline() {
         "--filename" "*.log"
 }
 
+collect_pcp_reports_for_vm() {
+    local -r vmdir="${1}"
+    local -r vmname="${2}"
+
+    echo "Collecting PCP data from ${vmname}"
+    run_command_on_vm "${vmname}" "sudo systemctl stop --now pmlogger" || true
+
+    if ! run_command_on_vm "${vmname}" "test -d /var/log/pcp/pmlogger" ; then
+        echo "WARNING: No PCP data directory on ${vmname}, skipping collection"
+        return 0
+    fi
+
+    run_command_on_vm "${vmname}" \
+        "sudo tar czf /tmp/pcp-archives.tar.gz -C /var/log/pcp/pmlogger ." || true
+
+    mkdir -p "${vmdir}/pcp"
+    copy_file_from_vm "${vmname}" "/tmp/pcp-archives.tar.gz" "${vmdir}/pcp/" || {
+        echo "WARNING: Failed to collect PCP data from ${vmname}"
+    }
+}
+
+collect_pcp_reports_for_vm_offline() {
+    local -r vmdir="${1}"
+    local -r vmname="${2}"
+    local -r full_vmname="$(full_vm_name "${vmname}")"
+
+    echo "Collecting PCP data offline from ${vmname}"
+
+    "${ROOTDIR}/scripts/fetch_tools.sh" "robotframework"
+
+    invoke_qemu_script "wait" \
+        "--vm" "${full_vmname}"
+
+    invoke_qemu_script "bash" \
+        "--vm" "${full_vmname}" \
+        "--args" "sudo systemctl stop --now pmlogger"
+
+    invoke_qemu_script "bash" \
+        "--vm" "${full_vmname}" \
+        "--args" "sudo tar czf /tmp/pcp-archives.tar.gz -C /var/log/pcp/pmlogger ."
+
+    mkdir -p "${vmdir}/pcp"
+
+    invoke_qemu_script "download" \
+        "--vm" "${full_vmname}" \
+        "--src_dir" "/tmp/" \
+        "--dst_dir" "${vmdir}/pcp/" \
+        "--filename" "pcp-archives.tar.gz"
+}
+
+collect_pcp_reports() {
+    if "${SKIP_PCP}"; then
+        echo "Skipping PCP collection"
+        return 0
+    fi
+
+    echo "Collecting PCP reports"
+    for vmdir in "${SCENARIO_INFO_DIR}"/"${SCENARIO}"/vms/*; do
+        if [ ! -d "${vmdir}" ]; then
+            continue
+        fi
+
+        local vmname
+        vmname=$(basename "${vmdir}")
+        local ip
+        ip=$(cat "$(vm_property_filename "${vmname}" "ip")" 2>/dev/null) || true
+
+        local pcp_func="collect_pcp_reports_for_vm"
+        if [ -z "${ip}" ]; then
+            echo "Collecting PCP reports offline"
+            pcp_func="collect_pcp_reports_for_vm_offline"
+        fi
+
+        "${pcp_func}" "${vmdir}" "${vmname}" || {
+            echo "WARNING: Failed to collect PCP data from ${vmname}"
+        }
+    done
+}
+
 get_lrel_release_image_url() {
     local -r brew_lrel_release_version="$1"
     local image_url=""
@@ -1526,13 +1606,14 @@ action_create() {
     fi
     record_junit "setup" "load_scenario_script" "OK"
 
-    # Set the exit handler to attempt the sos report collection and error logging
+    # Set the exit handler to attempt PCP and SOS report collection and error logging
     # - Preserve the original exit code
     # - Log junit message on failure
     # - Override the exit code if sos report collection fails
     # shellcheck disable=SC2154
     trap 'rc=$? ; \
         [ "${rc}" -ne 0 ] && record_junit "setup" "scenario_create_vms" "FAILED" ; \
+        collect_pcp_reports || true ; \
         sos_report true || rc=1 ; \
         close_junit ; exit "${rc}"' EXIT
 
@@ -1584,13 +1665,14 @@ action_run() {
     fi
     record_junit "run" "load_scenario_script" "OK"
 
-    # Set the exit handler to attempt the sos report collection and error logging
+    # Set the exit handler to attempt PCP and SOS report collection and error logging
     # - Preserve the original exit code
     # - Log junit message on failure
     # - Override the exit code if sos report collection fails
     # shellcheck disable=SC2154
     trap 'rc=$? ; \
         [ "${rc}" -ne 0 ] && record_junit "run" "scenario_run_tests" "FAILED" ; \
+        collect_pcp_reports || true ; \
         sos_report true || rc=1 ; \
         close_junit ; exit "${rc}"' EXIT
 

test/image-blueprints-bootc/el10/layer1-base/group1/rhel102-test-agent.containerfile

@@ -22,8 +22,8 @@ COPY ./rpm-repos/$USHIFT_RPM_REPO_NAME $USHIFT_RPM_REPO_PATH
 COPY ./bootc-images/$USHIFT_RPM_REPO_NAME.repo /etc/yum.repos.d/
 
 # Install the test agent and cleanup
-RUN dnf install -y microshift-test-agent && \
-    systemctl enable microshift-test-agent && \
+RUN dnf install -y microshift-test-agent pcp pcp-zeroconf && \
+    systemctl enable microshift-test-agent pmcd pmlogger && \
     rm -vf /etc/yum.repos.d/microshift-*.repo && \
     rm -rvf $USHIFT_RPM_REPO_PATH && \
     dnf clean all

test/image-blueprints-bootc/el9/layer1-base/group1/rhel96-test-agent.containerfile

@@ -15,8 +15,8 @@ COPY ./rpm-repos/$USHIFT_RPM_REPO_NAME $USHIFT_RPM_REPO_PATH
 COPY ./bootc-images/$USHIFT_RPM_REPO_NAME.repo /etc/yum.repos.d/
 
 # Install the test agent and cleanup
-RUN dnf install -y microshift-test-agent && \
-    systemctl enable microshift-test-agent && \
+RUN dnf install -y microshift-test-agent pcp pcp-zeroconf && \
+    systemctl enable microshift-test-agent pmcd pmlogger && \
     rm -vf /etc/yum.repos.d/microshift-*.repo && \
     rm -rvf $USHIFT_RPM_REPO_PATH && \
     dnf clean all

test/image-blueprints-bootc/el9/layer1-base/group1/rhel98-test-agent.containerfile

@@ -22,8 +22,8 @@ COPY ./rpm-repos/$USHIFT_RPM_REPO_NAME $USHIFT_RPM_REPO_PATH
 COPY ./bootc-images/$USHIFT_RPM_REPO_NAME.repo /etc/yum.repos.d/
 
 # Install the test agent and cleanup
-RUN dnf install -y microshift-test-agent && \
-    systemctl enable microshift-test-agent && \
+RUN dnf install -y microshift-test-agent pcp pcp-zeroconf && \
+    systemctl enable microshift-test-agent pmcd pmlogger && \
     rm -vf /etc/yum.repos.d/microshift-*.repo && \
     rm -rvf $USHIFT_RPM_REPO_PATH && \
     dnf clean all

test/image-blueprints-bootc/upstream/group1/cos10-test-agent.containerfile

@@ -15,8 +15,8 @@ COPY ./rpm-repos/$USHIFT_RPM_REPO_NAME $USHIFT_RPM_REPO_PATH
 COPY ./bootc-images/$USHIFT_RPM_REPO_NAME.repo /etc/yum.repos.d/
 
 # Install the test agent and cleanup
-RUN dnf install -y microshift-test-agent && \
-    systemctl enable microshift-test-agent && \
+RUN dnf install -y microshift-test-agent pcp pcp-zeroconf && \
+    systemctl enable microshift-test-agent pmcd pmlogger && \
     rm -vf /etc/yum.repos.d/microshift-*.repo && \
     rm -rvf $USHIFT_RPM_REPO_PATH && \
     dnf clean all

test/image-blueprints-bootc/upstream/group1/cos9-test-agent.containerfile

@@ -15,8 +15,8 @@ COPY ./rpm-repos/$USHIFT_RPM_REPO_NAME $USHIFT_RPM_REPO_PATH
 COPY ./bootc-images/$USHIFT_RPM_REPO_NAME.repo /etc/yum.repos.d/
 
 # Install the test agent and cleanup
-RUN dnf install -y microshift-test-agent && \
-    systemctl enable microshift-test-agent && \
+RUN dnf install -y microshift-test-agent pcp pcp-zeroconf && \
+    systemctl enable microshift-test-agent pmcd pmlogger && \
     rm -vf /etc/yum.repos.d/microshift-*.repo && \
     rm -rvf $USHIFT_RPM_REPO_PATH && \
     dnf clean all

test/image-blueprints/layer1-base/group1/rhel96.toml

@@ -13,8 +13,12 @@ version = "*"
 name = "iproute-tc"
 version = "*"
 
+[[packages]]
+name = "pcp-zeroconf"
+version = "*"
+
 [customizations.services]
-enabled = ["microshift-test-agent"]
+enabled = ["microshift-test-agent", "pmcd", "pmlogger"]
 
 [customizations.firewall]
 ports = ["22:tcp"]

test/image-blueprints/layer1-base/group1/rhel98.toml

@@ -13,8 +13,12 @@ version = "*"
 name = "iproute-tc"
 version = "*"
 
+[[packages]]
+name = "pcp-zeroconf"
+version = "*"
+
 [customizations.services]
-enabled = ["microshift-test-agent"]
+enabled = ["microshift-test-agent", "pmcd", "pmlogger"]
 
 [customizations.firewall]
 ports = ["22:tcp"]

test/image-blueprints/layer2-presubmit/group1/rhel98-source-base.toml

@@ -25,8 +25,12 @@ version = "{{ env.Getenv "SOURCE_VERSION_BASE" }}"
 name = "microshift-test-agent"
 version = "*"
 
+[[packages]]
+name = "pcp-zeroconf"
+version = "*"
+
 [customizations.services]
-enabled = ["microshift", "microshift-test-agent"]
+enabled = ["microshift", "microshift-test-agent", "pmcd", "pmlogger"]
 
 [customizations.firewall]
 ports = ["22:tcp", "80:tcp", "443:tcp", "5353:udp", "6443:tcp", "30000-32767:tcp", "30000-32767:udp"]

test/image-blueprints/layer2-presubmit/group1/rhel98-source-fake-next-minor.toml

@@ -35,8 +35,12 @@ version = "{{ .Env.FAKE_NEXT_MAJOR_VERSION }}.{{ .Env.FAKE_NEXT_MINOR_VERSION }}
 name = "microshift-test-agent"
 version = "*"
 
+[[packages]]
+name = "pcp-zeroconf"
+version = "*"
+
 [customizations.services]
-enabled = ["microshift", "microshift-test-agent"]
+enabled = ["microshift", "microshift-test-agent", "pmcd", "pmlogger"]
 
 [customizations.firewall]
 ports = ["22:tcp", "80:tcp", "443:tcp", "5353:udp", "6443:tcp", "30000-32767:tcp", "30000-32767:udp"]

test/image-blueprints/layer2-presubmit/group1/rhel98-source-with-optionals.toml

@@ -39,12 +39,16 @@ version = "{{ env.Getenv "SOURCE_VERSION" }}"
 name = "microshift-test-agent"
 version = "*"
 
+[[packages]]
+name = "pcp-zeroconf"
+version = "*"
+
 [[packages]]
 name = "systemd-resolved"
 version = "*"
 
 [customizations.services]
-enabled = ["microshift", "microshift-test-agent"]
+enabled = ["microshift", "microshift-test-agent", "pmcd", "pmlogger"]
 
 [customizations.firewall]
 ports = ["22:tcp", "80:tcp", "443:tcp", "5353:udp", "6443:tcp", "30000-32767:tcp", "30000-32767:udp"]

test/image-blueprints/layer2-presubmit/group1/rhel98-source.toml

@@ -25,12 +25,16 @@ version = "{{ env.Getenv "SOURCE_VERSION" }}"
 name = "microshift-test-agent"
 version = "*"
 
+[[packages]]
+name = "pcp-zeroconf"
+version = "*"
+
 [[packages]]
 name = "systemd-resolved"
 version = "*"
 
 [customizations.services]
-enabled = ["microshift", "microshift-test-agent"]
+enabled = ["microshift", "microshift-test-agent", "pmcd", "pmlogger"]
 
 [customizations.firewall]
 ports = ["22:tcp", "80:tcp", "443:tcp", "5353:udp", "6443:tcp", "30000-32767:tcp", "30000-32767:udp"]

test/image-blueprints/layer3-periodic/group1/rhel98-source-isolated.toml

@@ -15,6 +15,10 @@ version = "{{ env.Getenv "SOURCE_VERSION" }}"
 name = "microshift-test-agent"
 version = "*"
 
+[[packages]]
+name = "pcp-zeroconf"
+version = "*"
+
 [[packages]]
 name = "qemu-guest-agent"
 version = "*"
@@ -34,7 +38,7 @@ name = "skopeo"
 version = "*"
 
 [customizations.services]
-enabled = ["microshift", "microshift-test-agent", "qemu-guest-agent"]
+enabled = ["microshift", "microshift-test-agent", "qemu-guest-agent", "pmcd", "pmlogger"]
 
 [customizations.firewall]
 ports = ["22:tcp", "80:tcp", "443:tcp", "5353:udp", "6443:tcp", "30000-32767:tcp", "30000-32767:udp"]

test/image-blueprints/layer4-release/group1/rhel96-brew-y2-with-optionals.toml

@@ -39,8 +39,12 @@ version = "{{ env.Getenv "BREW_Y2_RELEASE_VERSION" }}"
 name = "microshift-test-agent"
 version = "*"
 
+[[packages]]
+name = "pcp-zeroconf"
+version = "*"
+
 [customizations.services]
-enabled = ["microshift", "microshift-test-agent"]
+enabled = ["microshift", "microshift-test-agent", "pmcd", "pmlogger"]
 
 [customizations.firewall]
 ports = [

test/image-blueprints/layer4-release/group2/rhel96-brew-y1-with-optionals.toml

@@ -43,8 +43,12 @@ version = "{{ env.Getenv "BREW_Y1_RELEASE_VERSION" }}"
 name = "microshift-test-agent"
 version = "*"
 
+[[packages]]
+name = "pcp-zeroconf"
+version = "*"
+
 [customizations.services]
-enabled = ["microshift", "microshift-test-agent"]
+enabled = ["microshift", "microshift-test-agent", "pmcd", "pmlogger"]
 
 [customizations.firewall]
 ports = [

test/image-blueprints/layer4-release/group3/rhel98-brew-lrel-optional.toml

@@ -43,12 +43,16 @@ version = "{{ env.Getenv "BREW_LREL_RELEASE_VERSION" }}"
 name = "microshift-test-agent"
 version = "*"
 
+[[packages]]
+name = "pcp-zeroconf"
+version = "*"
+
 [[packages]]
 name = "systemd-resolved"
 version = "*"
 
 [customizations.services]
-enabled = ["microshift", "microshift-test-agent"]
+enabled = ["microshift", "microshift-test-agent", "pmcd", "pmlogger"]
 
 [customizations.firewall]
 ports = [

test/image-blueprints/layer4-release/group3/rhel98-brew-nightly-with-optionals.toml

@@ -43,12 +43,16 @@ version = "{{ env.Getenv "BREW_NIGHTLY_RELEASE_VERSION" }}"
 name = "microshift-test-agent"
 version = "*"
 
+[[packages]]
+name = "pcp-zeroconf"
+version = "*"
+
 [[packages]]
 name = "systemd-resolved"
 version = "*"
 
 [customizations.services]
-enabled = ["microshift", "microshift-test-agent"]
+enabled = ["microshift", "microshift-test-agent", "pmcd", "pmlogger"]
 
 [customizations.firewall]
 ports = [

test/scenario_settings.sh.example

@@ -13,6 +13,10 @@ SSH_PUBLIC_KEY=${SSH_PUBLIC_KEY:-${HOME}/.ssh/id_rsa.pub}
 # local environment where sos can be run manually when necessary.
 #SKIP_SOS=true
 
+# Disable PCP metrics collection. This can speed up teardown in a
+# local environment where PCP can be run manually when necessary.
+#SKIP_PCP=true
+
 # Whether to add a VNC graphics console to hosts. This is useful in
 # local developer settings where cockpit can be used to login to the
 # host. Set to `true` to enable. Defaults to `false`.