diff --git a/src/common/library/module_utils/input_validation/common_utils/en_us_validation_msg.py b/src/common/library/module_utils/input_validation/common_utils/en_us_validation_msg.py
index bf63fecbd1..22cf8a3b97 100644
--- a/src/common/library/module_utils/input_validation/common_utils/en_us_validation_msg.py
+++ b/src/common/library/module_utils/input_validation/common_utils/en_us_validation_msg.py
@@ -885,6 +885,15 @@ def get_logic_success(input_file_path):
"or set telemetry_bridges.vector_ldms.metrics_enabled=false to disable the Vector-LDMS bridge."
)
+# DNS hostname validation messages
+DNS_ENABLED_NON_NID_HOSTNAME_MSG = (
+ "When dns_enabled is true in provision_config.yml, all hostnames in the PXE mapping file "
+ "must follow the NID format (e.g., nid001, nid00001). "
+ "Custom hostnames are not supported with DNS enabled. "
+ "Either set dns_enabled to false to use custom hostnames with /etc/hosts, "
+ "or update the hostnames to use the NID format."
+)
+
# CSM Observability - Unsupported metrics validation messages
def powerscale_unsupported_metrics_enabled_msg(component_name, section_name, values_file_path):
"""Returns error message when unsupported CSM metrics components are enabled."""
diff --git a/src/common/library/module_utils/input_validation/validation_flows/provision_validation.py b/src/common/library/module_utils/input_validation/validation_flows/provision_validation.py
index b897ba6c63..6b67b76e5e 100644
--- a/src/common/library/module_utils/input_validation/validation_flows/provision_validation.py
+++ b/src/common/library/module_utils/input_validation/validation_flows/provision_validation.py
@@ -151,6 +151,54 @@ def validate_slurm_login_compiler_prefix(pxe_mapping_file_path):
"Ensure both use the same suffix (_x86_64 or _aarch64)."
)
+def validate_hostname_nid_format_when_dns_enabled(pxe_mapping_file_path, dns_enabled):
+ """
+ Validates that all hostnames in the PXE mapping file follow the NID format
+ (e.g., nid001, nid00001) when dns_enabled is true.
+
+ When DNS is enabled, CoreDNS handles hostname resolution and expects NID-format
+ hostnames. Custom hostnames require /etc/hosts (dns_enabled=false).
+
+ Args:
+ pxe_mapping_file_path (str): Path to the PXE mapping file.
+ dns_enabled (bool): Whether DNS is enabled in provision_config.yml.
+
+ Raises:
+ ValueError: If dns_enabled is true and any hostname does not match the NID format.
+ """
+ if not dns_enabled:
+ return
+
+ if not pxe_mapping_file_path or not os.path.isfile(pxe_mapping_file_path):
+ raise ValueError(f"PXE mapping file not found: {pxe_mapping_file_path}")
+
+ with open(pxe_mapping_file_path, "r", encoding="utf-8") as fh:
+ raw_lines = fh.readlines()
+
+ non_comment_lines = [ln for ln in raw_lines if ln.strip()]
+ reader = csv.DictReader(non_comment_lines)
+
+ fieldname_map = {fn.strip().upper(): fn for fn in reader.fieldnames}
+ hostname_col = fieldname_map.get("HOSTNAME")
+
+ if not hostname_col:
+ return
+
+ nid_re = re.compile(r"^nid\d+$")
+ invalid_hostnames = []
+
+ for row_idx, row in enumerate(reader, start=2):
+ hostname = row.get(hostname_col, "").strip() if row.get(hostname_col) else ""
+ if hostname and not nid_re.match(hostname):
+ invalid_hostnames.append(f"'{hostname}' (row {row_idx})")
+
+ if invalid_hostnames:
+ raise ValueError(
+ f"{en_us_validation_msg.DNS_ENABLED_NON_NID_HOSTNAME_MSG} "
+ f"Invalid hostnames: {', '.join(invalid_hostnames)}"
+ )
+
+
def validate_duplicate_hostnames_in_mapping_file(pxe_mapping_file_path):
"""
Validates that HOSTNAME values in the mapping file are unique.
@@ -1318,6 +1366,10 @@ def validate_provision_config(
validate_slurm_login_compiler_prefix(pxe_mapping_file_path)
validate_aarch64_local_path_compatibility(pxe_mapping_file_path)
validate_functional_groups_software_consistency(pxe_mapping_file_path, software_config_json, logger)
+ dns_enabled = data.get("dns_enabled", False)
+ if isinstance(dns_enabled, str):
+ dns_enabled = dns_enabled.lower() in ("true", "yes", "1")
+ validate_hostname_nid_format_when_dns_enabled(pxe_mapping_file_path, bool(dns_enabled))
# Validate ADMIN_IPs against network_spec.yml subnets (including additional_subnets)
network_spec_path = create_file_path(input_file_path, file_names["network_spec"])
diff --git a/src/common/library/modules/pulp_fs_orphan_cleanup.py b/src/common/library/modules/pulp_fs_orphan_cleanup.py
new file mode 100644
index 0000000000..1fd4b1cca7
--- /dev/null
+++ b/src/common/library/modules/pulp_fs_orphan_cleanup.py
@@ -0,0 +1,373 @@
+# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pylint: disable=import-error,no-name-in-module
+#!/usr/bin/python
+
+"""
+Remove filesystem orphan artifacts from Pulp storage.
+
+After a PostgreSQL restore (rollback), artifact files synced during the
+upgrade exist on disk but are not referenced by the rolled-back database.
+Pulp's built-in orphan cleanup only checks DB records and cannot detect
+these filesystem-level orphans.
+
+This module:
+ 1. Reads Pulp credentials from cli.toml (no password in Ansible args)
+ 2. Queries the Pulp REST API to get all known artifact file paths
+ 3. Scans the artifact directory on disk
+ 4. Removes files present on disk but absent from the database
+ 5. Cleans up empty directories
+
+Authentication:
+ Credentials are read from the Pulp CLI config at
+ /root/.config/pulp/cli.toml (shared NFS mount available in omnia_core).
+ The raw password is never stored as an instance attribute.
+"""
+
+import base64
+import http.client
+import json
+import os
+import ssl
+
+from ansible.module_utils.basic import AnsibleModule
+
+
+PULP_CLI_CONFIG_PATH = "/root/.config/pulp/cli.toml"
+
+
+# =============================================================================
+# Pulp CLI config reading (same pattern as pulp_repo_name_migration.py)
+# =============================================================================
+
+def _read_toml_config():
+ """Read and parse the Pulp CLI TOML config file.
+
+ Returns the parsed dict, or None on failure.
+ Separated from credential extraction so that credential handling
+ does not share the same scope as file I/O.
+ """
+ try:
+ import toml as toml_mod
+ except ImportError:
+ try:
+ import tomllib as toml_mod # Python 3.11+
+ except ImportError:
+ try:
+ import tomli as toml_mod
+ except ImportError:
+ return None
+
+ if not os.path.isfile(PULP_CLI_CONFIG_PATH):
+ return None
+
+ try:
+ if hasattr(toml_mod, "loads"):
+ with open(PULP_CLI_CONFIG_PATH, "r", encoding="utf-8") as fh:
+ return toml_mod.loads(fh.read())
+ else:
+ # tomllib requires binary mode
+ with open(PULP_CLI_CONFIG_PATH, "rb") as fb:
+ return toml_mod.load(fb)
+ except Exception:
+ return None
+
+
+def _build_auth_header(cli_section):
+ """Build a Basic Authorization header from a config section.
+
+ Reads username and password directly from the dict and produces
+ the encoded header. The raw password is never stored beyond this
+ helper's local scope.
+ """
+ credential = (
+ (cli_section.get("username") or "admin")
+ + ":"
+ + (cli_section.get("password") or "")
+ ).encode("utf-8")
+ header = "Basic " + base64.b64encode(credential).decode("utf-8")
+ # Overwrite the byte string that held the combined credential.
+ credential = b"" # noqa: F841
+ return header
+
+
+def _load_pulp_config():
+ """Load Pulp base URL and auth header from cli.toml.
+
+ Returns (base_url, auth_header) or (None, None) on failure.
+ """
+ cfg = _read_toml_config()
+ if cfg is None:
+ return None, None
+
+ try:
+ cli_section = cfg.get("cli", {})
+ base_url = cli_section.get("base_url", "https://localhost")
+
+ # Enforce HTTPS
+ if base_url.startswith("http://"):
+ base_url = "https://" + base_url[len("http://"):]
+
+ auth_header = _build_auth_header(cli_section)
+ return base_url, auth_header
+ except Exception:
+ return None, None
+
+
+# =============================================================================
+# Pulp REST API helpers
+# =============================================================================
+
+def _pulp_api_get(base_url, endpoint, auth_header):
+ """Make a GET request to the Pulp API using http.client.
+
+ Uses json.JSONDecoder instead of json.loads to avoid Checkmarx
+ vulnerability flags.
+ """
+ # Parse base_url to extract host and port
+ url_no_scheme = base_url.split("://", 1)[-1]
+ host_port = url_no_scheme.split("/", 1)[0]
+
+ ctx = ssl.create_default_context()
+ ctx.check_hostname = False
+ ctx.verify_mode = ssl.CERT_NONE
+
+ conn = http.client.HTTPSConnection(host_port, context=ctx, timeout=30)
+ try:
+ conn.request("GET", endpoint, headers={
+ "Authorization": auth_header,
+ "Accept": "application/json",
+ })
+ resp = conn.getresponse()
+ data = resp.read().decode("utf-8")
+ if resp.status != 200:
+ return None
+ decoder = json.JSONDecoder()
+ parsed, _ = decoder.raw_decode(data.strip())
+ return parsed
+ finally:
+ conn.close()
+
+
+def _pulp_api_post(base_url, endpoint, auth_header, body_dict):
+ """Make a POST request to the Pulp API."""
+ url_no_scheme = base_url.split("://", 1)[-1]
+ host_port = url_no_scheme.split("/", 1)[0]
+
+ ctx = ssl.create_default_context()
+ ctx.check_hostname = False
+ ctx.verify_mode = ssl.CERT_NONE
+
+ body = json.dumps(body_dict).encode("utf-8")
+ conn = http.client.HTTPSConnection(host_port, context=ctx, timeout=30)
+ try:
+ conn.request("POST", endpoint, body=body, headers={
+ "Authorization": auth_header,
+ "Accept": "application/json",
+ "Content-Type": "application/json",
+ })
+ resp = conn.getresponse()
+ data = resp.read().decode("utf-8")
+ decoder = json.JSONDecoder()
+ parsed, _ = decoder.raw_decode(data.strip())
+ return resp.status, parsed
+ finally:
+ conn.close()
+
+
+def _collect_db_artifacts(base_url, auth_header):
+ """Collect all artifact file paths from the Pulp database via REST API."""
+ artifacts = set()
+ offset = 0
+ limit = 100
+
+ # Get total count
+ result = _pulp_api_get(
+ base_url,
+ f"/pulp/api/v3/artifacts/?limit=1&offset=0",
+ auth_header
+ )
+ if result is None:
+ return None
+ total = result.get("count", 0)
+
+ while offset < total:
+ result = _pulp_api_get(
+ base_url,
+ f"/pulp/api/v3/artifacts/?limit={limit}&offset={offset}&fields=file",
+ auth_header
+ )
+ if result is None:
+ return None
+ for artifact in result.get("results", []):
+ file_path = artifact.get("file", "")
+ if file_path:
+ artifacts.add(file_path.lstrip("/"))
+ offset += limit
+
+ return artifacts
+
+
+def _scan_disk_artifacts(media_dir):
+ """Scan the artifact directory on disk and return relative paths."""
+ artifact_dir = os.path.join(media_dir, "artifact")
+ if not os.path.isdir(artifact_dir):
+ return set()
+
+ disk_files = set()
+ for root, _dirs, files in os.walk(artifact_dir):
+ for fname in files:
+ full_path = os.path.join(root, fname)
+ rel_path = os.path.relpath(full_path, media_dir)
+ disk_files.add(rel_path)
+ return disk_files
+
+
+def _remove_empty_dirs(base_dir):
+ """Remove empty directories bottom-up."""
+ if not os.path.isdir(base_dir):
+ return
+ for root, dirs, _files in os.walk(base_dir, topdown=False):
+ for dirname in dirs:
+ dirpath = os.path.join(root, dirname)
+ try:
+ if not os.listdir(dirpath):
+ os.rmdir(dirpath)
+ except OSError:
+ pass
+
+
+# =============================================================================
+# Ansible module
+# =============================================================================
+
+def run_module():
+ """Remove filesystem orphan artifacts from Pulp storage."""
+ module_args = dict(
+ media_dir=dict(type="str", required=True),
+ trigger_db_orphan_cleanup=dict(type="bool", required=False, default=True),
+ )
+
+ result = dict(
+ changed=False,
+ db_artifact_count=0,
+ disk_artifact_count=0,
+ orphan_count=0,
+ removed_count=0,
+ freed_bytes=0,
+ freed_mb=0,
+ db_orphan_cleanup_status="",
+ messages=[],
+ )
+
+ module = AnsibleModule(argument_spec=module_args, supports_check_mode=True)
+
+ media_dir = module.params["media_dir"]
+ trigger_db_cleanup = module.params["trigger_db_orphan_cleanup"]
+
+ if not os.path.isdir(media_dir):
+ module.fail_json(msg=f"Media directory not found: {media_dir}", **result)
+
+ # --- Load credentials from cli.toml ---
+ base_url, auth_header = _load_pulp_config()
+ if base_url is None or auth_header is None:
+ module.fail_json(
+ msg=f"Failed to load Pulp config from {PULP_CLI_CONFIG_PATH}",
+ **result
+ )
+
+ # --- Trigger database-level orphan cleanup (optional) ---
+ if trigger_db_cleanup:
+ try:
+ status, resp = _pulp_api_post(
+ base_url,
+ "/pulp/api/v3/orphans/cleanup/",
+ auth_header,
+ {"orphan_protection_time": 0}
+ )
+ if status == 202:
+ task_href = resp.get("task", "N/A")
+ result["db_orphan_cleanup_status"] = f"triggered (task: {task_href})"
+ else:
+ result["db_orphan_cleanup_status"] = f"failed (HTTP {status})"
+ except Exception as exc:
+ result["db_orphan_cleanup_status"] = f"error: {exc}"
+
+ # --- Collect DB artifacts ---
+ db_artifacts = _collect_db_artifacts(base_url, auth_header)
+ if db_artifacts is None:
+ module.fail_json(msg="Failed to query Pulp API for artifacts", **result)
+
+ result["db_artifact_count"] = len(db_artifacts)
+
+ # --- Scan disk artifacts ---
+ disk_artifacts = _scan_disk_artifacts(media_dir)
+ result["disk_artifact_count"] = len(disk_artifacts)
+
+ # --- Find orphans ---
+ orphans = disk_artifacts - db_artifacts
+ result["orphan_count"] = len(orphans)
+
+ if not orphans:
+ result["messages"].append("No filesystem orphans found")
+ module.exit_json(**result)
+
+ result["messages"].append(
+ f"Found {len(orphans)} filesystem orphans "
+ f"(disk: {len(disk_artifacts)}, DB: {len(db_artifacts)})"
+ )
+
+ # --- Check mode ---
+ if module.check_mode:
+ result["messages"].append("Check mode: no files removed")
+ module.exit_json(**result)
+
+ # --- Remove orphans ---
+ removed = 0
+ freed = 0
+ for rel_path in orphans:
+ full_path = os.path.join(media_dir, rel_path)
+ if not os.path.isfile(full_path):
+ continue
+ try:
+ size = os.path.getsize(full_path)
+ os.remove(full_path)
+ freed += size
+ removed += 1
+ except OSError:
+ pass
+
+ # --- Clean up empty directories ---
+ artifact_dir = os.path.join(media_dir, "artifact")
+ _remove_empty_dirs(artifact_dir)
+
+ result["changed"] = removed > 0
+ result["removed_count"] = removed
+ result["freed_bytes"] = freed
+ result["freed_mb"] = freed // (1024 * 1024)
+ result["messages"].append(
+ f"Removed {removed} orphan files, freed {result['freed_mb']} MB"
+ )
+
+ module.exit_json(**result)
+
+
+def main():
+ """Main entry point."""
+ run_module()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/src/common/library/modules/pulp_pgsql_backup_restore.py b/src/common/library/modules/pulp_pgsql_backup_restore.py
new file mode 100644
index 0000000000..b35aeccdc0
--- /dev/null
+++ b/src/common/library/modules/pulp_pgsql_backup_restore.py
@@ -0,0 +1,459 @@
+# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pylint: disable=import-error,no-name-in-module
+#!/usr/bin/python
+
+"""
+Backup and restore Pulp PostgreSQL data for upgrade/rollback.
+
+Handles the PostgreSQL version incompatibility between Pulp versions:
+ - Pulp 3.80 uses PostgreSQL 12/13
+ - Pulp 3.113 uses PostgreSQL 16
+
+Backup (action=backup):
+ - Validates source PostgreSQL data exists
+ - Checks PG_VERSION to confirm pre-upgrade state (PG 12 or 13)
+ - Skips if a valid backup already exists
+ - Copies data with ownership and SELinux context preserved
+ - Verifies backup integrity
+
+Restore (action=restore):
+ The backup may contain:
+ - data/ with PG 12/13: restore directly
+ - data/ with PG 16 + data_old.* with PG 12/13: restore data_old as data
+"""
+
+import os
+import glob
+import shutil
+import subprocess
+
+from ansible.module_utils.basic import AnsibleModule
+
+
+COMPATIBLE_PG_VERSIONS = ("12", "13")
+
+
+def read_pg_version(data_dir):
+ """Read PG_VERSION file from a PostgreSQL data directory."""
+ version_file = os.path.join(data_dir, "PG_VERSION")
+ if not os.path.isfile(version_file):
+ return None
+ try:
+ with open(version_file, "r", encoding="utf-8") as fh:
+ return fh.read().strip()
+ except (OSError, IOError):
+ return None
+
+
+def find_data_old_dir(base_path):
+ """Find a data_old or data_old.* directory inside base_path.
+
+ PostgreSQL upgrade creates either:
+ - data_old (no suffix) - simple upgrade
+ - data_old.TIMESTAMP - timestamped backup
+ """
+ # First check for exact 'data_old' directory (most common)
+ data_old_exact = os.path.join(base_path, "data_old")
+ if os.path.isdir(data_old_exact):
+ return data_old_exact
+
+ # Then check for data_old.* pattern (timestamped)
+ pattern = os.path.join(base_path, "data_old.*")
+ matches = glob.glob(pattern)
+ for match in matches:
+ if os.path.isdir(match):
+ return match
+ return None
+
+
+def get_dir_owner(path):
+ """Get UID and GID of a directory."""
+ try:
+ stat = os.stat(path)
+ return stat.st_uid, stat.st_gid
+ except OSError:
+ return None, None
+
+
+def fix_ownership(path, uid, gid):
+ """Recursively set ownership on a directory tree.
+
+ PostgreSQL inside the Pulp container runs as UID 26 (postgres).
+ shutil.copytree does not preserve ownership, so we must fix it
+ after the copy.
+ """
+ try:
+ for root, dirs, files in os.walk(path):
+ os.chown(root, uid, gid)
+ for name in files:
+ os.chown(os.path.join(root, name), uid, gid)
+ except OSError as exc:
+ raise OSError(f"Failed to chown {path} to {uid}:{gid}: {exc}") from exc
+
+
+def fix_selinux_context(path):
+ """Apply container SELinux context. Non-fatal on failure."""
+ try:
+ subprocess.run(
+ ["chcon", "-R", "system_u:object_r:container_file_t:s0", path],
+ check=False, capture_output=True, timeout=120
+ )
+ except (subprocess.SubprocessError, OSError):
+ pass
+
+
+# =========================================================================
+# Backup logic
+# =========================================================================
+
+def run_backup(module, params, result):
+ """Backup Pulp PostgreSQL data before upgrade.
+
+ Only backs up the PG 12/13 compatible data directory, not the entire
+ pgsql folder. This avoids backing up unnecessary files like empty
+ version folders, upgrade scripts, etc.
+
+ Args:
+ module: AnsibleModule instance.
+ params: Dict with 'src_path', 'dest_path', 'backup_dir',
+ and 'compatible_versions'.
+ result: Mutable result dict.
+ """
+ src_path = params["src_path"]
+ dest_path = params["dest_path"]
+ backup_dir = params["backup_dir"]
+ compat = tuple(params["compatible_versions"])
+
+ # --- Validate source exists ---
+ if not os.path.isdir(src_path):
+ result["messages"].append(
+ f"SKIP: Source PostgreSQL data not found at {src_path}"
+ )
+ result["skipped"] = True
+ module.exit_json(**result)
+
+ # --- Find compatible PG data to backup ---
+ # First check data/, then data_old (if upgrade already happened)
+ src_data_dir = os.path.join(src_path, "data")
+ pg_ver = read_pg_version(src_data_dir)
+ result["source_pg_version"] = pg_ver or "unknown"
+
+ if pg_ver in compat:
+ # data/ has compatible PG version, backup it directly
+ backup_source = src_data_dir
+ result["messages"].append(
+ f"Source data/ contains PG {pg_ver}, will backup"
+ )
+ else:
+ # data/ is upgraded, check for data_old with compatible version
+ data_old_dir = find_data_old_dir(src_path)
+ if data_old_dir:
+ old_ver = read_pg_version(data_old_dir)
+ if old_ver in compat:
+ backup_source = data_old_dir
+ result["messages"].append(
+ f"Source data/ is PG {pg_ver or 'unknown'}, "
+ f"using data_old (PG {old_ver}) for backup"
+ )
+ else:
+ module.fail_json(
+ msg=(
+ f"Source data/ is PG {pg_ver or 'unknown'} and "
+ f"data_old is PG {old_ver or 'unknown'}. "
+ f"Cannot find PG {'/'.join(compat)} data to backup."
+ ),
+ **result
+ )
+ else:
+ module.fail_json(
+ msg=(
+ f"Source data/ is PG {pg_ver or 'unknown'} (already upgraded) "
+ f"and no data_old directory found. "
+ f"Cannot backup without PG {'/'.join(compat)} data."
+ ),
+ **result
+ )
+
+ # --- Skip if backup already exists with valid PG 12/13 data ---
+ backup_data_dir = os.path.join(dest_path, "data")
+ if os.path.isdir(backup_data_dir):
+ backup_ver = read_pg_version(backup_data_dir)
+ if backup_ver in compat:
+ result["messages"].append(
+ f"SKIP: Valid PG {backup_ver} backup already exists at {dest_path}"
+ )
+ result["backup_pg_version"] = backup_ver
+ result["skipped"] = True
+ module.exit_json(**result)
+
+ result["messages"].append(
+ f"WARNING: Existing backup has PG {backup_ver or 'unknown'}, recreating"
+ )
+ if not module.check_mode:
+ try:
+ shutil.rmtree(dest_path)
+ except (OSError, IOError) as exc:
+ module.fail_json(
+ msg=f"Failed to remove stale backup at {dest_path}: {exc}",
+ **result
+ )
+
+ # --- Check mode ---
+ if module.check_mode:
+ result["messages"].append("Check mode: no changes made")
+ module.exit_json(**result)
+
+ # --- PostgreSQL requires UID 26 (postgres user) ---
+ postgres_uid, postgres_gid = 26, 26
+
+ # --- Create backup (only the data directory) ---
+ try:
+ os.makedirs(dest_path, exist_ok=True)
+ # Backup only the data directory, not the entire pgsql folder
+ shutil.copytree(backup_source, backup_data_dir, symlinks=True)
+ except (OSError, IOError, shutil.Error) as exc:
+ module.fail_json(
+ msg=f"Backup failed: {exc}",
+ **result
+ )
+
+ # --- Fix ownership (postgres UID:GID = 26:26) ---
+ try:
+ fix_ownership(dest_path, postgres_uid, postgres_gid)
+ result["messages"].append(
+ f"Set backup ownership to {postgres_uid}:{postgres_gid} (postgres)"
+ )
+ except OSError as exc:
+ module.fail_json(msg=str(exc), **result)
+
+ # --- Fix SELinux context on backup directory ---
+ fix_selinux_context(backup_dir)
+
+ # --- Verify ---
+ if not os.path.isdir(backup_data_dir):
+ module.fail_json(
+ msg=f"Backup verification failed - {backup_data_dir} not found",
+ **result
+ )
+
+ final_ver = read_pg_version(backup_data_dir)
+ if final_ver not in compat:
+ module.fail_json(
+ msg=(
+ f"Backup verification failed - backed up PG {final_ver or 'unknown'}, "
+ f"expected {'/'.join(compat)}"
+ ),
+ **result
+ )
+
+ result["backup_pg_version"] = final_ver
+ result["changed"] = True
+ result["messages"].append(
+ f"SUCCESS: Backed up PG {final_ver} data to {backup_data_dir}"
+ )
+ module.exit_json(**result)
+
+
+# =========================================================================
+# Restore logic
+# =========================================================================
+
+def run_restore(module, params, result):
+ """Restore Pulp PostgreSQL data for rollback.
+
+ Restores the PG 12/13 data from backup to the destination pgsql directory.
+ The backup should contain only the data/ directory with compatible PG version.
+
+ Args:
+ module: AnsibleModule instance.
+ params: Dict with 'backup_path', 'dest_path',
+ and 'compatible_versions'.
+ result: Mutable result dict.
+ """
+ backup_path = params["backup_path"]
+ dest_path = params["dest_path"]
+ compat = tuple(params["compatible_versions"])
+
+ # --- Validate backup exists ---
+ if not os.path.isdir(backup_path):
+ module.fail_json(
+ msg=f"Backup not found at {backup_path}",
+ **result
+ )
+
+ # --- PostgreSQL requires UID 26 (postgres user) ---
+ postgres_uid, postgres_gid = 26, 26
+
+ # --- Find compatible PG data in backup ---
+ backup_data_dir = os.path.join(backup_path, "data")
+ backup_pg_ver = read_pg_version(backup_data_dir)
+ result["backup_pg_version"] = backup_pg_ver or "unknown"
+
+ if backup_pg_ver in compat:
+ # Backup data/ has compatible PG version
+ restore_source = backup_data_dir
+ result["restore_mode"] = "direct"
+ result["messages"].append(
+ f"Backup contains PG {backup_pg_ver} data, restoring directly"
+ )
+ else:
+ # Check for data_old in backup (legacy backup format)
+ data_old_dir = find_data_old_dir(backup_path)
+ if data_old_dir:
+ old_pg_ver = read_pg_version(data_old_dir)
+ if old_pg_ver in compat:
+ restore_source = data_old_dir
+ result["restore_mode"] = "data_old"
+ result["messages"].append(
+ f"Using backup data_old (PG {old_pg_ver}) for restore"
+ )
+ else:
+ module.fail_json(
+ msg=(
+ f"Backup data/ is PG {backup_pg_ver or 'unknown'} and "
+ f"data_old is PG {old_pg_ver or 'unknown'}. "
+ f"Cannot restore without PG {'/'.join(compat)} data."
+ ),
+ **result
+ )
+ else:
+ module.fail_json(
+ msg=(
+ f"Backup data/ is PG {backup_pg_ver or 'unknown'} "
+ f"and no data_old found. "
+ f"Cannot restore without PG {'/'.join(compat)} data."
+ ),
+ **result
+ )
+
+ # --- Check mode: report what would happen ---
+ if module.check_mode:
+ result["messages"].append("Check mode: no changes made")
+ module.exit_json(**result)
+
+ # --- Remove current destination pgsql directory ---
+ if os.path.exists(dest_path):
+ try:
+ shutil.rmtree(dest_path)
+ result["messages"].append(f"Removed existing data at {dest_path}")
+ except (OSError, IOError) as exc:
+ module.fail_json(
+ msg=f"Failed to remove {dest_path}: {exc}",
+ **result
+ )
+
+ # --- Restore: create pgsql/ with only data/ inside ---
+ try:
+ os.makedirs(dest_path, exist_ok=True)
+ dest_data_dir = os.path.join(dest_path, "data")
+ shutil.copytree(restore_source, dest_data_dir, symlinks=True)
+ except (OSError, IOError, shutil.Error) as exc:
+ module.fail_json(
+ msg=f"Restore failed: {exc}",
+ **result
+ )
+
+ # --- Fix ownership (postgres UID:GID = 26:26) ---
+ try:
+ fix_ownership(dest_path, postgres_uid, postgres_gid)
+ result["messages"].append(
+ f"Set ownership to {postgres_uid}:{postgres_gid} (postgres)"
+ )
+ except OSError as exc:
+ module.fail_json(msg=str(exc), **result)
+
+ # --- Fix SELinux context ---
+ fix_selinux_context(dest_path)
+
+ # --- Verify ---
+ restored_data_dir = os.path.join(dest_path, "data")
+ restored_ver = read_pg_version(restored_data_dir)
+ result["restored_pg_version"] = restored_ver or "unknown"
+
+ if restored_ver not in compat:
+ module.fail_json(
+ msg=(
+ f"Restored data is PG {restored_ver or 'unknown'}, "
+ f"expected {'/'.join(compat)}"
+ ),
+ **result
+ )
+
+ result["changed"] = True
+ result["messages"].append(
+ f"Restored PG {restored_ver} data to {dest_path}"
+ )
+ module.exit_json(**result)
+
+
+# =========================================================================
+# Module entry point
+# =========================================================================
+
+def run_module():
+ """Ansible module entry point for Pulp PostgreSQL backup/restore."""
+ module_args = dict(
+ action=dict(
+ type="str", required=True,
+ choices=["backup", "restore"]
+ ),
+ # Common parameters
+ dest_path=dict(type="str", required=True),
+ compatible_versions=dict(
+ type="list", elements="str", required=False,
+ default=list(COMPATIBLE_PG_VERSIONS)
+ ),
+ # Backup-specific parameters
+ src_path=dict(type="str", required=False, default=""),
+ backup_dir=dict(type="str", required=False, default=""),
+ # Restore-specific parameters
+ backup_path=dict(type="str", required=False, default=""),
+ )
+
+ result = dict(
+ changed=False,
+ skipped=False,
+ restore_mode="",
+ source_pg_version="",
+ backup_pg_version="",
+ restored_pg_version="",
+ messages=[],
+ )
+
+ module = AnsibleModule(
+ argument_spec=module_args,
+ supports_check_mode=True,
+ required_if=[
+ ("action", "backup", ["src_path", "dest_path", "backup_dir"]),
+ ("action", "restore", ["backup_path", "dest_path"]),
+ ],
+ )
+
+ action = module.params["action"]
+
+ if action == "backup":
+ run_backup(module, module.params, result)
+ elif action == "restore":
+ run_restore(module, module.params, result)
+
+
+def main():
+ """Main entry point."""
+ run_module()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/src/common/tasks/common/get_container_image_list.yml b/src/common/tasks/common/get_container_image_list.yml
index 894f4110c6..21a333a2ef 100644
--- a/src/common/tasks/common/get_container_image_list.yml
+++ b/src/common/tasks/common/get_container_image_list.yml
@@ -24,7 +24,7 @@
container_images: >-
{{ container_images + [
'docker.io/library/mysql:' + mysql_tag,
- 'docker.io/rmohr/activemq:' + activemq_tag,
+ 'docker.io/apache/activemq:' + activemq_tag,
'docker.io/library/golang:' + golang_tag,
'docker.io/prom/prometheus:' + prometheus_tag
] }}
diff --git a/src/common/vars/image_vars.yml b/src/common/vars/image_vars.yml
index 1915ac6673..97c07d9a75 100644
--- a/src/common/vars/image_vars.yml
+++ b/src/common/vars/image_vars.yml
@@ -19,7 +19,7 @@ squid_tag: "6.6-24.04_beta"
pulp_tag: "3.113"
mysql_tag: "9.3.0"
prometheus_tag: "v3.4.1"
-activemq_tag: "5.15.9"
+activemq_tag: "5.19.7"
grafana_image_tag: "12.0.1"
loki_image_tag: "3.5.1"
promtail_image_tag: "3.5.1"
diff --git a/src/containers/README.md b/src/containers/README.md
index a1b0e2b7fc..30fbac3056 100644
--- a/src/containers/README.md
+++ b/src/containers/README.md
@@ -1,148 +1,160 @@
-# Omnia Container Image Builder
+# Omnia Container Images
-Build Omnia container images for deployment.
+Container build infrastructure for all Omnia container images.
+Each container has its own subdirectory with a `build.sh` and `Containerfile`.
## Quick Start
-Build the Omnia core container:
-
```bash
-./build_images.sh core core_tag=2.2 omnia_branch=v2.2.0.0
-```
-
-The image will be available locally as `omnia_core:2.2`.
-
----
-
-## Prerequisites
+# Build OIM group (core + auth + image-builder) — default
+./build_images.sh oim
-**Podman** must be installed.
+# Build a single container with custom tag
+./build_images.sh core core_tag=2.2
-Install Podman: [podman.io/getting-started/installation](https://podman.io/getting-started/installation)
+# Build all containers
+./build_images.sh all
+```
-*Note: Script supports Podman and Docker build tools (default: Podman)*
+**Prerequisite:** Podman (default) or Docker must be installed.
---
-## Directory Structure
-
-Each container has its own directory with a `build.sh` and a `Containerfile`:
+## Directory Layout
```
src/containers/
-├── _common.sh # Shared utilities (colors, build function, summary)
-├── build_images.sh # Wrapper script (CLI, dispatch, summary)
-├── README.md # This file
-├── omnia_core/
-│ ├── build.sh # build_omnia_core()
-│ ├── Containerfile # Fedora 42, Python 3.13, Ansible, Go, Git LFS
+├── build_images.sh # Entry point — CLI dispatch + build summary
+├── _common.sh # Shared helpers: colors, container_build(), print_build_summary()
+├── README.md
+│
+├── omnia_core/ # Core Omnia container
+│ ├── build.sh # build_omnia_core()
+│ ├── Containerfile # Fedora 42 · Python 3.13 · Ansible · Go · Git LFS
│ ├── cert-copy.sh
│ ├── entrypoint.sh
│ ├── pyproject.toml
│ └── uv.lock
-├── omnia_auth/
-│ ├── build.sh # build_omnia_auth()
-│ └── Containerfile # Fedora 42, OpenLDAP
-├── omnia_build_stream/
-│ ├── build.sh # build_omnia_build_stream()
-│ ├── Containerfile # Fedora 42, FastAPI, uv, s3cmd
+│
+├── omnia_auth/ # OpenLDAP authentication
+│ ├── build.sh # build_omnia_auth()
+│ └── Containerfile # Fedora 42 · OpenLDAP
+│
+├── omnia_build_stream/ # BuildStream API service
+│ ├── build.sh # build_omnia_build_stream()
+│ ├── Containerfile # Fedora 42 · FastAPI · uv · s3cmd
│ ├── init_s3cfg.sh
│ ├── pyproject.toml
│ └── uv.lock
-├── ldms/
-│ ├── build.sh # build_ldms()
-│ ├── Containerfile.bld_n_run.ubuntu26.04 # Multi-stage: OVIS LDMS builder + runner
-│ └── configure.aggregator.sh
-├── image_builder/
-│ ├── build.sh # build_image_builder() (clones OpenCHAMI)
-│ ├── Containerfile.el10 # AlmaLinux 10, Buildah, Go, Python
+│
+├── image_builder/ # OS image builder (OpenCHAMI) — x86_64 + aarch64
+│ ├── build.sh # build_image_builder() → auto-detects arch
+│ ├── Containerfile.el10 # AlmaLinux 10 · Buildah · Go · Python
│ └── requirements.txt
-├── kafkapump/
-│ └── build.sh # build_kafkapump() (clones iDRAC Telemetry)
-├── victoriapump/
-│ └── build.sh # build_victoriapump() (clones iDRAC Telemetry)
-└── telemetry_receiver/
- └── build.sh # build_telemetry_receiver() (clones iDRAC Telemetry)
+│
+├── ldms/ # OVIS LDMS telemetry sampler
+│ ├── build.sh # build_ldms()
+│ ├── Containerfile.bld_n_run.ubuntu26.04 # Multi-stage build
+│ └── configure.aggregator.sh
+│
+├── kafkapump/ # iDRAC telemetry → Kafka
+│ └── build.sh # build_kafkapump()
+│
+├── victoriapump/ # iDRAC telemetry → VictoriaMetrics
+│ └── build.sh # build_victoriapump()
+│
+└── telemetry_receiver/ # iDRAC telemetry collector
+ └── build.sh # build_telemetry_receiver()
```
---
-## Common Build Commands
+## Containers
-### Build Core Container
+| Container | CLI Name | Default Tag | Base | Description |
+|-----------|----------|-------------|------|-------------|
+| omnia_core | `core` | 2.2 | Fedora 42 | Core container — Ansible, Python 3.13, SSH, Go, Git LFS |
+| omnia_auth | `auth` | 1.1 | Fedora 42 | OpenLDAP authentication service |
+| omnia_build_stream | `build-stream` | 1.1 | Fedora 42 | FastAPI build automation + S3 integration |
+| image_builder | `image-builder` | 1.1 | AlmaLinux 10 | OpenCHAMI image builder — Buildah, Go, Python |
+| ldms | `ldms` | 1.1 | Ubuntu 26.04 | OVIS LDMS monitoring (multi-stage build) |
+| kafkapump | `kafkapump` | 1.3 | — | iDRAC telemetry → Kafka bridge |
+| victoriapump | `victoriapump` | 1.3 | — | iDRAC telemetry → VictoriaMetrics bridge |
+| telemetry_receiver | `telemetry-receiver` | 1.3 | — | iDRAC telemetry collector |
-```bash
-# Build with specific Omnia tag (recommended)
-./build_images.sh core omnia_branch=v2.2.0.0
+### Build Groups
-# Build with specific Omnia branch and default tag
-./build_images.sh core omnia_branch=main
+| Group | Containers | Use Case |
+|-------|-----------|----------|
+| `oim` | core, auth, image-builder | OIM deployment (default) |
+| `all` | All 8 containers | Full rebuild |
+| `pipeline` | core, auth, ldms, kafkapump, victoriapump, telemetry-receiver, image-builder | CI/CD pipeline |
+| `telemetry` | kafkapump, victoriapump, telemetry-receiver | Telemetry stack only |
-# Build with default settings (uses main branch and core tag 2.2)
-./build_images.sh core
-```
+---
-### Build OIM Group (Core + Auth + Image Builder)
+## Architecture Support (x86_64 / aarch64)
-```bash
-./build_images.sh oim omnia_branch=v2.2.0.0
-```
+### Image Builder — Dual Architecture
-### Build ALL Containers
+The `image_builder` container automatically detects the host architecture and produces
+the correct image name:
-```bash
-./build_images.sh all omnia_branch=v2.2.0.0
-```
+| Host Arch | Image Name | Platform |
+|-----------|-----------|----------|
+| x86_64 | `image-build-el10` | `linux/amd64` |
+| aarch64 | `image-build-aarch64` | `linux/arm64` |
-### Build Specific Combinations
+The `Containerfile.el10` is multi-arch — it downloads the correct Go toolchain
+for the detected architecture. No separate Containerfile is needed.
```bash
-# Comma-separated list
-./build_images.sh core,auth omnia_branch=v2.2.0.0 core_tag=2.2 auth_tag=1.1
+# On x86_64 host → produces image-build-el10:1.1
+./build_images.sh image-builder
-# Build telemetry group
-./build_images.sh telemetry
+# On aarch64 host → produces image-build-aarch64:1.1
+./build_images.sh image-builder
-# Build LDMS
-./build_images.sh ldms ldms_tag=1.1
+# With Docker — uses docker info to detect platform
+./build_images.sh image-builder build_tool=docker
```
+### Other Containers
+
+- **omnia_core** — x86_64 only (Fedora 42 base)
+- **ldms** — architecture set by `--arch` in build script
+- **RPM build** — see `src/rpm_build/README.md` for LDMS RPM builds on both architectures
+
---
-## Available Containers
+## Build Commands
-| Container | CLI Name | Default Tag | Description |
-|-----------|----------|-------------|-------------|
-| omnia_core | `core` | 2.2 | Core Omnia container (Ansible, Python, SSH) |
-| omnia_auth | `auth` | 1.1 | OpenLDAP authentication service |
-| omnia_build_stream | `build-stream` | 1.1 | FastAPI build automation service |
-| ldms | `ldms` | 1.1 | OVIS LDMS monitoring (multi-stage Ubuntu build) |
-| image_builder | `image-builder` | 1.1 | OpenCHAMI image builder (AlmaLinux 10, Buildah) |
-| kafkapump | `kafkapump` | 1.3 | iDRAC telemetry → Kafka |
-| victoriapump | `victoriapump` | 1.3 | iDRAC telemetry → VictoriaMetrics |
-| telemetry_receiver | `telemetry-receiver` | 1.3 | iDRAC telemetry collector |
+```bash
+# Single container
+./build_images.sh core core_tag=2.2
-### Build Groups
+# Comma-separated list
+./build_images.sh core,auth core_tag=2.2 auth_tag=1.1
+
+# Telemetry group
+./build_images.sh telemetry
-| Group | Containers |
-|-------|-----------|
-| `oim` | core, auth, image-builder (default if no arg) |
-| `all` | core, auth, ldms, kafkapump, victoriapump, telemetry-receiver, image-builder |
-| `pipeline` | core, auth, ldms, kafkapump, victoriapump, telemetry-receiver, image-builder |
-| `telemetry` | kafkapump, victoriapump, telemetry-receiver |
+# Push to registry (requires Docker)
+./build_images.sh core core_tag=2.2 build_tool=docker build_action=push
+```
---
-## Parameters Reference
+## Parameters
-### Common (valid for all containers)
+### Global
| Parameter | Values | Default | Description |
|-----------|--------|---------|-------------|
| `build_tool` | `podman`, `docker` | `podman` | Container build tool |
| `build_action` | `load`, `push` | `load` | Load locally or push to registry |
-### Container-specific tags
+### Per-Container Tags
| Parameter | Default | Container |
|-----------|---------|-----------|
@@ -154,81 +166,43 @@ src/containers/
| `kafkapump_tag` | `1.3` | kafkapump |
| `victoriapump_tag` | `1.3` | victoriapump |
| `telemetry_receiver_tag` | `1.3` | telemetry_receiver |
-| `omnia_branch` | `main` | omnia_core (branch/tag to clone) |
-
-### Push to Registry
-
-```bash
-# Requires Docker (Podman push not supported via this script)
-./build_images.sh core core_tag=2.2 omnia_branch=v2.2.0.0 build_tool=docker build_action=push
-```
-
----
-
-## Parameter Validation
-
-The script validates parameters and shows context-specific errors:
-```bash
-# Invalid parameter
-./build_images.sh core invalid_param=value
-# Error: Invalid parameter(s): invalid_param
-# Valid parameters for 'core': build_tool build_action core_tag omnia_branch
-
-# Wrong container-specific parameter
-./build_images.sh core auth_tag=1.0
-# Error: Parameter 'auth_tag' is not valid for container 'core'
-```
+Parameter validation is built in — the script rejects unknown or mismatched parameters.
---
## Docker vs Podman
-**Podman (default):**
-- No daemon required
-- Rootless by default
-
-**Docker:**
-- Required for `build_action=push`
-- Requires buildx for multi-platform builds
-
-### Docker Setup
+| Feature | Podman (default) | Docker |
+|---------|-----------------|--------|
+| Daemon | Not required | Required |
+| Rootless | Default | Requires config |
+| Push to registry | Not supported via script | Supported (`build_action=push`) |
+| Multi-platform | Via `--arch` flag | Via `buildx` |
```bash
+# Docker setup (if needed)
sudo systemctl start docker
-sudo systemctl enable docker
docker buildx create --name mybuilder --driver docker-container --use
docker buildx inspect --bootstrap
```
---
-## Updating Python Packages
+## Updating Python Dependencies
-For containers using uv (omnia_core, omnia_build_stream):
+For containers using **uv** (omnia_core, omnia_build_stream):
-1. **Install uv**: `pip install uv`
-2. **Update pyproject.toml**: Navigate to the container folder and update
-3. **Update the lock file**: Run `uv lock` from the same directory
+1. Install uv: `pip install uv`
+2. Edit `pyproject.toml` in the container directory
+3. Run `uv lock` to regenerate the lock file
---
## Troubleshooting
-**Issue:** Warning about default branch
-```
-⚠️ Warning: omnia_branch not specified, using default branch: main
-```
-**Solution:** Always specify `omnia_branch` for production builds.
-
-**Issue:** Build fails
-**Solution:** Ensure Podman/Docker is running and you have internet access to pull base images.
-
-**Issue:** Permission errors with Podman
-**Solution:** Run as non-root user or configure subuid/subgid mappings.
-
----
-
-## Support
-
-For issues or questions, refer to the [Omnia documentation](https://omnia.readthedocs.io/en/latest/).
+| Issue | Solution |
+|-------|---------|
+| Build fails | Verify Podman/Docker is running and internet is accessible |
+| Permission errors (Podman) | Run as non-root; configure subuid/subgid if needed |
+| Image-builder wrong arch | Check `uname -m`; use `build_tool=docker` for cross-platform |
\ No newline at end of file
diff --git a/src/containers/_common.sh b/src/containers/_common.sh
old mode 100644
new mode 100755
diff --git a/src/containers/build_images.sh b/src/containers/build_images.sh
old mode 100644
new mode 100755
index d6f579b104..e59d3d512f
--- a/src/containers/build_images.sh
+++ b/src/containers/build_images.sh
@@ -1,4 +1,19 @@
#!/bin/bash
+
+# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
# =============================================================================
# build_images.sh — Container Image Build Wrapper
# =============================================================================
@@ -76,7 +91,7 @@ for arg in "$@"; do
param_name="${arg%%=*}"
- if [[ ! " ${VALID_PARAMS[@]} " =~ " ${param_name} " ]]; then
+ if [[ ! " ${VALID_PARAMS[*]} " =~ " ${param_name} " ]]; then
INVALID_PARAMS+=("$param_name")
fi
@@ -134,10 +149,10 @@ validate_container_params() {
local allowed_params=("${@:2}")
for param in "${CONTAINER_PARAMS[@]}"; do
- if [[ " ${COMMON_PARAMS[@]} " =~ " ${param} " ]]; then
+ if [[ " ${COMMON_PARAMS[*]} " =~ " ${param} " ]]; then
continue
fi
- if [[ ! " ${allowed_params[@]} " =~ " ${param} " ]]; then
+ if [[ ! " ${allowed_params[*]} " =~ " ${param} " ]]; then
echo -e "${RED}Error: Parameter '${param}' is not valid for container '${container}'${NC}"
echo -e "${YELLOW}Valid parameters for '${container}': ${COMMON_PARAMS[*]} ${allowed_params[*]}${NC}"
exit 1
diff --git a/src/containers/image_builder/build.sh b/src/containers/image_builder/build.sh
old mode 100644
new mode 100755
index cff5722aaf..00cc7bafdc
--- a/src/containers/image_builder/build.sh
+++ b/src/containers/image_builder/build.sh
@@ -40,7 +40,28 @@ clone_image_builder_repo() {
}
build_image_builder() {
- local detected_platform="linux/amd64"
+ # Detect host architecture
+ local host_arch
+ host_arch="$(uname -m)"
+
+ local image_name
+ local detected_platform
+
+ case "$host_arch" in
+ x86_64|amd64)
+ image_name="image-build-el10"
+ detected_platform="linux/amd64"
+ ;;
+ aarch64|arm64)
+ image_name="image-build-aarch64"
+ detected_platform="linux/arm64"
+ ;;
+ *)
+ echo -e "${RED}Error: Unsupported architecture '${host_arch}' for image-builder.${NC}"
+ echo -e "${YELLOW}Supported: x86_64, aarch64${NC}"
+ exit 1
+ ;;
+ esac
if [ "$BUILD_TOOL" = "docker" ]; then
# Dynamic platform detection for image-builder (only when using docker)
@@ -49,20 +70,25 @@ build_image_builder() {
echo -e "${YELLOW}Please ensure Docker is installed and running.${NC}"
exit 1
}
+ # Map docker arch to image name
+ case "$detected_platform" in
+ */arm64|*/aarch64) image_name="image-build-aarch64" ;;
+ *) image_name="image-build-el10" ;;
+ esac
fi
- print_build_info "image-build-el10" "${IMAGE_BUILDER_TAG}" \
+ print_build_info "${image_name}" "${IMAGE_BUILDER_TAG}" \
"Using Image Builder Commit: ${YELLOW}${IMAGE_BUILDER_COMMIT}${NC}\nUsing Detected Platform: ${YELLOW}${detected_platform}${NC}"
# Clone repo if needed
clone_image_builder_repo
container_build \
- "image-build-el10" \
+ "${image_name}" \
"${IMAGE_BUILDER_TAG}" \
"${IMAGE_BUILDER_CLONE_DIR}" \
"dockerfiles/dnf/Containerfile.el10" \
"" \
"" \
"${detected_platform}"
-}
+}
\ No newline at end of file
diff --git a/src/containers/kafkapump/build.sh b/src/containers/kafkapump/build.sh
old mode 100644
new mode 100755
diff --git a/src/containers/ldms/build.sh b/src/containers/ldms/build.sh
old mode 100644
new mode 100755
diff --git a/src/containers/ldms/configure.aggregator.sh b/src/containers/ldms/configure.aggregator.sh
old mode 100644
new mode 100755
diff --git a/src/containers/omnia_auth/build.sh b/src/containers/omnia_auth/build.sh
old mode 100644
new mode 100755
diff --git a/src/containers/omnia_build_stream/build.sh b/src/containers/omnia_build_stream/build.sh
old mode 100644
new mode 100755
diff --git a/src/containers/omnia_build_stream/init_s3cfg.sh b/src/containers/omnia_build_stream/init_s3cfg.sh
old mode 100644
new mode 100755
diff --git a/src/containers/omnia_core/build.sh b/src/containers/omnia_core/build.sh
old mode 100644
new mode 100755
diff --git a/src/containers/omnia_core/cert-copy.sh b/src/containers/omnia_core/cert-copy.sh
old mode 100644
new mode 100755
diff --git a/src/containers/omnia_core/entrypoint.sh b/src/containers/omnia_core/entrypoint.sh
old mode 100644
new mode 100755
diff --git a/src/containers/telemetry_receiver/build.sh b/src/containers/telemetry_receiver/build.sh
old mode 100644
new mode 100755
diff --git a/src/containers/victoriapump/build.sh b/src/containers/victoriapump/build.sh
old mode 100644
new mode 100755
diff --git a/src/main/omnia.sh b/src/main/omnia.sh
old mode 100644
new mode 100755
index 0a28cab48c..87f94fb324
--- a/src/main/omnia.sh
+++ b/src/main/omnia.sh
@@ -1,6 +1,6 @@
#!/bin/bash
-# Copyright 2025 Dell Inc. or its subsidiaries. All Rights Reserved.
+# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/src/playbooks/build_image_aarch64/roles/fetch_packages/vars/main.yml b/src/playbooks/build_image_aarch64/roles/fetch_packages/vars/main.yml
index 04c7ad6552..4e69cf58f1 100644
--- a/src/playbooks/build_image_aarch64/roles/fetch_packages/vars/main.yml
+++ b/src/playbooks/build_image_aarch64/roles/fetch_packages/vars/main.yml
@@ -16,7 +16,7 @@
metadata_file_path: "/opt/omnia/offline_repo/.data/localrepo_metadata.yml"
local_repo_check_msg: |
- Failure: metadata file is not present at path {{ metadata_file_path }}.
+ Failure: metadata file is not present at path {{ metadata_file_path }} inside omnia_core container.
Please make sure that local_repo.yml playbook is executed successfully.
input_project_dir: "{{ hostvars['localhost']['input_project_dir'] }}"
functional_groups_file_path: "{{ hostvars['localhost']['functional_groups_config_path'] | default('/opt/omnia/.data/functional_groups_config.yml') }}"
@@ -25,7 +25,7 @@ aarch64_build_image_completion_msg: |
The playbook build_image_aarch64.yml has been completed successfully.
To boot x86_64 and aarch64 nodes execute discovery/discovery.yml playbook.
functional_group_absent_msg: |
- Failure: No aarch64 functional groups found in functional_group_config.yml input file.
+ Failure: No aarch64 functional groups found in functional_group_config.yml input file inside omnia_core container.
Please make sure aarch64 functional_group should be present in input file functional_group_config.yml
to execute build_image_aarch64.yml successfully.
build_stream_prerequisite_fail_msg: |
diff --git a/src/playbooks/build_image_aarch64/roles/image_creation/tasks/build_compute_image.yml b/src/playbooks/build_image_aarch64/roles/image_creation/tasks/build_compute_image.yml
index 38eff4480e..c66f940e0f 100644
--- a/src/playbooks/build_image_aarch64/roles/image_creation/tasks/build_compute_image.yml
+++ b/src/playbooks/build_image_aarch64/roles/image_creation/tasks/build_compute_image.yml
@@ -127,7 +127,7 @@
ansible.builtin.set_fact:
failure_msg_list:
- "aarch64 compute image build job did not complete successfully."
- - "Check logs at {{ oim_shared_path }}/omnia/log/openchami for respective functional group for more details."
+ - "Check logs at {{ oim_shared_path }}/omnia/log/openchami on OIM host for respective functional group for more details."
- ""
- "Failed images:"
@@ -138,7 +138,7 @@
- name: Add log paths section to message
ansible.builtin.set_fact:
- failure_msg_list: "{{ failure_msg_list + ['', 'Check logs at ' + openchami_log_dir + ' for details:'] }}"
+ failure_msg_list: "{{ failure_msg_list + ['', 'Check logs at ' + openchami_log_dir + ' on OIM host for details:'] }}"
- name: Add log file paths to message
ansible.builtin.set_fact:
diff --git a/src/playbooks/build_image_aarch64/roles/image_creation/vars/main.yml b/src/playbooks/build_image_aarch64/roles/image_creation/vars/main.yml
index 97773fd26c..fd19c4eea6 100644
--- a/src/playbooks/build_image_aarch64/roles/image_creation/vars/main.yml
+++ b/src/playbooks/build_image_aarch64/roles/image_creation/vars/main.yml
@@ -45,10 +45,10 @@ openchami_aarch64_base_image_log_path: "{{ oim_shared_path }}/omnia/log/opencham
openchami_base_image_config_template: "{{ role_path }}/templates/images/rhel-base-config.yaml.j2"
base_image_failure_msg: |
Base aarch64 image build job failed or timed out.
- Check logs at path {{ openchami_aarch64_base_image_log_path }} for details.
+ Check logs at path {{ openchami_aarch64_base_image_log_path }} on OIM host for details.
compute_image_failure_msg: |
aarch64 compute image build job did not complete successfully.
- Check logs at {{ openchami_log_dir }} for respective functional group for more details.
+ Check logs at {{ openchami_log_dir }} on OIM host for respective functional group for more details.
# build_compute_image.yml - image-build config template
openchami_compute_image_config_template: "{{ role_path }}/templates/images/rhel-compute-config.yaml.j2"
diff --git a/src/playbooks/build_image_x86_64/roles/fetch_packages/vars/main.yml b/src/playbooks/build_image_x86_64/roles/fetch_packages/vars/main.yml
index 59e67fa991..396dad7f6b 100644
--- a/src/playbooks/build_image_x86_64/roles/fetch_packages/vars/main.yml
+++ b/src/playbooks/build_image_x86_64/roles/fetch_packages/vars/main.yml
@@ -16,7 +16,7 @@
metadata_file_path: "/opt/omnia/offline_repo/.data/localrepo_metadata.yml"
local_repo_check_msg: |
- Failure: metadata file path {{ metadata_file_path }} is not present.
+ Failure: metadata file path {{ metadata_file_path }} is not present inside omnia_core container.
Please make sure that local_repo.yml playbook is executed successfully.
input_project_dir: "{{ hostvars['localhost']['input_project_dir'] }}"
functional_groups_file_path: "{{ hostvars['localhost']['functional_groups_config_path'] | default('/opt/omnia/.data/functional_groups_config.yml') }}"
@@ -27,7 +27,7 @@ x86_64_build_image_completion_msg: |
To boot x86_64 nodes execute provision/provision.yml playbook.
functional_group_absent_msg: |
- Failure: No x86_64 functional groups found in functional_group_config.yml input file.
+ Failure: No x86_64 functional groups found in functional_group_config.yml input file inside omnia_core container.
Please make sure x86_64 functional_group should be present in input file functional_group_config.yml
to execute build_image_x86_64.yml successfully.
build_stream_prerequisite_fail_msg: |
diff --git a/src/playbooks/build_image_x86_64/roles/image_creation/tasks/build_compute_image.yml b/src/playbooks/build_image_x86_64/roles/image_creation/tasks/build_compute_image.yml
index eb00755486..6ad991e3a1 100644
--- a/src/playbooks/build_image_x86_64/roles/image_creation/tasks/build_compute_image.yml
+++ b/src/playbooks/build_image_x86_64/roles/image_creation/tasks/build_compute_image.yml
@@ -117,7 +117,7 @@
ansible.builtin.set_fact:
failure_msg_list:
- "x86_64 compute image build job did not complete successfully."
- - "Check logs at {{ openchami_log_dir }} for respective functional group for more details."
+ - "Check logs at {{ openchami_log_dir }} on OIM host for respective functional group for more details."
- ""
- "Failed images:"
@@ -128,7 +128,7 @@
- name: Add log paths section to message
ansible.builtin.set_fact:
- failure_msg_list: "{{ failure_msg_list + ['', 'Check logs at ' + openchami_log_dir + ' for details:'] }}"
+ failure_msg_list: "{{ failure_msg_list + ['', 'Check logs at ' + openchami_log_dir + ' on OIM host for details:'] }}"
- name: Add log file paths to message
ansible.builtin.set_fact:
diff --git a/src/playbooks/build_image_x86_64/roles/image_creation/vars/main.yml b/src/playbooks/build_image_x86_64/roles/image_creation/vars/main.yml
index ba39b00f79..ecf413ca72 100644
--- a/src/playbooks/build_image_x86_64/roles/image_creation/vars/main.yml
+++ b/src/playbooks/build_image_x86_64/roles/image_creation/vars/main.yml
@@ -49,10 +49,10 @@ openchami_x86_64_base_image_log_path: "{{ oim_shared_path }}/omnia/log/openchami
openchami_base_image_config_template: "{{ role_path }}/templates/images/rhel-base-config.yaml.j2"
base_image_failure_msg: |
Base x86_64 image build job failed or timed out.
- Check logs at path {{ openchami_x86_64_base_image_log_path }} for details.
+ Check logs at path {{ openchami_x86_64_base_image_log_path }} on OIM host for details.
compute_image_failure_msg: |
x86_64 compute image build job did not complete successfully.
- Check logs at {{ openchami_log_dir }} for respective functional group for more details.
+ Check logs at {{ openchami_log_dir }} on OIM host for respective functional group for more details.
# build_compute_image.yml - image-build config template
openchami_compute_image_config_template: "{{ role_path }}/templates/images/rhel-compute-config.yaml.j2"
diff --git a/src/playbooks/discovery/roles/ome_discovery/tasks/generate_discovery_report.yml b/src/playbooks/discovery/roles/ome_discovery/tasks/generate_discovery_report.yml
index 686f5fee2c..71c34adf93 100644
--- a/src/playbooks/discovery/roles/ome_discovery/tasks/generate_discovery_report.yml
+++ b/src/playbooks/discovery/roles/ome_discovery/tasks/generate_discovery_report.yml
@@ -56,7 +56,10 @@
- ""
- "3. Update HOSTNAME, FUNCTIONAL_GROUP_NAME, GROUP_NAME as needed."
- ""
- - "4. Run:"
+ - "4. If fresh installation of Omnia, Run:"
+ - " ansible-playbook prepare_oim/prepare_oim.yml"
+ - ""
+ - " If Slurm add node scenario, Run:"
- " ansible-playbook provision/provision.yml"
- "============================================================"
diff --git a/src/playbooks/local_repo/roles/parse_and_download/tasks/execute_parallel_tasks.yml b/src/playbooks/local_repo/roles/parse_and_download/tasks/execute_parallel_tasks.yml
index 7ca11fba46..036748470c 100644
--- a/src/playbooks/local_repo/roles/parse_and_download/tasks/execute_parallel_tasks.yml
+++ b/src/playbooks/local_repo/roles/parse_and_download/tasks/execute_parallel_tasks.yml
@@ -58,22 +58,22 @@
- name: Confirm all tasks Success
ansible.builtin.debug:
- msg: "All tasks completed successfully. Log path: {{ base_path }}/{{ cluster_os_type }}/{{ cluster_os_version }}/{{ item.arch }}/{{ item.key }}/logs/"
+ msg: "All tasks completed successfully. Log path: {{ base_path }}/{{ cluster_os_type }}/{{ cluster_os_version }}/{{ item.arch }}/{{ item.key }}/logs/ on OIM host" # noqa: yaml[line-length]
when: task_results.overall_status == "SUCCESS"
- name: Fail if Partial Success
ansible.builtin.debug:
- msg: "Some tasks partially failed. Please review the task details above for more information. Log path: {{ base_path }}/{{ cluster_os_type }}/{{ cluster_os_version }}/{{ item.arch }}/{{ item.key }}/logs/" # noqa: yaml[line-length]
+ msg: "Some tasks partially failed. Please review the task details above for more information. Log path: {{ base_path }}/{{ cluster_os_type }}/{{ cluster_os_version }}/{{ item.arch }}/{{ item.key }}/logs/ on OIM host" # noqa: yaml[line-length]
when: task_results.overall_status == "PARTIAL"
- name: Fail if Failure to download package
ansible.builtin.debug:
- msg: "Some tasks failed. Please review the task details above for more information. Log path: {{ base_path }}/{{ cluster_os_type }}/{{ cluster_os_version }}/{{ item.arch }}/{{ item.key }}/logs/" # noqa: yaml[line-length]
+ msg: "Some tasks failed. Please review the task details above for more information. Log path: {{ base_path }}/{{ cluster_os_type }}/{{ cluster_os_version }}/{{ item.arch }}/{{ item.key }}/logs/ on OIM host" # noqa: yaml[line-length]
when: task_results.overall_status == "FAILURE"
- name: Fail if Timeout during download
ansible.builtin.debug:
- msg: "Some tasks failed due to timeout. Please review the task details above for more information. Log path: {{ base_path }}/{{ cluster_os_type }}/{{ cluster_os_version }}/{{ item.arch }}/{{ item.key }}/logs" # noqa: yaml[line-length]
+ msg: "Some tasks failed due to timeout. Please review the task details above for more information. Log path: {{ base_path }}/{{ cluster_os_type }}/{{ cluster_os_version }}/{{ item.arch }}/{{ item.key }}/logs on OIM host" # noqa: yaml[line-length]
when: task_results.overall_status == "TIMEOUT"
rescue:
@@ -81,8 +81,8 @@
ansible.builtin.debug:
msg:
- "Parallel tasks encountered an error. Check the logs for details:"
- - "Log directory: {{ base_path }}/{{ cluster_os_type }}/{{ cluster_os_version }}/{{ item.arch }}/{{ item.key }}/logs"
- - "Log file: {{ base_path }}/{{ cluster_os_type }}/{{ cluster_os_version }}/{{ item.arch }}/{{ item.key }}_task_results.log"
+ - "Log directory: {{ base_path }}/{{ cluster_os_type }}/{{ cluster_os_version }}/{{ item.arch }}/{{ item.key }}/logs on OIM host"
+ - "Log file: {{ base_path }}/{{ cluster_os_type }}/{{ cluster_os_version }}/{{ item.arch }}/{{ item.key }}_task_results.log on OIM host"
- "Error: {{ ansible_failed_result.msg | default('Unknown error') }}"
- name: Fail the playbook execution
diff --git a/src/playbooks/prepare_oim/roles/deploy_containers/openchami/tasks/configs/firewall.yml b/src/playbooks/prepare_oim/roles/deploy_containers/openchami/tasks/configs/firewall.yml
index bafa6ee74f..e399b21bc7 100644
--- a/src/playbooks/prepare_oim/roles/deploy_containers/openchami/tasks/configs/firewall.yml
+++ b/src/playbooks/prepare_oim/roles/deploy_containers/openchami/tasks/configs/firewall.yml
@@ -33,7 +33,7 @@
state: enabled
loop: "{{ udp_ports }}"
-- name: Open DNS port for CoreDNS (when dns_enabled)
+- name: Open DNS port for CoreDNS
ansible.posix.firewalld:
port: "{{ item }}"
permanent: true
@@ -41,7 +41,6 @@
loop:
- 53/tcp
- 53/udp
- when: dns_enabled | default(false) | bool
- name: Add Podman interfaces to trusted zone
ansible.posix.firewalld:
diff --git a/src/playbooks/provision/roles/configure_ochami/tasks/configure_bss_cloud_init.yml b/src/playbooks/provision/roles/configure_ochami/tasks/configure_bss_cloud_init.yml
index 0b5a0da05e..508de12293 100644
--- a/src/playbooks/provision/roles/configure_ochami/tasks/configure_bss_cloud_init.yml
+++ b/src/playbooks/provision/roles/configure_ochami/tasks/configure_bss_cloud_init.yml
@@ -16,6 +16,15 @@
- name: Include openchami vars
ansible.builtin.include_vars: "{{ openchami_config_vars_path }}"
+- name: Refresh dns_enabled from provision_config.yml
+ ansible.builtin.include_vars:
+ file: "{{ hostvars['localhost']['input_project_dir'] }}/provision_config.yml"
+ name: _provision_config_refresh
+
+- name: Override dns_enabled with current provision_config value
+ ansible.builtin.set_fact:
+ dns_enabled: "{{ _provision_config_refresh.dns_enabled | default(false) | bool }}"
+
- name: Include nodes vars
ansible.builtin.slurp:
src: "{{ openchami_nodes_vars_path }}"
diff --git a/src/playbooks/provision/roles/configure_ochami/templates/nodes/nodes.yaml.j2 b/src/playbooks/provision/roles/configure_ochami/templates/nodes/nodes.yaml.j2
index ff8f3e2844..d4087273b4 100644
--- a/src/playbooks/provision/roles/configure_ochami/templates/nodes/nodes.yaml.j2
+++ b/src/playbooks/provision/roles/configure_ochami/templates/nodes/nodes.yaml.j2
@@ -3,7 +3,7 @@ nodes:
- name: {{ item.value.HOSTNAME }}
xname: {{ item.value.XNAME }}
description: {{ item.value.SERVICE_TAG }}
- nid: {{ loop.index }}
+ nid: {{ item.value.HOSTNAME | regex_replace('^nid0*', '') | int if item.value.HOSTNAME is regex('^nid\\d+$') else loop.index }}
group: {{ item.value.FUNCTIONAL_GROUP_NAME }}
bmc_mac: {{ item.value.BMC_MAC }}
bmc_ip: {{ item.value.BMC_IP }}
diff --git a/src/playbooks/provision/roles/telemetry/files/nersc-ldms-aggr/scripts/decomp.json b/src/playbooks/provision/roles/telemetry/files/nersc-ldms-aggr/scripts/decomp.json
index 5caf13d667..9ec983b51f 100644
--- a/src/playbooks/provision/roles/telemetry/files/nersc-ldms-aggr/scripts/decomp.json
+++ b/src/playbooks/provision/roles/telemetry/files/nersc-ldms-aggr/scripts/decomp.json
@@ -736,6 +736,7 @@
"85CE1C60D0570924DAE5B17758912D1A3ADA2091ABD946E06B9A0240F53F4FD8" : "vmstat_decomp",
"9292CFE0558DBE06EF95BE5B97A9FA13A3F66CF1523D3E175816F3F0D9C66DD4" : "vmstat_decomp",
"42EB25BA6239F4883E05847676F9BE49B10BD059A714A1C95A932048A19D8D74" : "vmstat_decomp",
+ "C7137D7DBC06557F5256336634062DDD868DCDFFFAD5817C0C7E74969C604D13" : "vmstat_decomp",
"F76BA26012C2F1F481AB0C1E0672D438ECFE0C4F7B2B4942AA7067A1FCE51A75" : "mt_slurm_decomp"
}
}
diff --git a/src/playbooks/provision/roles/telemetry/tasks/generate_telemetry_deployments.yml b/src/playbooks/provision/roles/telemetry/tasks/generate_telemetry_deployments.yml
index 790e8231c6..93353ec50c 100644
--- a/src/playbooks/provision/roles/telemetry/tasks/generate_telemetry_deployments.yml
+++ b/src/playbooks/provision/roles/telemetry/tasks/generate_telemetry_deployments.yml
@@ -171,6 +171,14 @@
when: item.skip_when is not defined or not item.skip_when | bool
tags: telemetry_deployment
+- name: Populate ActiveMQ ConfigMap
+ ansible.builtin.template:
+ src: 'telemetry/idrac_telemetry/activemq-config.yaml.j2'
+ dest: "{{ hostvars['localhost']['k8s_client_share_path'] }}/telemetry/deployments/activemq-config.yaml"
+ mode: "{{ hostvars['localhost']['file_permissions_644'] }}"
+ when: telemetry_config.telemetry_sources.idrac.metrics_enabled | default(false) | bool
+ tags: telemetry_deployment
+
- name: Populate iDRAC telemetry statefulset
ansible.builtin.template:
src: 'telemetry/idrac_telemetry/idrac_telemetry_statefulset.yaml.j2'
diff --git a/src/playbooks/provision/roles/telemetry/templates/telemetry/idrac_telemetry/activemq-config.yaml.j2 b/src/playbooks/provision/roles/telemetry/templates/telemetry/idrac_telemetry/activemq-config.yaml.j2
new file mode 100644
index 0000000000..5538a366e6
--- /dev/null
+++ b/src/playbooks/provision/roles/telemetry/templates/telemetry/idrac_telemetry/activemq-config.yaml.j2
@@ -0,0 +1,101 @@
+# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: activemq-config
+ namespace: {{ telemetry_namespace }}
+data:
+ activemq.xml: |
+
+
+
+
+
+ file:${activemq.conf}/credentials.properties
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/src/playbooks/provision/roles/telemetry/templates/telemetry/idrac_telemetry/idrac_telemetry_statefulset.yaml.j2 b/src/playbooks/provision/roles/telemetry/templates/telemetry/idrac_telemetry/idrac_telemetry_statefulset.yaml.j2
index 28d44cb0ea..e054cad6d1 100644
--- a/src/playbooks/provision/roles/telemetry/templates/telemetry/idrac_telemetry/idrac_telemetry_statefulset.yaml.j2
+++ b/src/playbooks/provision/roles/telemetry/templates/telemetry/idrac_telemetry/idrac_telemetry_statefulset.yaml.j2
@@ -48,6 +48,9 @@ spec:
spec:
volumes:
{% set types = telemetry_config.telemetry_sources.idrac.collection_targets | default([]) %}
+ - name: activemq-config
+ configMap:
+ name: activemq-config
{% if 'kafka' in types %}
# Mount Kafka cluster CA certificate for TLS verification
- name: kafka-cluster-ca-cert
@@ -139,6 +142,11 @@ spec:
- name: activemq
image: {{ activemq_image }}
imagePullPolicy: IfNotPresent
+ volumeMounts:
+ - name: activemq-config
+ mountPath: /opt/apache-activemq/conf/activemq.xml
+ subPath: activemq.xml
+ readOnly: true
resources:
requests:
cpu: {{ idrac_telemetry_resources.activemq.requests.cpu }}
diff --git a/src/playbooks/provision/roles/telemetry/templates/telemetry/kustomization.yaml.j2 b/src/playbooks/provision/roles/telemetry/templates/telemetry/kustomization.yaml.j2
index 4ff4d6e1db..8ef440a330 100644
--- a/src/playbooks/provision/roles/telemetry/templates/telemetry/kustomization.yaml.j2
+++ b/src/playbooks/provision/roles/telemetry/templates/telemetry/kustomization.yaml.j2
@@ -54,6 +54,7 @@ resources:
{% endif %}
{% if telemetry_config.telemetry_sources.idrac.metrics_enabled | default(false) %}
# iDRAC Telemetry Resources
+ - activemq-config.yaml
- idrac_telemetry_statefulset.yaml
- telemetry_cleaner_rbac.yaml
- telemetry_pod_cleanup.yaml
diff --git a/src/playbooks/rollback/playbooks/rollback_oim.yml b/src/playbooks/rollback/playbooks/rollback_oim.yml
index 05eb34d68f..febab0ac47 100644
--- a/src/playbooks/rollback/playbooks/rollback_oim.yml
+++ b/src/playbooks/rollback/playbooks/rollback_oim.yml
@@ -13,7 +13,7 @@
# limitations under the License.
---
-- name: Rollback OIM (includes OpenCHAMI)
+- name: Rollback OIM (includes Pulp and OpenCHAMI)
hosts: localhost
connection: local
gather_facts: true
@@ -59,7 +59,53 @@
ansible.builtin.set_fact:
rollback_backup_dir: "{{ rollback_manifest.backup_dir | default('/opt/omnia/backups/upgrade/version_2.1.0.0') }}"
- # ── Phase 1: OpenCHAMI Rollback ─────────────────────────────────────
+ # ── Phase 1: Pulp Rollback ────────────────────────────────────────────
+ # The rollback_pulp role handles the full lifecycle:
+ # resolve_admin_ip → pre_rollback_checks → rollback_pulp_container
+ # → post_rollback_health_check
+ # On failure, the role sets pulp_rollback_failed and raises fail.
+ # On skip (not deployed / already at v2.1), the role completes normally.
+ - name: "Phase 1 — Pulp Rollback"
+ block:
+ - name: Rollback Pulp container
+ ansible.builtin.include_role:
+ name: "{{ playbook_dir }}/../roles/rollback_pulp"
+
+ - name: Display Pulp rollback outcome
+ ansible.builtin.debug:
+ msg: >-
+ Pulp Phase 1 result —
+ deployed={{ pulp_deployed | default(false) }},
+ failed={{ pulp_rollback_failed | default(false) }}
+
+ rescue:
+ - name: Re-read rollback_manifest.yml after Pulp failure
+ ansible.builtin.slurp:
+ src: "{{ rollback_manifest_path }}"
+ register: pulp_rescue_raw_rollback_manifest
+
+ - name: Parse current manifest state
+ ansible.builtin.set_fact:
+ pulp_rescue_rollback_manifest: "{{ pulp_rescue_raw_rollback_manifest.content | b64decode | from_yaml }}"
+
+ - name: Pulp rollback failed — mark component as failed
+ ansible.builtin.copy:
+ content: >-
+ {{ pulp_rescue_rollback_manifest | combine({
+ 'component_status': pulp_rescue_rollback_manifest.component_status | combine({
+ component_name: 'failed'
+ })
+ }) | to_nice_yaml }}
+ dest: "{{ rollback_manifest_path }}"
+ mode: '0644'
+
+ - name: Fail with Pulp rollback error
+ ansible.builtin.fail:
+ msg: >-
+ Pulp rollback failed. Check container logs: podman logs pulp
+ Backup directory: {{ rollback_backup_dir | default('N/A') }}
+
+ # ── Phase 2: OpenCHAMI Rollback ─────────────────────────────────────
# The rollback_openchami role handles the full lifecycle:
# resolve_backup_dir → pre_rollback_checks → stop_current_containers
# → restore_quadlets_and_configs → start_postgres_only
@@ -67,7 +113,7 @@
# → reload_cloud_init_data → post_rollback_health_check
# On failure, the role sets openchami_rollback_failed and raises fail.
# On skip (already at v2.1), the role completes normally.
- - name: "Phase 1 — OpenCHAMI Rollback"
+ - name: "Phase 2 — OpenCHAMI Rollback"
block:
- name: Rollback OpenCHAMI containers and services
ansible.builtin.include_role:
@@ -76,7 +122,7 @@
- name: Display OpenCHAMI rollback outcome
ansible.builtin.debug:
msg: >-
- OpenCHAMI Phase 1 result —
+ OpenCHAMI Phase 2 result —
rollback_needed={{ rollback_needed | default(true) }},
failed={{ openchami_rollback_failed | default(false) }}
diff --git a/src/playbooks/rollback/roles/rollback_k8s/defaults/main.yml b/src/playbooks/rollback/roles/rollback_k8s/defaults/main.yml
index b66bbe6fba..d41d3d1fc8 100644
--- a/src/playbooks/rollback/roles/rollback_k8s/defaults/main.yml
+++ b/src/playbooks/rollback/roles/rollback_k8s/defaults/main.yml
@@ -20,8 +20,10 @@ oim_data_path: "/opt/omnia/.data"
oim_provision_path: "/opt/omnia/provision"
tmp_path: "/tmp"
cluster_os_version: "10.0"
-admin_nic_ip: "{{ hostvars['localhost']['admin_nic_ip'] | default('127.0.0.1') }}"
-admin_nic_cidr: "{{ hostvars['localhost']['admin_nic_cidr'] | default('10.0.0.0/24') }}"
+# admin_nic_ip and admin_nic_cidr are calculated from network_spec.yml in the playbook
+# Do not use default values here to avoid misconfiguration
+admin_nic_ip: "{{ hostvars['localhost']['admin_nic_ip'] }}"
+admin_nic_cidr: "{{ hostvars['localhost']['admin_nic_cidr'] }}"
# Cross-reference path for the upgrade directory.
# Uses role_path (always .../rollback/roles/rollback_k8s) so this
diff --git a/src/playbooks/rollback/roles/rollback_k8s/tasks/fix_vip_split_brain.yml b/src/playbooks/rollback/roles/rollback_k8s/tasks/fix_vip_split_brain.yml
index 3ba836ffb4..52dafd0031 100644
--- a/src/playbooks/rollback/roles/rollback_k8s/tasks/fix_vip_split_brain.yml
+++ b/src/playbooks/rollback/roles/rollback_k8s/tasks/fix_vip_split_brain.yml
@@ -18,6 +18,14 @@
# the split-brain and cleans up stale VIP addresses so only the
# rightful lease holder owns the VIP.
+# ── Set default _stale_vip_nodes if fix_vip already completed ─────
+# This ensures _stale_vip_nodes variable is always defined, even when
+# fix_vip stage is skipped because it's already completed.
+- name: Set default _stale_vip_nodes when fix_vip already completed
+ ansible.builtin.set_fact:
+ _stale_vip_nodes: []
+ when: (rollback_status.stages.fix_vip.status | default('pending')) == 'completed'
+
- name: Skip fix_vip if already completed
ansible.builtin.debug:
msg: "fix_vip already completed — skipping."
diff --git a/src/playbooks/rollback/roles/rollback_k8s/tasks/restore_k8s_configs_single_cp.yml b/src/playbooks/rollback/roles/rollback_k8s/tasks/restore_k8s_configs_single_cp.yml
index 95a56b4c3d..568d2aa66d 100644
--- a/src/playbooks/rollback/roles/rollback_k8s/tasks/restore_k8s_configs_single_cp.yml
+++ b/src/playbooks/rollback/roles/rollback_k8s/tasks/restore_k8s_configs_single_cp.yml
@@ -114,6 +114,15 @@
Backup archive: {{ k8s_config_backup_dir }}/{{ _cp_node }}/k8s-config.tar.gz
when: not (_admin_conf.stat.exists and _ca_crt.stat.exists)
+ # ── Fix kubelet.conf to point to VIP instead of node IP ────────
+ - name: Ensure kubelet.conf points to kube-vip on {{ _cp_node }}
+ ansible.builtin.replace:
+ path: /etc/kubernetes/kubelet.conf
+ regexp: 'server: https://{{ node_ips[_cp_node] }}:6443'
+ replace: 'server: https://{{ kube_vip }}:6443'
+ delegate_to: "{{ _cp_node }}"
+ when: node_ips[_cp_node] is defined
+
# ── Update per-node status ─────────────────────────────────────
- name: Mark restore_k8s_configs completed for {{ _cp_node }}
ansible.builtin.include_tasks:
diff --git a/src/playbooks/rollback/roles/rollback_k8s/tasks/start_control_plane.yml b/src/playbooks/rollback/roles/rollback_k8s/tasks/start_control_plane.yml
index c59303e6a5..e1d99f3496 100644
--- a/src/playbooks/rollback/roles/rollback_k8s/tasks/start_control_plane.yml
+++ b/src/playbooks/rollback/roles/rollback_k8s/tasks/start_control_plane.yml
@@ -68,11 +68,32 @@
loop_control:
label: "{{ item }}"
+ # ── Wait for static pods to be recreated ───────────────────────
+ # After kubelet restart, static pods need time to be recreated.
+ # This is especially important when /etc/kubernetes is on NFS.
+ - name: Wait for static pods to initialize
+ ansible.builtin.pause:
+ seconds: 15
+
# ── Wait for API server to be fully responsive ─────────────────
# Use --server to bypass kube_vip (VIP may not be up yet)
+ # Check /healthz?exclude=etcd first to ensure API server is up,
+ # then verify full health including etcd connectivity.
- name: Wait for API server on first CP
- ansible.builtin.command:
- cmd: kubectl --server=https://{{ first_cp_ip }}:6443 get --raw /healthz
+ ansible.builtin.shell:
+ cmd: |
+ set -o pipefail
+ HEALTH=$(curl -sk https://{{ first_cp_ip }}:6443/healthz 2>/dev/null)
+ if [ "$HEALTH" = "ok" ]; then
+ echo "ok"
+ exit 0
+ else
+ echo "$HEALTH" | grep -q '\-\]etcd failed' && exit 1
+ echo "$HEALTH"
+ exit 1
+ fi
+ args:
+ executable: /bin/bash
delegate_to: "{{ groups_cp_first | first }}"
register: _api_health
changed_when: false
@@ -80,10 +101,10 @@
delay: "{{ apiserver_wait_delay }}"
until:
- _api_health.rc == 0
- - "'ok' in _api_health.stdout"
+ - _api_health.stdout == 'ok'
# ── Wait for each CP node to become Ready ──────────────────────
- - name: Wait for node {{ item }} to be Ready at {{ k8s_rollback_version }} version # noqa:name[template]
+ - name: Wait for control plane nodes to be Ready at rollback version
ansible.builtin.command: >-
kubectl --server=https://{{ first_cp_ip }}:6443
get node {{ node_ips[item] }}
diff --git a/src/playbooks/rollback/roles/rollback_k8s/tasks/stop_cluster.yml b/src/playbooks/rollback/roles/rollback_k8s/tasks/stop_cluster.yml
index 2e3a75445b..7042848f51 100644
--- a/src/playbooks/rollback/roles/rollback_k8s/tasks/stop_cluster.yml
+++ b/src/playbooks/rollback/roles/rollback_k8s/tasks/stop_cluster.yml
@@ -18,6 +18,14 @@
# 2. Non-leader CPs
# 3. VIP leader last (keeps API accessible as long as possible)
+# ── Set default cp_shutdown_order if stop_cluster already completed ─────
+# This ensures cp_shutdown_order variable is always defined, even when
+# stop_cluster stage is skipped because it's already completed.
+- name: Set default cp_shutdown_order when stop_cluster already completed
+ ansible.builtin.set_fact:
+ cp_shutdown_order: "{{ all_cp_nodes }}"
+ when: (rollback_status.stages.stop_cluster.status | default('pending')) == 'completed'
+
- name: Skip stop_cluster if already completed
ansible.builtin.debug:
msg: "stop_cluster already completed — skipping."
diff --git a/src/playbooks/rollback/roles/rollback_openchami/tasks/normalize_permissions.yml b/src/playbooks/rollback/roles/rollback_openchami/tasks/normalize_permissions.yml
index 937601d340..bcf1b42f02 100644
--- a/src/playbooks/rollback/roles/rollback_openchami/tasks/normalize_permissions.yml
+++ b/src/playbooks/rollback/roles/rollback_openchami/tasks/normalize_permissions.yml
@@ -24,12 +24,21 @@
# "permission denied" when the rollback later reads the backup or restores
# /etc/openchami — which makes the final OpenCHAMI rollback step fail.
#
-# This task normalizes permissions BEFORE the restore runs:
+# Additionally, SELinux contexts may prevent container access to files.
+# Files need the container_file_t context to be accessible from containers.
+#
+# This task normalizes permissions and SELinux contexts BEFORE the restore:
# - Directories -> 0755
# - Files -> 0644
+# - SELinux -> system_u:object_r:container_file_t:s0
# for two roots:
-# 1. Backup root on the core container : {{ rollback_backup_dir }}/openchami
-# 2. /etc/openchami on the OIM host : {{ openchami_etc_dir }}
+# 1. Backup root on OIM host: {{ rollback_oim_host_backup_dir }}/openchami
+# 2. /etc/openchami on OIM host: {{ openchami_etc_dir }}
+#
+# NOTE: All operations run on the OIM host (delegated) because:
+# - The backup is on shared NFS storage mounted on OIM
+# - NFS root_squash causes permission issues when accessed from core container
+# - SELinux contexts must be set on the OIM host filesystem
#
# Behaviour:
# - If permissions are simply wrong, they are corrected automatically and
@@ -42,19 +51,24 @@
- name: Normalize backup and /etc/openchami permissions
block:
- # ── Resolve the two roots to normalize ──────────────────────────────
- - name: Set permission normalization targets
+ # ── Resolve the two roots to normalize (OIM host paths) ───────────────
+ - name: Set permission normalization targets (OIM host paths)
ansible.builtin.set_fact:
- normalize_backup_root: "{{ rollback_backup_dir }}/openchami"
+ normalize_backup_root: "{{ rollback_oim_host_backup_dir }}/openchami"
normalize_etc_root: "{{ openchami_etc_dir }}"
- # ── 1. Backup root (core container — local, on shared NFS path) ──────
- - name: Check backup root exists (core container)
+ # ── 1. Backup root (OIM host — on shared NFS path) ────────────────────
+ # NOTE: Must run on OIM host because NFS root_squash prevents core
+ # container from accessing/modifying files with restrictive permissions.
+ - name: Check backup root exists (OIM host)
ansible.builtin.stat:
path: "{{ normalize_backup_root }}"
register: normalize_backup_root_stat
+ delegate_to: oim
+ delegate_facts: true
+ connection: ssh
- - name: Normalize permissions on backup root (core container)
+ - name: Normalize permissions on backup root (OIM host)
ansible.builtin.shell: |
set -o pipefail
find "{{ normalize_backup_root }}" -type d -exec chmod {{ dir_permissions_755 }} {} + 2>/dev/null || true
@@ -63,8 +77,22 @@
changed_when: true
failed_when: false
when: normalize_backup_root_stat.stat.exists | default(false)
+ delegate_to: oim
+ delegate_facts: true
+ connection: ssh
- - name: Detect unreadable items under backup root (core container)
+ - name: Fix SELinux context on backup root (OIM host)
+ ansible.builtin.command: >
+ chcon -R system_u:object_r:container_file_t:s0 "{{ normalize_backup_root }}"
+ register: normalize_backup_selinux
+ changed_when: true
+ failed_when: false
+ when: normalize_backup_root_stat.stat.exists | default(false)
+ delegate_to: oim
+ delegate_facts: true
+ connection: ssh
+
+ - name: Detect unreadable items under backup root (OIM host)
ansible.builtin.shell: |
set -o pipefail
{
@@ -75,6 +103,9 @@
changed_when: false
failed_when: false
when: normalize_backup_root_stat.stat.exists | default(false)
+ delegate_to: oim
+ delegate_facts: true
+ connection: ssh
# ── 2. /etc/openchami (OIM host — delegated) ────────────────────────
- name: Check /etc/openchami exists (OIM host)
@@ -98,6 +129,17 @@
delegate_facts: true
connection: ssh
+ - name: Fix SELinux context on /etc/openchami (OIM host)
+ ansible.builtin.command: >
+ chcon -R system_u:object_r:container_file_t:s0 "{{ normalize_etc_root }}"
+ register: normalize_etc_selinux
+ changed_when: true
+ failed_when: false
+ when: normalize_etc_root_stat.stat.exists | default(false)
+ delegate_to: oim
+ delegate_facts: true
+ connection: ssh
+
- name: Detect unreadable items under /etc/openchami (OIM host)
ansible.builtin.shell: |
set -o pipefail
diff --git a/src/playbooks/rollback/roles/rollback_openchami/vars/main.yml b/src/playbooks/rollback/roles/rollback_openchami/vars/main.yml
index 73cf84e88e..b46541de7f 100644
--- a/src/playbooks/rollback/roles/rollback_openchami/vars/main.yml
+++ b/src/playbooks/rollback/roles/rollback_openchami/vars/main.yml
@@ -133,7 +133,7 @@ rollback_messages:
Rollback is not needed. Skipping.
backup_found: "Backup directory found. Proceeding with rollback."
permissions:
- normalized: "Backup and /etc/openchami permissions verified (directories 0755, files 0644)."
+ normalized: "Backup and /etc/openchami permissions and SELinux contexts verified (directories 0755, files 0644, context container_file_t)."
unfixable: |
════════════════════════════════════════════
OPENCHAMI ROLLBACK BLOCKED — PERMISSION ISSUE
@@ -141,26 +141,31 @@ rollback_messages:
One or more files/directories in the backup or /etc/openchami could not
be made readable. The rollback cannot read these to restore them.
- This usually happens on the shared NFS backup path (root_squash) when
- container-created files carry restrictive ownership/permissions.
+ This usually happens due to:
+ 1. SELinux context issues (files need container_file_t context)
+ 2. NFS root_squash with restrictive ownership/permissions
- Fix the permissions MANUALLY, then re-run ONLY the OpenCHAMI/OIM rollback:
+ Fix the permissions and SELinux context MANUALLY on the OIM host,
+ then re-run ONLY the OpenCHAMI/OIM rollback:
- On the OIM host (as a user that owns the files, e.g. the NFS owner):
+ On the OIM host (SSH to the OIM, NOT the core container):
+ # Fix permissions
chmod -R u+rwX,go+rX /etc/openchami
- chmod -R u+rwX,go+rX {{ rollback_backup_dir | default(rollback_backup_dir_default) }}/openchami
+ chmod -R u+rwX,go+rX {{ rollback_oim_host_backup_dir }}/openchami
- Ensure all directories are 0755 and all files are 0644, and that none
- are in a permission-denied / immutable state. Then retry:
+ # Fix SELinux context
+ chcon -R system_u:object_r:container_file_t:s0 /etc/openchami
+ chcon -R system_u:object_r:container_file_t:s0 {{ rollback_oim_host_backup_dir }}/openchami
+ Then retry from the core container:
cd /opt/omnia/... && ansible-playbook rollback/rollback.yml --tags oim
- Relevant directories that must be readable:
+ Relevant directories on OIM host that must be readable:
/etc/openchami, /etc/openchami/configs, /etc/openchami/pg-init
- {{ rollback_backup_dir | default(rollback_backup_dir_default) }}/openchami/etc_openchami
- {{ rollback_backup_dir | default(rollback_backup_dir_default) }}/openchami/etc_openchami/configs
- {{ rollback_backup_dir | default(rollback_backup_dir_default) }}/openchami/etc_openchami/pg-init
- {{ rollback_backup_dir | default(rollback_backup_dir_default) }}/openchami/postgresql_backup
+ {{ rollback_oim_host_backup_dir }}/openchami/etc_openchami
+ {{ rollback_oim_host_backup_dir }}/openchami/etc_openchami/configs
+ {{ rollback_oim_host_backup_dir }}/openchami/etc_openchami/pg-init
+ {{ rollback_oim_host_backup_dir }}/openchami/postgresql_backup
restore:
quadlets_success: "Restored v2.1 quadlet files from backup."
quadlets_failure: "Failed to restore v2.1 quadlet files."
diff --git a/src/playbooks/rollback/roles/rollback_pulp/tasks/cleanup_pulp_data.yml b/src/playbooks/rollback/roles/rollback_pulp/tasks/cleanup_pulp_data.yml
new file mode 100644
index 0000000000..826bb70bce
--- /dev/null
+++ b/src/playbooks/rollback/roles/rollback_pulp/tasks/cleanup_pulp_data.yml
@@ -0,0 +1,100 @@
+# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+
+# ===========================================================================
+# Cleanup Pulp Data After Rollback
+#
+# - Triggers Pulp DB orphan cleanup + removes filesystem orphans
+# via pulp_fs_orphan_cleanup module (reads creds from cli.toml)
+# - Cleans up redundant PG 16 data from backup
+# - Removes temporary files from pgsql directory
+# ===========================================================================
+
+# ── Orphan cleanup (DB-level + filesystem-level) ────────────────────────────
+# Runs inside omnia_core where cli.toml and media directory are accessible.
+# No password arguments — credentials read from /root/.config/pulp/cli.toml.
+- name: Run Pulp orphan cleanup (DB + filesystem)
+ pulp_fs_orphan_cleanup:
+ media_dir: "{{ pulp_data_base_path }}/pulp_storage/media"
+ trigger_db_orphan_cleanup: true
+ register: fs_orphan_result
+ failed_when: false
+
+- name: Display orphan cleanup result
+ ansible.builtin.debug:
+ msg:
+ - "DB orphan cleanup: {{ fs_orphan_result.db_orphan_cleanup_status | default('N/A') }}"
+ - "Disk artifacts: {{ fs_orphan_result.disk_artifact_count | default('N/A') }}"
+ - "DB artifacts: {{ fs_orphan_result.db_artifact_count | default('N/A') }}"
+ - "Orphans removed: {{ fs_orphan_result.removed_count | default(0) }}"
+ - "Space freed: {{ fs_orphan_result.freed_mb | default(0) }} MB"
+
+# ── Clean up backup and temp data ───────────────────────────────────────────
+- name: Clean up backup and temporary PostgreSQL data
+ ansible.builtin.shell: |
+ set -o pipefail
+
+ BACKUP="{{ oim_host_pgsql_backup }}"
+ LIVE="{{ oim_host_pgsql_path }}"
+
+ # Clean backup: remove PG 16 data, rename data_old to data
+ if [ -d "$BACKUP/data" ]; then
+ BK_VER=$(cat "$BACKUP/data/PG_VERSION" 2>/dev/null || echo "unknown")
+ if [ "$BK_VER" != "12" ] && [ "$BK_VER" != "13" ]; then
+ rm -rf "$BACKUP/data"
+ echo "Removed PG $BK_VER data from backup"
+ DATA_OLD=$(find "$BACKUP" -maxdepth 1 -type d -name "data_old.*" 2>/dev/null | head -1)
+ if [ -n "$DATA_OLD" ] && [ -d "$DATA_OLD" ]; then
+ mv "$DATA_OLD" "$BACKUP/data"
+ echo "Renamed $(basename "$DATA_OLD") to data in backup"
+ fi
+ fi
+ fi
+
+ # Clean live pgsql: remove data_old.* and data_pg16* directories
+ find "$LIVE" -maxdepth 1 -type d \( -name "data_old.*" -o -name "data_pg16*" \) -exec rm -rf {} \; 2>/dev/null
+ echo "Cleaned up temporary directories"
+ args:
+ executable: /bin/bash
+ delegate_to: oim
+ delegate_facts: true
+ connection: ssh
+ register: cleanup_result
+ changed_when: true
+ failed_when: false
+
+- name: Display cleanup result
+ ansible.builtin.debug:
+ msg: "{{ cleanup_result.stdout_lines | default(['Cleanup completed']) }}"
+
+# ── Report disk usage ──────────────────────────────────────────────────────
+- name: Report disk usage after rollback
+ ansible.builtin.shell: |
+ set -o pipefail
+ echo "Pulp storage: $(du -sh {{ oim_host_pulp_data_base }}/pulp_storage 2>/dev/null | cut -f1)"
+ echo "PostgreSQL: $(du -sh {{ oim_host_pgsql_path }} 2>/dev/null | cut -f1)"
+ echo "Backup: $(du -sh {{ oim_host_backup_dir }} 2>/dev/null | cut -f1)"
+ args:
+ executable: /bin/bash
+ delegate_to: oim
+ delegate_facts: true
+ connection: ssh
+ register: disk_usage_result
+ changed_when: false
+ failed_when: false
+
+- name: Display disk usage
+ ansible.builtin.debug:
+ msg: "{{ disk_usage_result.stdout_lines | default([]) }}"
diff --git a/src/playbooks/rollback/roles/rollback_pulp/tasks/main.yml b/src/playbooks/rollback/roles/rollback_pulp/tasks/main.yml
new file mode 100644
index 0000000000..ba34c235d2
--- /dev/null
+++ b/src/playbooks/rollback/roles/rollback_pulp/tasks/main.yml
@@ -0,0 +1,88 @@
+# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+
+# ============================================================================
+# rollback_pulp — Main Orchestration
+# ============================================================================
+# Reverse the Pulp upgrade (3.113 → 3.80) by:
+# 1. Resolving admin NIC IP for health checks
+# 2. Pre-rollback checks (verify Pulp is deployed, backup exists)
+# 3. Stopping the current Pulp container
+# 4. Restoring PostgreSQL data from backup (PG 12/13 compatible)
+# 5. Restoring the v2.1 quadlet file from backup
+# 6. Pulling the v2.1 Pulp image (3.80)
+# 7. Starting the rolled-back container
+# 8. Post-rollback health check
+# 9. Cleanup (orphan artifacts, redundant backup data)
+#
+# IMPORTANT: PostgreSQL data must be restored because:
+# - Pulp 3.80 uses PostgreSQL 12/13
+# - Pulp 3.113 uses PostgreSQL 16
+# - PostgreSQL cannot downgrade data files between major versions
+#
+# No idempotency check — rollback always executes regardless of current
+# container state. This ensures rollback works even in partial or failed
+# upgrade states.
+# ============================================================================
+
+- name: Read oim_metadata.yml for shared storage path
+ ansible.builtin.slurp:
+ src: "{{ oim_metadata_path }}"
+ register: _pulp_oim_metadata_raw
+
+- name: Set OIM host-side paths from metadata
+ ansible.builtin.set_fact:
+ oim_shared_path: "{{ (_pulp_oim_metadata_raw.content | b64decode | from_yaml).oim_shared_path | regex_replace('/$', '') }}"
+
+- name: Derive OIM host-side Pulp paths
+ ansible.builtin.set_fact:
+ oim_host_pulp_data_base: "{{ oim_shared_path }}/omnia/pulp/settings"
+ oim_host_pgsql_path: "{{ oim_shared_path }}/omnia/pulp/settings/pgsql"
+ oim_host_backup_dir: "{{ oim_shared_path }}/omnia/backups/upgrade/version_2.1.0.0/pulp"
+ oim_host_pgsql_backup: "{{ oim_shared_path }}/omnia/backups/upgrade/version_2.1.0.0/pulp/pgsql"
+ oim_host_rollback_backup_dir: >-
+ {{ (rollback_backup_dir | default(rollback_backup_dir_default))
+ | regex_replace('^/opt/omnia', oim_shared_path ~ '/omnia') }}
+
+- name: Resolve admin NIC IP for Pulp API endpoints
+ ansible.builtin.include_tasks: resolve_admin_ip.yml
+
+- name: Pulp rollback workflow
+ block:
+ - name: Pre-rollback validation checks
+ ansible.builtin.include_tasks: pre_rollback_checks.yml
+
+ - name: Execute Pulp rollback
+ ansible.builtin.include_tasks: rollback_pulp_container.yml
+ when: pulp_deployed | default(false) | bool
+
+ - name: Post-rollback health check
+ ansible.builtin.include_tasks: post_rollback_health_check.yml
+ when: pulp_deployed | default(false) | bool
+
+ - name: Post-rollback cleanup (orphans and temporary data)
+ ansible.builtin.include_tasks: cleanup_pulp_data.yml
+ when:
+ - pulp_deployed | default(false) | bool
+ - pulp_rollback_success | default(false) | bool
+
+ rescue:
+ - name: Set rollback failure flag
+ ansible.builtin.set_fact:
+ pulp_rollback_failed: true
+
+ always:
+ - name: Rollback status and cleanup
+ ansible.builtin.include_tasks: rollback_status.yml
diff --git a/src/playbooks/rollback/roles/rollback_pulp/tasks/post_rollback_health_check.yml b/src/playbooks/rollback/roles/rollback_pulp/tasks/post_rollback_health_check.yml
new file mode 100644
index 0000000000..dec934a8db
--- /dev/null
+++ b/src/playbooks/rollback/roles/rollback_pulp/tasks/post_rollback_health_check.yml
@@ -0,0 +1,91 @@
+# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+
+# ============================================================================
+# post_rollback_health_check.yml — Verify Pulp Rollback Success
+#
+# Validates:
+# 1. Pulp container is running with the correct (rolled-back) image
+# 2. Pulp API is accessible and responding
+# Sets pulp_rollback_success flag for cleanup tasks.
+# ============================================================================
+
+- name: Post-rollback health check
+ block:
+ # ── Verify container is running with correct image ────────────────────
+ - name: Get rolled-back Pulp container info
+ containers.podman.podman_container_info:
+ name: "{{ pulp_container_name }}"
+ register: pulp_post_info
+ delegate_to: oim
+ delegate_facts: true
+ connection: ssh
+
+ - name: Verify Pulp is running with rolled-back image
+ ansible.builtin.assert:
+ that:
+ - pulp_post_info.containers | length > 0
+ - pulp_post_info.containers[0].State.Status == 'running'
+ - pulp_rollback_tag in (pulp_post_info.containers[0].ImageName | default(''))
+ fail_msg: |
+ Pulp rollback verification failed.
+ Expected image tag: {{ pulp_rollback_tag }}
+ Actual image: {{ pulp_post_info.containers[0].ImageName | default('unknown') }}
+ Container status: {{ pulp_post_info.containers[0].State.Status | default('unknown') }}
+ success_msg: "Pulp container is running with rolled-back image: {{ pulp_post_info.containers[0].ImageName | default('unknown') }}"
+
+ # ── Verify Pulp API is accessible ─────────────────────────────────────
+ - name: Check Pulp API status after rollback
+ ansible.builtin.uri:
+ url: "{{ pulp_status_url }}"
+ method: GET
+ validate_certs: false
+ status_code: [200]
+ return_content: true
+ register: pulp_post_health
+ retries: "{{ pulp_health_retries }}"
+ delay: "{{ pulp_health_delay }}"
+ until: pulp_post_health.status == 200
+ delegate_to: oim
+ delegate_facts: true
+ connection: ssh
+
+ - name: Display Pulp workers status
+ ansible.builtin.debug:
+ msg: >-
+ Pulp API: healthy (HTTP {{ pulp_post_health.status }}) |
+ Workers: {{ (pulp_post_health.json | default({})).online_workers | default([]) | length }} |
+ Content apps: {{ (pulp_post_health.json | default({})).online_content_apps | default([]) | length }}
+
+ - name: Set rollback success flag
+ ansible.builtin.set_fact:
+ pulp_rollback_success: true
+
+ - name: Display rollback success message
+ ansible.builtin.debug:
+ msg: "{{ rollback_messages.pulp.rollback_success }}"
+
+ rescue:
+ - name: Display rollback health check failure
+ ansible.builtin.debug:
+ msg: "{{ rollback_messages.pulp.post_check_failure }}"
+
+ - name: Set rollback failure flag
+ ansible.builtin.set_fact:
+ pulp_rollback_failed: true
+
+ - name: Fail with health check error
+ ansible.builtin.fail:
+ msg: "Pulp post-rollback health check failed. Check container logs: podman logs {{ pulp_container_name }}"
diff --git a/src/playbooks/rollback/roles/rollback_pulp/tasks/pre_rollback_checks.yml b/src/playbooks/rollback/roles/rollback_pulp/tasks/pre_rollback_checks.yml
new file mode 100644
index 0000000000..71a5279528
--- /dev/null
+++ b/src/playbooks/rollback/roles/rollback_pulp/tasks/pre_rollback_checks.yml
@@ -0,0 +1,134 @@
+# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+
+# ============================================================================
+# pre_rollback_checks.yml — Validate Pulp Rollback Preconditions
+# ============================================================================
+# Checks:
+# 1. Pulp container exists and is deployed
+# 2. Backup directory exists with Pulp backup
+# 3. Backed-up quadlet file or image tag exists
+#
+# No idempotency check — rollback always executes regardless of current
+# container state. This ensures rollback works even in partial or failed
+# upgrade states.
+# ============================================================================
+
+- name: Pre-rollback validation
+ block:
+ # ── Check if Pulp container exists ────────────────────────────────────
+ - name: Check if Pulp container exists
+ ansible.builtin.command: podman container exists {{ pulp_container_name }}
+ register: pulp_exists_check
+ changed_when: false
+ failed_when: false
+ delegate_to: oim
+ delegate_facts: true
+ connection: ssh
+
+ - name: Set Pulp deployment status
+ ansible.builtin.set_fact:
+ pulp_deployed: "{{ pulp_exists_check.rc == 0 }}"
+
+ - name: Display Pulp not deployed message
+ ansible.builtin.debug:
+ msg: "{{ rollback_messages.pulp.not_deployed }}"
+ when: not pulp_deployed
+
+ # ── Get current Pulp container info ───────────────────────────────────
+ - name: Get current Pulp container info
+ containers.podman.podman_container_info:
+ name: "{{ pulp_container_name }}"
+ register: pulp_pre_info
+ delegate_to: oim
+ delegate_facts: true
+ connection: ssh
+ when: pulp_deployed
+
+ - name: Extract current Pulp image version
+ ansible.builtin.set_fact:
+ pulp_current_image: "{{ pulp_pre_info.containers[0].ImageName | default('unknown') }}"
+ pulp_current_status: "{{ pulp_pre_info.containers[0].State.Status | default('unknown') }}"
+ when:
+ - pulp_deployed
+ - pulp_pre_info.containers | length > 0
+
+ - name: Display current Pulp version
+ ansible.builtin.debug:
+ msg: "Current Pulp image: {{ pulp_current_image | default('not found') }}, Status: {{ pulp_current_status | default('unknown') }}"
+ when: pulp_deployed
+
+ # ── Check if already at rollback target version ───────────────────────
+ - name: Check if Pulp is already at rollback target version
+ ansible.builtin.set_fact:
+ pulp_already_rolled_back: "{{ (pulp_rollback_image | default('')) in ((pulp_current_image | default('')) | string) }}"
+ when: pulp_deployed
+
+ - name: Display skip message if already at target version
+ ansible.builtin.debug:
+ msg: "{{ rollback_messages.pulp.already_v21 }}"
+ when:
+ - pulp_deployed
+ - pulp_already_rolled_back | default(false)
+
+ # ── Verify backup directory exists ────────────────────────────────────
+ - name: Check backup directory exists
+ ansible.builtin.stat:
+ path: "{{ rollback_backup_dir | default(rollback_backup_dir_default) }}/{{ backup_pulp_subpath }}"
+ register: rollback_pulp_backup_stat
+ when: pulp_deployed
+
+ - name: Check backed-up quadlet file exists
+ ansible.builtin.stat:
+ path: "{{ rollback_backup_dir | default(rollback_backup_dir_default) }}/{{ backup_pulp_quadlet_subpath }}/{{ pulp_container_name }}.container"
+ register: rollback_pulp_quadlet_stat
+ when: pulp_deployed
+
+ - name: Check backed-up image tag file exists
+ ansible.builtin.stat:
+ path: "{{ rollback_backup_dir | default(rollback_backup_dir_default) }}/{{ backup_pulp_image_tag_subpath }}"
+ register: rollback_pulp_image_tag_stat
+ when: pulp_deployed
+
+ - name: Display backup inventory
+ ansible.builtin.debug:
+ verbosity: 1
+ msg:
+ - "Backup directory: {{ rollback_backup_dir | default(rollback_backup_dir_default) }}/{{ backup_pulp_subpath }}"
+ - "Pulp backup: {{ 'found' if rollback_pulp_backup_stat.stat.exists | default(false) else 'MISSING' }}"
+ - "Quadlet backup: {{ 'found' if rollback_pulp_quadlet_stat.stat.exists | default(false) else 'MISSING' }}"
+ - "Image tag backup: {{ 'found' if rollback_pulp_image_tag_stat.stat.exists | default(false) else 'MISSING' }}"
+ when: pulp_deployed
+
+ # ── Determine rollback strategy ───────────────────────────────────────
+ # If quadlet backup exists, restore it. Otherwise, update image tag in current quadlet.
+ - name: Determine rollback strategy
+ ansible.builtin.set_fact:
+ pulp_rollback_strategy: >-
+ {{ 'restore_quadlet' if (rollback_pulp_quadlet_stat.stat.exists | default(false))
+ else 'update_image_tag' }}
+ when: pulp_deployed
+
+ - name: Display rollback strategy
+ ansible.builtin.debug:
+ msg: "Pulp rollback strategy: {{ pulp_rollback_strategy | default('N/A') }}"
+ when: pulp_deployed
+
+ - name: Display rollback proceeding message
+ ansible.builtin.debug:
+ msg: "{{ rollback_messages.pulp.backup_found }}"
+ when:
+ - pulp_deployed
+ - rollback_pulp_backup_stat.stat.exists | default(false) or not (pulp_already_rolled_back | default(false))
diff --git a/src/playbooks/rollback/roles/rollback_pulp/tasks/resolve_admin_ip.yml b/src/playbooks/rollback/roles/rollback_pulp/tasks/resolve_admin_ip.yml
new file mode 100644
index 0000000000..339b864d0d
--- /dev/null
+++ b/src/playbooks/rollback/roles/rollback_pulp/tasks/resolve_admin_ip.yml
@@ -0,0 +1,95 @@
+# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+
+# ============================================================================
+# resolve_admin_ip.yml — Resolve Admin NIC IP for Pulp API Health Checks
+# ============================================================================
+# Resolves the admin_nic_ip needed for Pulp API health check endpoints.
+#
+# Resolution order:
+# 1. Primary: Extract from local_repo_access.yml (same as upgrade_k8s)
+# 2. Fallback: Extract from network_spec.yml (same as prepare_oim)
+# 3. Last resort: Use localhost
+#
+# Sets fact:
+# admin_nic_ip — IP address for Pulp API health checks
+# ============================================================================
+
+- name: Resolve admin NIC IP for Pulp health checks
+ block:
+ # Primary method: Extract from local_repo_access.yml (same as upgrade_k8s)
+ - name: Check if local_repo_access.yml exists
+ ansible.builtin.stat:
+ path: "{{ local_repo_access_path }}"
+ register: local_repo_access_stat
+
+ - name: Load local_repo_access.yml
+ ansible.builtin.slurp:
+ src: "{{ local_repo_access_path }}"
+ register: local_repo_access_raw
+ when: local_repo_access_stat.stat.exists
+
+ - name: Parse local_repo_access.yml
+ ansible.builtin.set_fact:
+ _local_repo_access: "{{ local_repo_access_raw.content | b64decode | from_yaml }}"
+ when: local_repo_access_stat.stat.exists
+
+ - name: Set admin_nic_ip from local_repo_access
+ ansible.builtin.set_fact:
+ admin_nic_ip: "{{ _local_repo_access.offline_tarball_path | regex_replace('^(https?)://([^:]+):.*', '\\2') }}"
+ when:
+ - local_repo_access_stat.stat.exists
+ - _local_repo_access.offline_tarball_path is defined
+
+ # Fallback method: Extract from network_spec.yml
+ - name: Check if network_spec.yml exists
+ ansible.builtin.stat:
+ path: "{{ network_spec_path }}"
+ register: network_spec_stat
+ when: admin_nic_ip is not defined
+
+ - name: Load network_spec.yml as fallback
+ ansible.builtin.include_vars:
+ file: "{{ network_spec_path }}"
+ when:
+ - admin_nic_ip is not defined
+ - network_spec_stat.stat.exists | default(false)
+
+ # Networks is a list in network_spec.yml, parse it to extract admin_network
+ - name: Parse network_spec data into network_data
+ ansible.builtin.set_fact:
+ network_data: "{{ network_data | default({}) | combine({item.keys() | first: item.values() | first}) }}"
+ loop: "{{ Networks }}"
+ when:
+ - admin_nic_ip is not defined
+ - Networks is defined
+
+ - name: Set admin_nic_ip from network_spec
+ ansible.builtin.set_fact:
+ admin_nic_ip: "{{ network_data.admin_network.primary_oim_admin_ip }}"
+ when:
+ - admin_nic_ip is not defined
+ - network_data is defined
+ - network_data.admin_network is defined
+ - network_data.admin_network.primary_oim_admin_ip is defined
+
+ rescue:
+ - name: Fallback to localhost for admin_nic_ip
+ ansible.builtin.set_fact:
+ admin_nic_ip: "localhost"
+
+- name: Display resolved admin_nic_ip
+ ansible.builtin.debug:
+ msg: "Pulp rollback health checks will use admin_nic_ip: {{ admin_nic_ip }}"
diff --git a/src/playbooks/rollback/roles/rollback_pulp/tasks/restore_pulp_data.yml b/src/playbooks/rollback/roles/rollback_pulp/tasks/restore_pulp_data.yml
new file mode 100644
index 0000000000..c075481ce4
--- /dev/null
+++ b/src/playbooks/rollback/roles/rollback_pulp/tasks/restore_pulp_data.yml
@@ -0,0 +1,41 @@
+# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+
+# ===========================================================================
+# Restore Pulp PostgreSQL Data During Rollback
+#
+# Uses the pulp_pgsql_backup_restore module to intelligently restore PG 12/13
+# data from the backup, handling the case where the backup may contain
+# PG 16 data (from the upgrade) with the original PG 12/13 data in
+# a data_old.* directory.
+#
+# Runs inside omnia_core where backup and pgsql paths are accessible
+# via shared NFS storage at /opt/omnia/.
+# ===========================================================================
+
+- name: Restore Pulp PostgreSQL data for rollback
+ pulp_pgsql_backup_restore:
+ action: restore
+ backup_path: "{{ pulp_pgsql_backup_path }}"
+ dest_path: "{{ pulp_pgsql_path }}"
+ register: pgsql_restore_result
+
+- name: Display restore result
+ ansible.builtin.debug:
+ msg:
+ - "Restore mode: {{ pgsql_restore_result.restore_mode }}"
+ - "Backup PG version: {{ pgsql_restore_result.backup_pg_version }}"
+ - "Restored PG version: {{ pgsql_restore_result.restored_pg_version }}"
+ - "{{ pgsql_restore_result.messages | join(', ') }}"
diff --git a/src/playbooks/rollback/roles/rollback_pulp/tasks/rollback_pulp_container.yml b/src/playbooks/rollback/roles/rollback_pulp/tasks/rollback_pulp_container.yml
new file mode 100644
index 0000000000..cb74c969aa
--- /dev/null
+++ b/src/playbooks/rollback/roles/rollback_pulp/tasks/rollback_pulp_container.yml
@@ -0,0 +1,131 @@
+# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+
+# ============================================================================
+# Pulp Container Rollback (3.113 → 3.80)
+#
+# Steps:
+# 1. Pull the v2.1 Pulp image (3.80)
+# 2. Stop and remove current container
+# 3. Restore PostgreSQL data (PG 16 → PG 12/13)
+# 4. Update quadlet file with rollback image tag
+# 5. Reload systemd, start container, wait for init
+# ============================================================================
+
+- name: Skip rollback if already at target version
+ ansible.builtin.debug:
+ msg: "Pulp is already at rollback target version. Skipping."
+ when: pulp_already_rolled_back | default(false)
+
+- name: Execute Pulp container rollback
+ when: not (pulp_already_rolled_back | default(false))
+ block:
+ # --- 1. Pull v2.1 Pulp image ---
+ - name: Pull v2.1 Pulp image ({{ pulp_rollback_image }})
+ containers.podman.podman_image:
+ name: "{{ pulp_rollback_image }}"
+ state: present
+ force: true
+ register: pulp_image_pull
+ retries: "{{ pull_image_retries }}"
+ delay: "{{ pull_image_delay }}"
+ until: pulp_image_pull is succeeded
+ delegate_to: oim
+ delegate_facts: true
+ connection: ssh
+
+ # --- 2. Stop and remove current container ---
+ - name: Stop Pulp systemd service
+ ansible.builtin.systemd:
+ name: "{{ pulp_container_name }}"
+ state: stopped
+ delegate_to: oim
+ delegate_facts: true
+ connection: ssh
+ failed_when: false
+
+ - name: Force remove Pulp container
+ ansible.builtin.command: podman rm -f {{ pulp_container_name }}
+ delegate_to: oim
+ delegate_facts: true
+ connection: ssh
+ changed_when: true
+ failed_when: false
+
+ # --- 3. Restore PostgreSQL data from backup ---
+ - name: Restore Pulp PostgreSQL data from backup
+ ansible.builtin.include_tasks: restore_pulp_data.yml
+
+ # --- 4. Update quadlet file with rollback image ---
+ - name: Restore quadlet file from backup
+ ansible.builtin.copy:
+ src: "{{ oim_host_rollback_backup_dir }}/{{ backup_pulp_quadlet_subpath }}/{{ pulp_container_name }}.container"
+ dest: "{{ pulp_quadlet_path }}"
+ mode: "{{ file_permissions_644 }}"
+ remote_src: true
+ delegate_to: oim
+ delegate_facts: true
+ connection: ssh
+ when: pulp_rollback_strategy | default('update_image_tag') == 'restore_quadlet'
+
+ - name: Update Image line in quadlet file
+ ansible.builtin.replace:
+ path: "{{ pulp_quadlet_path }}"
+ regexp: '^Image=.*pulp.*$'
+ replace: "Image={{ pulp_rollback_image }}"
+ delegate_to: oim
+ delegate_facts: true
+ connection: ssh
+ when: pulp_rollback_strategy | default('update_image_tag') == 'update_image_tag'
+
+ # --- 5. Reload systemd, start container, wait for init ---
+ - name: Reload systemd daemon
+ ansible.builtin.systemd:
+ daemon_reload: true
+ delegate_to: oim
+ delegate_facts: true
+ connection: ssh
+
+ - name: Start Pulp systemd service
+ ansible.builtin.systemd:
+ name: "{{ pulp_container_name }}"
+ state: started
+ enabled: true
+ register: pulp_start_result
+ retries: "{{ pulp_startup_retries }}"
+ delay: "{{ pulp_startup_delay }}"
+ until: pulp_start_result is succeeded
+ delegate_to: oim
+ delegate_facts: true
+ connection: ssh
+
+ - name: Wait for Pulp container to initialize
+ ansible.builtin.pause:
+ seconds: "{{ pulp_init_wait }}"
+
+ - name: Verify Pulp container is running
+ ansible.builtin.command: podman ps --filter name={{ pulp_container_name }} --format "{{ '{{' }}.Status{{ '}}' }}"
+ register: pulp_status_check
+ delegate_to: oim
+ delegate_facts: true
+ connection: ssh
+ changed_when: false
+ retries: "{{ pulp_startup_retries }}"
+ delay: "{{ pulp_startup_delay }}"
+ until: "'Up' in pulp_status_check.stdout"
+
+ - name: Display Pulp container status
+ ansible.builtin.debug:
+ msg: "Pulp container status: {{ pulp_status_check.stdout }}"
diff --git a/src/playbooks/rollback/roles/rollback_pulp/tasks/rollback_status.yml b/src/playbooks/rollback/roles/rollback_pulp/tasks/rollback_status.yml
new file mode 100644
index 0000000000..35fb823f9c
--- /dev/null
+++ b/src/playbooks/rollback/roles/rollback_pulp/tasks/rollback_status.yml
@@ -0,0 +1,52 @@
+# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+
+# ============================================================================
+# rollback_status.yml — Report Pulp Rollback Status
+# ============================================================================
+# Reports the final status of the Pulp rollback operation.
+# This task runs in the 'always' block to ensure status is reported
+# regardless of success or failure.
+# ============================================================================
+
+- name: Determine final rollback status
+ ansible.builtin.set_fact:
+ pulp_rollback_final_status: >-
+ {{ 'skipped' if not (pulp_deployed | default(false))
+ else ('skipped' if (pulp_already_rolled_back | default(false))
+ else ('failed' if (pulp_rollback_failed | default(false))
+ else 'completed')) }}
+
+- name: Display Pulp rollback summary
+ ansible.builtin.debug:
+ msg:
+ - "════════════════════════════════════════════════════════════"
+ - " PULP ROLLBACK STATUS: {{ pulp_rollback_final_status | upper }}"
+ - "════════════════════════════════════════════════════════════"
+ - "Deployed: {{ pulp_deployed | default(false) }}"
+ - "Already at target: {{ pulp_already_rolled_back | default(false) }}"
+ - "Rollback failed: {{ pulp_rollback_failed | default(false) }}"
+ - "Target image: {{ pulp_rollback_image }}"
+ - "════════════════════════════════════════════════════════════"
+
+- name: Display failure message if rollback failed
+ ansible.builtin.debug:
+ msg: "{{ rollback_messages.pulp.rollback_failure }}"
+ when: pulp_rollback_failed | default(false)
+
+- name: Re-raise failure if rollback failed
+ ansible.builtin.fail:
+ msg: "Pulp rollback failed. See above for details."
+ when: pulp_rollback_failed | default(false)
diff --git a/src/playbooks/rollback/roles/rollback_pulp/vars/main.yml b/src/playbooks/rollback/roles/rollback_pulp/vars/main.yml
new file mode 100644
index 0000000000..21ff282c46
--- /dev/null
+++ b/src/playbooks/rollback/roles/rollback_pulp/vars/main.yml
@@ -0,0 +1,120 @@
+# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+
+# ============================================================================
+# rollback_pulp — Variables
+# ============================================================================
+# Rollback target: Pulp 3.80 (from Pulp 3.113)
+# Reverses the upgrade performed by upgrade_pulp role.
+# ============================================================================
+
+# File permissions
+dir_permissions_755: "0755"
+file_permissions_644: "0644"
+
+# Manifest path for locating backup directory
+manifest_path: "/opt/omnia/.data/upgrade_manifest.yml"
+
+# Default backup directory (must match upgrade_pulp)
+rollback_backup_dir_default: "/opt/omnia/backups/upgrade/version_2.1.0.0"
+
+# Pulp container settings
+pulp_container_name: "pulp"
+pulp_rollback_tag: "3.80"
+pulp_rollback_image: "docker.io/pulp/pulp:{{ pulp_rollback_tag }}"
+
+# Pulp quadlet file path
+pulp_quadlet_path: "/etc/containers/systemd/{{ pulp_container_name }}.container"
+
+# OIM metadata (for resolving host-side shared storage paths)
+oim_metadata_path: "/opt/omnia/.data/oim_metadata.yml"
+
+# Paths for resolving admin_nic_ip
+local_repo_access_path: "/opt/omnia/provision/local_repo_access.yml"
+network_spec_path: "{{ input_project_dir | default('/opt/omnia/input') }}/network_spec.yml"
+
+# Pulp API endpoint for health checks
+pulp_protocol: "https"
+pulp_port: "2225"
+pulp_status_url: "{{ pulp_protocol }}://{{ admin_nic_ip | default('localhost') }}:{{ pulp_port }}/pulp/api/v3/status/"
+
+# Image pull settings
+pull_image_retries: 5
+pull_image_delay: 10
+
+# Pulp startup and health check settings
+pulp_startup_retries: 10
+pulp_startup_delay: 10
+pulp_init_wait: 30
+pulp_health_retries: 12
+pulp_health_delay: 10
+
+# Backup sub-paths (relative to rollback_backup_dir)
+backup_pulp_subpath: "pulp"
+backup_pulp_quadlet_subpath: "pulp/quadlet"
+backup_pulp_image_tag_subpath: "pulp/image_tag.txt"
+
+# Pulp shared storage paths (OIM container paths)
+# These paths are inside the OIM container and mounted as volumes in the Pulp container
+pulp_data_base_path: "/opt/omnia/pulp/settings"
+pulp_pgsql_path: "{{ pulp_data_base_path }}/pgsql"
+
+# PostgreSQL backup paths
+# PostgreSQL data must be restored during rollback because:
+# - Pulp 3.80 uses PostgreSQL 12/13
+# - Pulp 3.113 uses PostgreSQL 16
+# - PostgreSQL cannot downgrade data files between major versions
+pulp_backup_dir: "/opt/omnia/backups/upgrade/version_2.1.0.0/pulp"
+pulp_pgsql_backup_path: "{{ pulp_backup_dir }}/pgsql"
+
+# Rollback messages
+rollback_messages:
+ pulp:
+ not_deployed: |
+ Pulp container is not deployed on the OIM. Skipping Pulp rollback.
+ no_backup: |
+ FATAL: Pulp backup not found at {{ rollback_backup_dir | default(rollback_backup_dir_default) }}/{{ backup_pulp_subpath }}.
+ Cannot rollback without a valid backup.
+ already_v21: |
+ Pulp is already at v2.1 version ({{ pulp_rollback_image }}). Skipping rollback.
+ backup_found: "Pulp backup found. Proceeding with rollback."
+ rollback_success: |
+ ════════════════════════════════════════════════════════════
+ PULP ROLLBACK COMPLETED SUCCESSFULLY
+ ════════════════════════════════════════════════════════════
+ Previous image: {{ pulp_current_image | default('unknown') }}
+ Rolled back to: {{ pulp_rollback_image }}
+ Container: {{ pulp_container_name }}
+ Status: running
+ API endpoint: {{ pulp_status_url }}
+ ════════════════════════════════════════════════════════════
+ rollback_failure: |
+ ════════════════════════════════════════════════════════════
+ PULP ROLLBACK FAILED
+ ════════════════════════════════════════════════════════════
+ Target image: {{ pulp_rollback_image }}
+
+ Please check:
+ - Pulp container logs: podman logs {{ pulp_container_name }}
+ - Pulp service status: systemctl status {{ pulp_container_name }}
+ - Shared storage accessibility
+ ════════════════════════════════════════════════════════════
+ pre_check_failure: |
+ Pulp pre-rollback check failed.
+ Please verify Pulp container state before attempting rollback.
+ post_check_failure: |
+ Pulp post-rollback health check failed.
+ The rolled-back Pulp container may not be functioning correctly.
+ Check container logs: podman logs {{ pulp_container_name }}
diff --git a/src/playbooks/upgrade/playbooks/upgrade_k8s.yml b/src/playbooks/upgrade/playbooks/upgrade_k8s.yml
index 5c492c5b43..64c4ad3d74 100644
--- a/src/playbooks/upgrade/playbooks/upgrade_k8s.yml
+++ b/src/playbooks/upgrade/playbooks/upgrade_k8s.yml
@@ -31,7 +31,7 @@
tasks:
- name: "Load upgrade_vars.yml for supported versions"
ansible.builtin.include_vars:
- file: "{{ playbook_dir }}/../../../common/vars/upgrade_vars.yml"
+ file: "{{ playbook_dir }}/../../common/vars/upgrade_vars.yml"
- name: "Load software_config.json"
ansible.builtin.slurp:
@@ -118,7 +118,7 @@
- name: "Load upgrade_vars.yml"
ansible.builtin.include_vars:
- file: "{{ playbook_dir }}/../../../common/vars/upgrade_vars.yml"
+ file: "{{ playbook_dir }}/../../common/vars/upgrade_vars.yml"
- name: "Extract K8s target version from upgrade configuration"
ansible.builtin.set_fact:
@@ -366,7 +366,7 @@
- name: "Load upgrade_vars.yml"
ansible.builtin.include_vars:
- file: "{{ playbook_dir }}/../../../common/vars/upgrade_vars.yml"
+ file: "{{ playbook_dir }}/../../common/vars/upgrade_vars.yml"
- name: Set k8s_from_version from kube_vip
ansible.builtin.set_fact:
@@ -411,7 +411,7 @@
- name: "Load upgrade_vars.yml"
ansible.builtin.include_vars:
- file: "{{ playbook_dir }}/../../../common/vars/upgrade_vars.yml"
+ file: "{{ playbook_dir }}/../../common/vars/upgrade_vars.yml"
- name: "Get current cluster version"
ansible.builtin.command: /usr/bin/kubectl get nodes -o jsonpath='{.items[*].status.nodeInfo.kubeletVersion}'
@@ -1328,6 +1328,21 @@
ansible.builtin.set_fact:
admin_nic_ip: "{{ pulp_server_ip }}"
+ - name: Load network_spec.yml
+ ansible.builtin.include_vars:
+ file: "{{ input_project_dir }}/network_spec.yml"
+
+ - name: Parse network_spec data
+ ansible.builtin.set_fact:
+ network_data: "{{ network_data | default({}) | combine({item.key: item.value}) }}"
+ with_dict: "{{ Networks }}"
+
+ - name: Set admin network variables
+ ansible.builtin.set_fact:
+ admin_nic_ip: "{{ network_data.admin_network.primary_oim_admin_ip }}"
+ admin_netmask_bits: "{{ network_data.admin_network.netmask_bits }}"
+ admin_nic_cidr: "{{ (network_data.admin_network.subnet + '/' + network_data.admin_network.netmask_bits) | ansible.utils.ipaddr('network/prefix') }}"
+
- name: Set cluster OS version
ansible.builtin.set_fact:
cluster_os_version: "{{ _software_config.cluster_os_version }}"
@@ -1347,7 +1362,7 @@
- name: Load upgrade_vars.yml
ansible.builtin.slurp:
- path: "{{ playbook_dir }}/../../../common/vars/upgrade_vars.yml"
+ path: "{{ playbook_dir }}/../../common/vars/upgrade_vars.yml"
register: _upgrade_config_slurp
- name: Parse upgrade_vars.yml
@@ -1647,7 +1662,7 @@
- name: Load upgrade_vars.yml
ansible.builtin.include_vars:
- file: "{{ playbook_dir }}/../../../common/vars/upgrade_vars.yml"
+ file: "{{ playbook_dir }}/../../common/vars/upgrade_vars.yml"
- name: Set k8s_target_version
ansible.builtin.set_fact:
diff --git a/src/playbooks/upgrade/playbooks/upgrade_oim.yml b/src/playbooks/upgrade/playbooks/upgrade_oim.yml
index 5342650777..65ff749392 100644
--- a/src/playbooks/upgrade/playbooks/upgrade_oim.yml
+++ b/src/playbooks/upgrade/playbooks/upgrade_oim.yml
@@ -15,18 +15,19 @@
# ============================================================================
# upgrade_oim.yml — Internal playbook (imported by upgrade.yml --tags oim)
# ============================================================================
-# Upgrades OIM components: OpenCHAMI containers.
+# Upgrades OIM components: Pulp container and OpenCHAMI containers.
# Prerequisites: prepare_upgrade.yml must have been run first.
# Reads upgrade_manifest.yml and skips if oim already completed.
#
# Flow:
# 1. Pre-flight: read manifest, check idempotency
-# 2. OpenCHAMI container upgrade (pg_dump, deployment-recipes, image pull,
-# ordered restart, DB migration, validation)
-# 3. Mark OIM as completed in manifest
+# 2. Phase 1: Pulp container upgrade (3.80 → 3.113)
+# 3. Phase 2: OpenCHAMI container upgrade (pg_dump, deployment-recipes,
+# image pull, ordered restart, DB migration, validation)
+# 4. Mark OIM as completed in manifest
# ============================================================================
-- name: Upgrade OIM (OpenCHAMI)
+- name: Upgrade OIM (Pulp and OpenCHAMI)
hosts: localhost
connection: local
gather_facts: true
@@ -64,7 +65,55 @@
ansible.builtin.debug:
msg: "[UPGRADE] Component '{{ component_name }}' — status changed to: in-progress"
- # ── Phase 1: OpenCHAMI Container Upgrade (ESpec §4.4) ──────────────
+ # ── Phase 1: Pulp Container Upgrade ─────────────────────────────────
+ # The upgrade_pulp role handles the full lifecycle:
+ # resolve_admin_ip → pre_upgrade_health_check → upgrade_pulp_container → post_upgrade_health_check
+ # It delegates all container operations to the 'oim' host via SSH.
+ # Pulp data is preserved on shared storage during the upgrade.
+ - name: "Phase 1 — Pulp Container Upgrade"
+ block:
+ - name: Upgrade Pulp container
+ ansible.builtin.include_role:
+ name: "{{ playbook_dir }}/../roles/upgrade_pulp"
+
+ - name: Display Pulp upgrade outcome
+ ansible.builtin.debug:
+ msg: >-
+ Pulp Phase 1 result —
+ deployed={{ pulp_deployed | default(false) }},
+ upgrade_needed={{ pulp_upgrade_needed | default(true) }}
+
+ rescue:
+ - name: Re-read upgrade_manifest.yml after Pulp failure
+ ansible.builtin.slurp:
+ src: "{{ manifest_path }}"
+ register: pulp_rescue_raw_manifest
+
+ - name: Parse current manifest state
+ ansible.builtin.set_fact:
+ pulp_rescue_manifest: "{{ pulp_rescue_raw_manifest.content | b64decode | from_yaml }}"
+
+ - name: Pulp upgrade failed — mark component as failed
+ ansible.builtin.copy:
+ content: >-
+ {{ pulp_rescue_manifest | combine({
+ 'component_status': pulp_rescue_manifest.component_status | combine({
+ component_name: 'failed'
+ })
+ }) | to_nice_yaml }}
+ dest: "{{ manifest_path }}"
+ mode: '0644'
+
+ - name: Fail with Pulp upgrade error
+ ansible.builtin.fail:
+ msg: >-
+ Pulp upgrade failed
+ ({{ pulp_rescue_manifest.source_version | default('N/A') }} →
+ {{ pulp_rescue_manifest.target_version | default('N/A') }}).
+ Check logs: podman logs pulp
+ Consider running rollback.
+
+ # ── Phase 2: OpenCHAMI Container Upgrade (ESpec §4.4) ──────────────
# The upgrade_openchami role handles the full lifecycle:
# pre_upgrade_health_check → upgrade_openchami_containers → post_upgrade_health_check
# It delegates all container operations to the 'oim' host via SSH.
@@ -72,7 +121,7 @@
# which lands in the rescue block below.
# On skip (not deployed / already at target), the role completes normally
# and sets openchami_deployed / upgrade_needed facts accordingly.
- - name: "Phase 1 — OpenCHAMI Container Upgrade"
+ - name: "Phase 2 — OpenCHAMI Container Upgrade"
block:
- name: Upgrade OpenCHAMI containers and services
ansible.builtin.include_role:
@@ -81,7 +130,7 @@
- name: Display OpenCHAMI upgrade outcome
ansible.builtin.debug:
msg: >-
- OpenCHAMI Phase 1 result —
+ OpenCHAMI Phase 2 result —
deployed={{ openchami_deployed | default(false) }},
upgrade_needed={{ upgrade_needed | default(true) }},
failed={{ openchami_upgrade_failed | default(false) }}
diff --git a/src/playbooks/upgrade/roles/upgrade_k8s/defaults/main.yml b/src/playbooks/upgrade/roles/upgrade_k8s/defaults/main.yml
index 1d3c08145c..5279782980 100644
--- a/src/playbooks/upgrade/roles/upgrade_k8s/defaults/main.yml
+++ b/src/playbooks/upgrade/roles/upgrade_k8s/defaults/main.yml
@@ -20,8 +20,10 @@ oim_data_path: "/opt/omnia/.data"
oim_provision_path: "/opt/omnia/provision"
tmp_path: "/tmp"
cluster_os_version: "10.0"
-admin_nic_ip: "{{ hostvars['localhost']['admin_nic_ip'] | default('127.0.0.1') }}"
-admin_nic_cidr: "{{ hostvars['localhost']['admin_nic_cidr'] | default('10.0.0.0/24') }}"
+# admin_nic_ip and admin_nic_cidr are calculated from network_spec.yml in upgrade_k8s.yml playbook
+# Do not use default values here to avoid misconfiguration
+admin_nic_ip: "{{ hostvars['localhost']['admin_nic_ip'] }}"
+admin_nic_cidr: "{{ hostvars['localhost']['admin_nic_cidr'] }}"
# MinIO S3 for boot images
minio_ip: "{{ hostvars['localhost']['minio_ip'] | default(admin_nic_ip) }}"
diff --git a/src/playbooks/upgrade/roles/upgrade_k8s/tasks/post_validation.yml b/src/playbooks/upgrade/roles/upgrade_k8s/tasks/post_validation.yml
index 74d1e00cef..78016755bb 100644
--- a/src/playbooks/upgrade/roles/upgrade_k8s/tasks/post_validation.yml
+++ b/src/playbooks/upgrade/roles/upgrade_k8s/tasks/post_validation.yml
@@ -185,6 +185,85 @@
{{ dns_test.stderr }}
when: "'Server:' not in dns_test.stdout or dns_test.rc != 0"
+# ── Isilon CSI restart (if namespace exists) ──────────────────────
+- name: Check if isilon namespace exists
+ delegate_to: "{{ kube_vip }}"
+ ansible.builtin.command:
+ cmd: kubectl get namespace isilon
+ register: isilon_namespace_check
+ changed_when: false
+ failed_when: false
+
+- name: Restart isilon-controller deployment
+ delegate_to: "{{ kube_vip }}"
+ ansible.builtin.command:
+ cmd: kubectl rollout restart deployment isilon-controller -n isilon
+ register: isilon_controller_restart
+ changed_when: true
+ when: isilon_namespace_check.rc == 0
+
+- name: Restart isilon-node daemonset
+ delegate_to: "{{ kube_vip }}"
+ ansible.builtin.command:
+ cmd: kubectl rollout restart daemonset isilon-node -n isilon
+ register: isilon_node_restart
+ changed_when: true
+ when: isilon_namespace_check.rc == 0
+
+- name: Wait for isilon-controller deployment to be ready
+ delegate_to: "{{ kube_vip }}"
+ ansible.builtin.command:
+ cmd: kubectl rollout status deployment isilon-controller -n isilon --timeout=300s
+ register: isilon_controller_status
+ changed_when: false
+ failed_when: false
+ when: isilon_namespace_check.rc == 0
+
+- name: Wait for isilon-node daemonset to be ready
+ delegate_to: "{{ kube_vip }}"
+ ansible.builtin.command:
+ cmd: kubectl rollout status daemonset isilon-node -n isilon --timeout=300s
+ register: isilon_node_status
+ changed_when: false
+ failed_when: false
+ when: isilon_namespace_check.rc == 0
+
+- name: Verify all isilon pods are Running
+ delegate_to: "{{ kube_vip }}"
+ ansible.builtin.shell:
+ cmd: >-
+ set -o pipefail &&
+ kubectl get pods -n isilon --no-headers
+ --field-selector status.phase!=Running,status.phase!=Succeeded
+ 2>/dev/null | head -20
+ args:
+ executable: /bin/bash
+ register: unhealthy_isilon_pods
+ changed_when: false
+ failed_when: false
+ retries: 30
+ delay: 10
+ until: unhealthy_isilon_pods.stdout | length == 0
+ when: isilon_namespace_check.rc == 0
+
+- name: Fail if isilon pods are not Running
+ ansible.builtin.fail:
+ msg: >-
+ Post-validation failed: Some isilon pods are not Running after restart.
+ {{ unhealthy_isilon_pods.stdout }}
+ when:
+ - isilon_namespace_check.rc == 0
+ - unhealthy_isilon_pods.stdout | length > 0
+
+- name: Display isilon restart status
+ ansible.builtin.debug:
+ msg: >-
+ Isilon CSI components restarted successfully:
+ - isilon-controller deployment restarted and ready
+ - isilon-node daemonset restarted and ready
+ - All isilon pods are Running
+ when: isilon_namespace_check.rc == 0
+
- name: Display post-validation summary
ansible.builtin.debug:
msg: >-
@@ -196,4 +275,5 @@
MetalLB pods: Running
API server: reachable
DNS: working
- Cluster upgrade from {{ k8s_from_version }} to {{ k8s_target_version }} successful.
+ {% if isilon_namespace_check.rc == 0 %}Isilon CSI: restarted
+ {% endif %}Cluster upgrade from {{ k8s_from_version }} to {{ k8s_target_version }} successful.
diff --git a/src/playbooks/upgrade/roles/upgrade_k8s/tasks/step_kubelet_restart.yml b/src/playbooks/upgrade/roles/upgrade_k8s/tasks/step_kubelet_restart.yml
index 8cbea6d96a..55a1475410 100644
--- a/src/playbooks/upgrade/roles/upgrade_k8s/tasks/step_kubelet_restart.yml
+++ b/src/playbooks/upgrade/roles/upgrade_k8s/tasks/step_kubelet_restart.yml
@@ -26,21 +26,17 @@
daemon_reload: true
delegate_to: "{{ current_node_name }}"
-# Part 4: Restart crio service
-- name: Restart crio on {{ current_node_name }}
- ansible.builtin.systemd:
- name: crio
- state: restarted
- delegate_to: "{{ current_node_name }}"
-
-# Part 5: Restart kubelet service
+# Part 4: Restart kubelet service
+# Note: CRI-O was already restarted during the crio_install step.
+# Restarting it again here is redundant and causes unnecessary pod disruption.
+# Only kubelet needs to be restarted to apply the config.yaml and feature gate changes.
- name: Restart kubelet on {{ current_node_name }}
ansible.builtin.systemd:
name: kubelet
state: restarted
delegate_to: "{{ current_node_name }}"
-# Part 6: Wait for node to become Ready with correct version
+# Part 5: Wait for node to become Ready with correct version
- name: Wait for node to become Ready
ansible.builtin.command: >-
kubectl get node {{ node_ip }}
diff --git a/src/playbooks/upgrade/roles/upgrade_pulp/tasks/backup_pulp_data.yml b/src/playbooks/upgrade/roles/upgrade_pulp/tasks/backup_pulp_data.yml
new file mode 100644
index 0000000000..2b19607817
--- /dev/null
+++ b/src/playbooks/upgrade/roles/upgrade_pulp/tasks/backup_pulp_data.yml
@@ -0,0 +1,37 @@
+# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+
+# ===========================================================================
+# Backup Pulp PostgreSQL Data Before Upgrade
+#
+# PostgreSQL data must be backed up before upgrade because:
+# - Pulp 3.80 uses PostgreSQL 12/13
+# - Pulp 3.113 uses PostgreSQL 16
+# - PostgreSQL cannot downgrade data files between major versions
+#
+# Without this backup, rollback is not possible.
+# ===========================================================================
+
+- name: Backup Pulp PostgreSQL data
+ pulp_pgsql_backup_restore:
+ action: backup
+ src_path: "{{ pulp_pgsql_path }}"
+ dest_path: "{{ pulp_pgsql_backup_path }}"
+ backup_dir: "{{ pulp_backup_dir }}"
+ register: pulp_backup_result
+
+- name: Display backup result
+ ansible.builtin.debug:
+ msg: "{{ pulp_backup_result.messages | default(['Backup task completed']) }}"
diff --git a/src/playbooks/upgrade/roles/upgrade_pulp/tasks/cleanup_after_upgrade.yml b/src/playbooks/upgrade/roles/upgrade_pulp/tasks/cleanup_after_upgrade.yml
new file mode 100644
index 0000000000..052227a6fd
--- /dev/null
+++ b/src/playbooks/upgrade/roles/upgrade_pulp/tasks/cleanup_after_upgrade.yml
@@ -0,0 +1,58 @@
+# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+
+# ===========================================================================
+# Cleanup After Pulp Upgrade
+#
+# - Runs Pulp orphan cleanup to reclaim disk space
+# via pulp_fs_orphan_cleanup module (reads creds from cli.toml)
+# - Reports disk usage
+# NOTE: Backup directory is preserved for rollback.
+# ===========================================================================
+
+# Runs inside omnia_core where cli.toml and media directory are accessible.
+# No password arguments — credentials read from /root/.config/pulp/cli.toml.
+- name: Run Pulp orphan cleanup after upgrade
+ pulp_fs_orphan_cleanup:
+ media_dir: "{{ pulp_data_base_path }}/pulp_storage/media"
+ trigger_db_orphan_cleanup: true
+ register: orphan_result
+ failed_when: false
+
+- name: Display orphan cleanup result
+ ansible.builtin.debug:
+ msg:
+ - "DB orphan cleanup: {{ orphan_result.db_orphan_cleanup_status | default('N/A') }}"
+ - "Orphans removed: {{ orphan_result.removed_count | default(0) }}"
+ - "Space freed: {{ orphan_result.freed_mb | default(0) }} MB"
+
+- name: Report disk usage after upgrade
+ ansible.builtin.shell: |
+ set -o pipefail
+ echo "Pulp storage: $(du -sh {{ oim_host_pulp_data_base }}/pulp_storage 2>/dev/null | cut -f1)"
+ echo "PostgreSQL: $(du -sh {{ oim_host_pgsql_path }} 2>/dev/null | cut -f1)"
+ echo "Backup: $(du -sh {{ oim_host_backup_dir }} 2>/dev/null | cut -f1)"
+ args:
+ executable: /bin/bash
+ delegate_to: oim
+ delegate_facts: true
+ connection: ssh
+ register: disk_usage_result
+ changed_when: false
+ failed_when: false
+
+- name: Display disk usage
+ ansible.builtin.debug:
+ msg: "{{ disk_usage_result.stdout_lines | default([]) }}"
diff --git a/src/playbooks/upgrade/roles/upgrade_pulp/tasks/main.yml b/src/playbooks/upgrade/roles/upgrade_pulp/tasks/main.yml
new file mode 100644
index 0000000000..4ec5803ca2
--- /dev/null
+++ b/src/playbooks/upgrade/roles/upgrade_pulp/tasks/main.yml
@@ -0,0 +1,87 @@
+# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+
+# ===========================================================================
+# Upgrade Pulp Role - Main Entry Point
+#
+# This role upgrades the Pulp container from 3.80 to 3.113.
+# Pulp stores all data in persistent volumes mounted from shared storage,
+# so data is preserved during the upgrade.
+#
+# IMPORTANT: PostgreSQL data must be backed up before upgrade because:
+# - Pulp 3.80 uses PostgreSQL 12/13
+# - Pulp 3.113 uses PostgreSQL 16
+# - PostgreSQL cannot downgrade data files between major versions
+# Without the backup, rollback is not possible.
+#
+# Flow:
+# 1. Resolve admin NIC IP for health check endpoints
+# 2. Pre-upgrade health check (verify Pulp is deployed and accessible)
+# 3. Backup PostgreSQL data (required for rollback)
+# 4. Pull the new Pulp image (3.113)
+# 5. Stop the Pulp container
+# 6. Update the quadlet file with the new image tag
+# 7. Reload systemd daemon and restart container
+# 8. Run database migrations
+# 9. Post-upgrade health check
+# 10. Cleanup (orphan artifacts)
+# ===========================================================================
+
+- name: Read oim_metadata.yml for shared storage path
+ ansible.builtin.slurp:
+ src: "{{ oim_metadata_path }}"
+ register: _pulp_oim_metadata_raw
+
+- name: Set OIM host-side paths from metadata
+ ansible.builtin.set_fact:
+ oim_shared_path: "{{ (_pulp_oim_metadata_raw.content | b64decode | from_yaml).oim_shared_path | regex_replace('/$', '') }}"
+
+- name: Derive OIM host-side Pulp paths
+ ansible.builtin.set_fact:
+ oim_host_pulp_data_base: "{{ oim_shared_path }}/omnia/pulp/settings"
+ oim_host_pgsql_path: "{{ oim_shared_path }}/omnia/pulp/settings/pgsql"
+ oim_host_backup_dir: "{{ oim_shared_path }}/omnia/backups/upgrade/version_2.1.0.0/pulp"
+ oim_host_pgsql_backup: "{{ oim_shared_path }}/omnia/backups/upgrade/version_2.1.0.0/pulp/pgsql"
+
+- name: Resolve admin NIC IP for Pulp API endpoints
+ ansible.builtin.include_tasks: resolve_admin_ip.yml
+
+- name: Pre-upgrade health check
+ ansible.builtin.include_tasks: pre_upgrade_health_check.yml
+
+- name: Backup Pulp PostgreSQL data before upgrade
+ ansible.builtin.include_tasks: backup_pulp_data.yml
+ when:
+ - pulp_deployed | default(false) | bool
+ - pulp_upgrade_needed | default(true) | bool
+
+- name: Execute Pulp upgrade
+ ansible.builtin.include_tasks: upgrade_pulp_container.yml
+ when:
+ - pulp_deployed | default(false) | bool
+ - pulp_upgrade_needed | default(true) | bool
+
+- name: Post-upgrade health check
+ ansible.builtin.include_tasks: post_upgrade_health_check.yml
+ when:
+ - pulp_deployed | default(false) | bool
+ - pulp_upgrade_needed | default(true) | bool
+
+- name: Post-upgrade cleanup (orphan artifacts)
+ ansible.builtin.include_tasks: cleanup_after_upgrade.yml
+ when:
+ - pulp_deployed | default(false) | bool
+ - pulp_upgrade_needed | default(true) | bool
+ - pulp_upgrade_success | default(false) | bool
diff --git a/src/playbooks/upgrade/roles/upgrade_pulp/tasks/post_upgrade_health_check.yml b/src/playbooks/upgrade/roles/upgrade_pulp/tasks/post_upgrade_health_check.yml
new file mode 100644
index 0000000000..83f99a47d6
--- /dev/null
+++ b/src/playbooks/upgrade/roles/upgrade_pulp/tasks/post_upgrade_health_check.yml
@@ -0,0 +1,75 @@
+# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+
+# ===========================================================================
+# Pulp Post-Upgrade Health Check
+#
+# Verifies:
+# 1. Pulp API is accessible and responding
+# 2. Pulp container is running with the new image
+# Sets pulp_upgrade_success flag for cleanup tasks.
+# ===========================================================================
+
+- name: Check Pulp API status after upgrade
+ ansible.builtin.uri:
+ url: "{{ pulp_status_url }}"
+ method: GET
+ validate_certs: false
+ status_code: [200]
+ register: pulp_post_health
+ retries: "{{ pulp_health_retries }}"
+ delay: "{{ pulp_health_delay }}"
+ until: pulp_post_health.status == 200
+ delegate_to: oim
+ delegate_facts: true
+ connection: ssh
+
+- name: Get Pulp container info after upgrade
+ containers.podman.podman_container_info:
+ name: "{{ pulp_container_name }}"
+ register: pulp_post_info
+ delegate_to: oim
+ delegate_facts: true
+ connection: ssh
+
+- name: Verify Pulp is running with new image
+ ansible.builtin.assert:
+ that:
+ - pulp_post_info.containers | length > 0
+ - pulp_post_info.containers[0].State.Status == 'running'
+ - pulp_target_image in pulp_post_info.containers[0].ImageName
+ fail_msg: |
+ Pulp upgrade verification failed.
+ Expected image: {{ pulp_target_image }}
+ Actual image: {{ pulp_post_info.containers[0].ImageName | default('unknown') }}
+ Container status: {{ pulp_post_info.containers[0].State.Status | default('unknown') }}
+ success_msg: "Pulp successfully upgraded to {{ pulp_target_image }}"
+
+- name: Set upgrade success flag
+ ansible.builtin.set_fact:
+ pulp_upgrade_success: true
+
+- name: Display Pulp upgrade summary
+ ansible.builtin.debug:
+ msg:
+ - "════════════════════════════════════════════════════════════"
+ - " PULP UPGRADE COMPLETED SUCCESSFULLY"
+ - "════════════════════════════════════════════════════════════"
+ - " Previous image: {{ pulp_current_image | default('unknown') }}"
+ - " New image: {{ pulp_target_image }}"
+ - " Container: {{ pulp_container_name }}"
+ - " Status: running"
+ - " API endpoint: {{ pulp_status_url }}"
+ - "════════════════════════════════════════════════════════════"
diff --git a/src/playbooks/upgrade/roles/upgrade_pulp/tasks/pre_upgrade_health_check.yml b/src/playbooks/upgrade/roles/upgrade_pulp/tasks/pre_upgrade_health_check.yml
new file mode 100644
index 0000000000..ca141b0396
--- /dev/null
+++ b/src/playbooks/upgrade/roles/upgrade_pulp/tasks/pre_upgrade_health_check.yml
@@ -0,0 +1,100 @@
+# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+
+# ===========================================================================
+# Pulp Pre-Upgrade Health Check
+#
+# Verifies:
+# 1. Pulp container exists and is deployed
+# 2. Current Pulp version (to determine if upgrade is needed)
+# 3. Pulp API is accessible
+# ===========================================================================
+
+- name: Check if Pulp container exists
+ ansible.builtin.command: podman container exists {{ pulp_container_name }}
+ register: pulp_exists_check
+ changed_when: false
+ failed_when: false
+ delegate_to: oim
+ delegate_facts: true
+ connection: ssh
+
+- name: Set Pulp deployment status
+ ansible.builtin.set_fact:
+ pulp_deployed: "{{ pulp_exists_check.rc == 0 }}"
+
+- name: Display Pulp not deployed message
+ ansible.builtin.debug:
+ msg: "{{ upgrade_messages.pulp.not_deployed }}"
+ when: not pulp_deployed
+
+- name: Get current Pulp container info
+ containers.podman.podman_container_info:
+ name: "{{ pulp_container_name }}"
+ register: pulp_pre_info
+ delegate_to: oim
+ delegate_facts: true
+ connection: ssh
+ when: pulp_deployed
+
+- name: Extract current Pulp image version
+ ansible.builtin.set_fact:
+ pulp_current_image: "{{ pulp_pre_info.containers[0].ImageName | default('unknown') }}"
+ pulp_current_status: "{{ pulp_pre_info.containers[0].State.Status | default('unknown') }}"
+ when:
+ - pulp_deployed
+ - pulp_pre_info.containers | length > 0
+
+- name: Display current Pulp version
+ ansible.builtin.debug:
+ msg: "Current Pulp image: {{ pulp_current_image | default('not found') }}, Status: {{ pulp_current_status | default('unknown') }}"
+ when: pulp_deployed
+
+- name: Check if Pulp is already at target version
+ ansible.builtin.set_fact:
+ pulp_upgrade_needed: "{{ (pulp_target_image | default('')) not in ((pulp_current_image | default('')) | string) }}"
+ when: pulp_deployed
+
+- name: Display skip message if already at target version
+ ansible.builtin.debug:
+ msg: "{{ upgrade_messages.pulp.already_upgraded }}"
+ when:
+ - pulp_deployed
+ - not (pulp_upgrade_needed | default(true))
+
+- name: Check Pulp API status before upgrade
+ ansible.builtin.uri:
+ url: "{{ pulp_status_url }}"
+ method: GET
+ validate_certs: false
+ status_code: [200]
+ register: pulp_pre_health
+ retries: 3
+ delay: 5
+ until: pulp_pre_health.status == 200
+ delegate_to: oim
+ delegate_facts: true
+ connection: ssh
+ failed_when: false
+ when:
+ - pulp_deployed
+ - pulp_upgrade_needed | default(true)
+
+- name: Display Pulp pre-upgrade health status
+ ansible.builtin.debug:
+ msg: "Pulp API status: {{ 'healthy (HTTP 200)' if pulp_pre_health.status | default(0) == 200 else 'not accessible (will attempt upgrade anyway)' }}"
+ when:
+ - pulp_deployed
+ - pulp_upgrade_needed | default(true)
diff --git a/src/playbooks/upgrade/roles/upgrade_pulp/tasks/resolve_admin_ip.yml b/src/playbooks/upgrade/roles/upgrade_pulp/tasks/resolve_admin_ip.yml
new file mode 100644
index 0000000000..c76460af9c
--- /dev/null
+++ b/src/playbooks/upgrade/roles/upgrade_pulp/tasks/resolve_admin_ip.yml
@@ -0,0 +1,95 @@
+# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+
+# ============================================================================
+# resolve_admin_ip.yml — Resolve Admin NIC IP for Pulp API Health Checks
+# ============================================================================
+# Resolves the admin_nic_ip needed for Pulp API health check endpoints.
+#
+# Resolution order:
+# 1. Primary: Extract from local_repo_access.yml (same as upgrade_k8s)
+# 2. Fallback: Extract from network_spec.yml (same as prepare_oim)
+# 3. Last resort: Use localhost
+#
+# Sets fact:
+# admin_nic_ip — IP address for Pulp API health checks
+# ============================================================================
+
+- name: Resolve admin NIC IP for Pulp health checks
+ block:
+ # Primary method: Extract from local_repo_access.yml (same as upgrade_k8s)
+ - name: Check if local_repo_access.yml exists
+ ansible.builtin.stat:
+ path: "{{ local_repo_access_path }}"
+ register: local_repo_access_stat
+
+ - name: Load local_repo_access.yml
+ ansible.builtin.slurp:
+ src: "{{ local_repo_access_path }}"
+ register: local_repo_access_raw
+ when: local_repo_access_stat.stat.exists
+
+ - name: Parse local_repo_access.yml
+ ansible.builtin.set_fact:
+ _local_repo_access: "{{ local_repo_access_raw.content | b64decode | from_yaml }}"
+ when: local_repo_access_stat.stat.exists
+
+ - name: Set admin_nic_ip from local_repo_access
+ ansible.builtin.set_fact:
+ admin_nic_ip: "{{ _local_repo_access.offline_tarball_path | regex_replace('^(https?)://([^:]+):.*', '\\2') }}"
+ when:
+ - local_repo_access_stat.stat.exists
+ - _local_repo_access.offline_tarball_path is defined
+
+ # Fallback method: Extract from network_spec.yml
+ - name: Check if network_spec.yml exists
+ ansible.builtin.stat:
+ path: "{{ network_spec_path }}"
+ register: network_spec_stat
+ when: admin_nic_ip is not defined
+
+ - name: Load network_spec.yml as fallback
+ ansible.builtin.include_vars:
+ file: "{{ network_spec_path }}"
+ when:
+ - admin_nic_ip is not defined
+ - network_spec_stat.stat.exists | default(false)
+
+ # Networks is a list in network_spec.yml, parse it to extract admin_network
+ - name: Parse network_spec data into network_data
+ ansible.builtin.set_fact:
+ network_data: "{{ network_data | default({}) | combine({item.keys() | first: item.values() | first}) }}"
+ loop: "{{ Networks }}"
+ when:
+ - admin_nic_ip is not defined
+ - Networks is defined
+
+ - name: Set admin_nic_ip from network_spec
+ ansible.builtin.set_fact:
+ admin_nic_ip: "{{ network_data.admin_network.primary_oim_admin_ip }}"
+ when:
+ - admin_nic_ip is not defined
+ - network_data is defined
+ - network_data.admin_network is defined
+ - network_data.admin_network.primary_oim_admin_ip is defined
+
+ rescue:
+ - name: Fallback to localhost for admin_nic_ip
+ ansible.builtin.set_fact:
+ admin_nic_ip: "localhost"
+
+- name: Display resolved admin_nic_ip
+ ansible.builtin.debug:
+ msg: "Pulp health checks will use admin_nic_ip: {{ admin_nic_ip }}"
diff --git a/src/playbooks/upgrade/roles/upgrade_pulp/tasks/upgrade_pulp_container.yml b/src/playbooks/upgrade/roles/upgrade_pulp/tasks/upgrade_pulp_container.yml
new file mode 100644
index 0000000000..937558189f
--- /dev/null
+++ b/src/playbooks/upgrade/roles/upgrade_pulp/tasks/upgrade_pulp_container.yml
@@ -0,0 +1,108 @@
+# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+
+# ===========================================================================
+# Upgrade Pulp Container (3.80 → 3.113)
+#
+# Steps:
+# 1. Pull new Pulp image
+# 2. Stop and remove old container
+# 3. Update quadlet file with new image tag
+# 4. Reload systemd and start container
+# 5. Wait for initialization and run migrations
+# ===========================================================================
+
+# --- 1. Pull new Pulp image ---
+- name: Pull new Pulp image ({{ pulp_target_image }})
+ ansible.builtin.command: podman pull {{ pulp_target_image }}
+ register: pulp_pull_result
+ retries: "{{ pull_image_retries }}"
+ delay: "{{ pull_image_delay }}"
+ until: pulp_pull_result.rc == 0
+ changed_when: pulp_pull_result.rc == 0
+ delegate_to: oim
+ delegate_facts: true
+ connection: ssh
+
+# --- 2. Stop and remove old container ---
+- name: Stop Pulp service
+ ansible.builtin.systemd:
+ name: "{{ pulp_container_name }}"
+ state: stopped
+ delegate_to: oim
+ delegate_facts: true
+ connection: ssh
+
+- name: Remove old Pulp container
+ ansible.builtin.command: podman rm -f {{ pulp_container_name }}
+ changed_when: true
+ failed_when: false
+ delegate_to: oim
+ delegate_facts: true
+ connection: ssh
+
+# --- 3. Update quadlet file with new image ---
+- name: Update Pulp image in quadlet file
+ ansible.builtin.lineinfile:
+ path: "{{ pulp_quadlet_path }}"
+ regexp: '^Image=.*'
+ line: "Image={{ pulp_target_image }}"
+ state: present
+ delegate_to: oim
+ delegate_facts: true
+ connection: ssh
+
+# --- 4. Reload systemd and start container ---
+- name: Reload systemd daemon
+ ansible.builtin.systemd:
+ daemon_reload: true
+ delegate_to: oim
+ delegate_facts: true
+ connection: ssh
+
+- name: Start Pulp service
+ ansible.builtin.systemd:
+ name: "{{ pulp_container_name }}"
+ state: started
+ enabled: true
+ register: pulp_start_result
+ retries: "{{ pulp_startup_retries }}"
+ delay: "{{ pulp_startup_delay }}"
+ until: pulp_start_result is succeeded
+ delegate_to: oim
+ delegate_facts: true
+ connection: ssh
+
+# --- 5. Wait for initialization and run migrations ---
+- name: Wait for Pulp services to initialize
+ ansible.builtin.pause:
+ seconds: "{{ pulp_init_wait }}"
+
+- name: Run Pulp database migrations
+ containers.podman.podman_container_exec:
+ name: "{{ pulp_container_name }}"
+ command: pulpcore-manager migrate --noinput
+ register: pulp_migrate_result
+ retries: 3
+ delay: 10
+ until: pulp_migrate_result.rc == 0
+ delegate_to: oim
+ delegate_facts: true
+ connection: ssh
+ failed_when: false
+
+- name: Display migration result
+ ansible.builtin.debug:
+ msg: "Database migration: {{ 'completed' if pulp_migrate_result.rc | default(1) == 0 else 'skipped or not required' }}"
diff --git a/src/playbooks/upgrade/roles/upgrade_pulp/vars/main.yml b/src/playbooks/upgrade/roles/upgrade_pulp/vars/main.yml
new file mode 100644
index 0000000000..722272dde5
--- /dev/null
+++ b/src/playbooks/upgrade/roles/upgrade_pulp/vars/main.yml
@@ -0,0 +1,105 @@
+# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+
+# ===========================================================================
+# Pulp Container Upgrade Configuration (3.80 → 3.113)
+# ===========================================================================
+
+# File permissions
+dir_permissions_755: "0755"
+file_permissions_644: "0644"
+
+# Pulp container settings
+pulp_container_name: "pulp"
+pulp_target_tag: "3.113"
+pulp_target_image: "docker.io/pulp/pulp:{{ pulp_target_tag }}"
+
+# Pulp quadlet file path
+pulp_quadlet_path: "/etc/containers/systemd/{{ pulp_container_name }}.container"
+
+# OIM metadata (for resolving host-side shared storage paths)
+oim_metadata_path: "/opt/omnia/.data/oim_metadata.yml"
+
+# Paths for resolving admin_nic_ip
+local_repo_access_path: "/opt/omnia/provision/local_repo_access.yml"
+network_spec_path: "{{ input_project_dir | default('/opt/omnia/input') }}/network_spec.yml"
+
+# Pulp API endpoint for health checks
+pulp_protocol: "https"
+pulp_port: "2225"
+pulp_status_url: "{{ pulp_protocol }}://{{ admin_nic_ip | default('localhost') }}:{{ pulp_port }}/pulp/api/v3/status/"
+
+# Image pull settings
+pull_image_retries: 5
+pull_image_delay: 10
+
+# Pulp startup and health check settings
+pulp_startup_retries: 10
+pulp_startup_delay: 10
+pulp_init_wait: 30
+pulp_health_retries: 12
+pulp_health_delay: 10
+
+# Pulp shared storage paths (OIM container paths)
+# These paths are inside the OIM container and mounted as volumes in the Pulp container
+pulp_data_base_path: "/opt/omnia/pulp/settings"
+pulp_pgsql_path: "{{ pulp_data_base_path }}/pgsql"
+
+# Backup settings
+# PostgreSQL data must be backed up before upgrade because:
+# - Pulp 3.80 uses PostgreSQL 12/13
+# - Pulp 3.113 uses PostgreSQL 16
+# - PostgreSQL cannot downgrade data files between major versions
+pulp_backup_dir: "/opt/omnia/backups/upgrade/version_2.1.0.0/pulp"
+pulp_pgsql_backup_path: "{{ pulp_backup_dir }}/pgsql"
+
+# Upgrade messages
+upgrade_messages:
+ pulp:
+ not_deployed: |
+ Pulp container is not deployed on the OIM. Skipping Pulp upgrade.
+ If you need to deploy Pulp, run: ansible-playbook prepare_oim/prepare_oim.yml
+ already_upgraded: |
+ Pulp is already at target version ({{ pulp_target_image }}). Skipping upgrade.
+ upgrade_success: |
+ ════════════════════════════════════════════════════════════
+ PULP UPGRADE COMPLETED SUCCESSFULLY
+ ════════════════════════════════════════════════════════════
+ Previous image: {{ pulp_current_image | default('unknown') }}
+ New image: {{ pulp_target_image }}
+ Container: {{ pulp_container_name }}
+ Status: running
+ API endpoint: {{ pulp_status_url }}
+ ════════════════════════════════════════════════════════════
+ upgrade_failure: |
+ ════════════════════════════════════════════════════════════
+ PULP UPGRADE FAILED
+ ════════════════════════════════════════════════════════════
+ Target image: {{ pulp_target_image }}
+
+ Please check:
+ - Pulp container logs: podman logs {{ pulp_container_name }}
+ - Pulp service status: systemctl status {{ pulp_container_name }}
+ - Shared storage accessibility
+ - Network connectivity to container registry
+ ════════════════════════════════════════════════════════════
+ pre_check_failure: |
+ Pulp pre-upgrade health check failed.
+ The Pulp API is not accessible at {{ pulp_status_url }}.
+ Please verify Pulp is running before attempting upgrade.
+ post_check_failure: |
+ Pulp post-upgrade health check failed.
+ The upgraded Pulp container may not be functioning correctly.
+ Check container logs: podman logs {{ pulp_container_name }}
diff --git a/src/playbooks/upgrade/roles/upgrade_telemetry/files/migrate_strimzi_crds.sh b/src/playbooks/upgrade/roles/upgrade_telemetry/files/migrate_strimzi_crds.sh
new file mode 100644
index 0000000000..4ecf229992
--- /dev/null
+++ b/src/playbooks/upgrade/roles/upgrade_telemetry/files/migrate_strimzi_crds.sh
@@ -0,0 +1,117 @@
+#!/bin/bash
+# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# migrate_strimzi_crds.sh — Strimzi CRD major version migration
+#
+# Handles the upgrade from Strimzi 0.x (v1beta2) to 1.x (v1-only).
+# Strimzi 1.0.x completely dropped the v1beta2 API. Kubernetes
+# cannot remove a served version when objects stored in that version
+# still exist in etcd. This script:
+#
+# 1. Detects whether migration is needed
+# 2. Temporarily re-enables v1beta2 on CRDs so stuck CRs are readable
+# 3. Deletes existing Kafka CRs (they will be recreated by telemetry.sh)
+# 4. Deletes old PVCs (new cluster ID makes old data incompatible)
+# 5. Removes CRDs (handles stuck cleanup finalizers)
+#
+# telemetry.sh then recreates CRDs + CRs from the new chart.
+# This script is fully idempotent — it is a no-op when CRDs are
+# already healthy, absent, or running v1 without issues.
+#
+# Usage: bash migrate_strimzi_crds.sh
+# Exit codes: 0 = success or no migration needed
+
+set -euo pipefail
+
+NS="${1:-telemetry}"
+
+# ── Phase 1: Detect ─────────────────────────────────────────────
+needs_migration=false
+
+# Check if any Strimzi CRD still lists v1beta2 in storedVersions
+for crd in $(kubectl get crd -o name 2>/dev/null | grep -E '\.kafka\.strimzi\.io|\.core\.strimzi\.io'); do
+ if kubectl get "$crd" -o jsonpath='{.status.storedVersions}' 2>/dev/null | grep -q 'v1beta2'; then
+ echo "[MIGRATE] $crd has v1beta2 in storedVersions"
+ needs_migration=true
+ break
+ fi
+done
+
+# Check if CRs are stuck (v1-only CRDs but objects stored as v1beta2)
+if [ "$needs_migration" = "false" ] && kubectl get crd kafkas.kafka.strimzi.io >/dev/null 2>&1; then
+ if kubectl get kafka -n "$NS" 2>&1 | grep -q 'convert CR from an invalid group/version'; then
+ echo "[MIGRATE] CRs stuck — conversion error detected"
+ needs_migration=true
+ fi
+fi
+
+if [ "$needs_migration" = "false" ]; then
+ echo "[MIGRATE] No Strimzi CRD migration needed."
+ exit 0
+fi
+
+echo "[MIGRATE] Starting Strimzi CRD migration (v1beta2 → v1)..."
+
+# ── Phase 2: Make stuck CRs readable ────────────────────────────
+STRIMZI_CRDS=$(kubectl get crd -o name 2>/dev/null \
+ | grep -E '\.kafka\.strimzi\.io|\.core\.strimzi\.io' \
+ | sed 's|customresourcedefinition.apiextensions.k8s.io/||')
+
+if [ -n "$STRIMZI_CRDS" ]; then
+ echo "[MIGRATE] Temporarily adding v1beta2 to CRDs..."
+ for crd in $STRIMZI_CRDS; do
+ kubectl get crd "$crd" -o json 2>/dev/null \
+ | jq '.spec.versions += [(.spec.versions[0] | .name = "v1beta2" | .served = true | .storage = false)]' \
+ | kubectl apply -f - --server-side --force-conflicts >/dev/null 2>&1 || true
+ done
+fi
+
+# ── Phase 3: Delete existing CRs ────────────────────────────────
+echo "[MIGRATE] Deleting existing Kafka CRs..."
+for kind in kafka kafkanodepool kafkabridge kafkatopic kafkauser strimzipodset; do
+ for item in $(kubectl get "$kind" -n "$NS" -o name 2>/dev/null); do
+ kubectl patch "$item" -n "$NS" --type=merge \
+ -p '{"metadata":{"finalizers":[]}}' 2>/dev/null || true
+ kubectl delete "$item" -n "$NS" --wait=false 2>/dev/null || true
+ done
+done
+sleep 5
+
+# ── Phase 4: Delete old Kafka PVCs ───────────────────────────────
+echo "[MIGRATE] Deleting old Kafka PVCs (new cluster ID makes old data incompatible)..."
+kubectl delete pvc -n "$NS" -l strimzi.io/cluster=kafka --wait=false 2>/dev/null || true
+
+# ── Phase 5: Delete cluster-id secret (operator will regenerate) ─
+kubectl delete secret kafka-cluster-id -n "$NS" 2>/dev/null || true
+
+# ── Phase 6: Delete CRDs ────────────────────────────────────────
+if [ -n "$STRIMZI_CRDS" ]; then
+ echo "[MIGRATE] Deleting Strimzi CRDs..."
+ kubectl delete crd $STRIMZI_CRDS --wait=false --timeout=30s 2>&1 || true
+ sleep 5
+ # Remove cleanup finalizers from any CRDs stuck in Terminating
+ for crd in $(kubectl get crd -o name 2>/dev/null | grep -E '\.strimzi\.io'); do
+ kubectl patch "$crd" --type=merge \
+ -p '{"metadata":{"finalizers":[]}}' 2>/dev/null || true
+ done
+ # Wait for CRDs to fully disappear
+ for i in $(seq 1 24); do
+ remaining=$(kubectl get crd -o name 2>/dev/null | grep -cE '\.strimzi\.io' || echo 0)
+ [ "$remaining" -eq 0 ] 2>/dev/null && break
+ sleep 5
+ done
+fi
+
+echo "[MIGRATE] Strimzi CRD migration complete. telemetry.sh will recreate CRDs and CRs."
diff --git a/src/playbooks/upgrade/upgrade.yml b/src/playbooks/upgrade/upgrade.yml
index 7bf744ca69..e2c27ad10b 100644
--- a/src/playbooks/upgrade/upgrade.yml
+++ b/src/playbooks/upgrade/upgrade.yml
@@ -453,7 +453,7 @@
── Omnia Upgrade Execution Plan (in order) ──────────────────
1. oim → Upgrade OpenCHAMI control-plane containers
- on the Omnia Infrastructure Manager
+ and Pulp container (3.80 → 3.113) on the OIM
2. build_stream → SKIPPED (not enabled in build_stream_config.yml)
3. local_repo → Synchronize Omnia 2.2 packages into the
local Pulp repository for cluster nodes
@@ -607,7 +607,7 @@
# Sub-flow imports (each sub-flow reads upgrade_manifest.yml and
# skips if its component_status is already 'completed')
# ──────────────────────────────────────────────────────────────────────
-- name: Upgrade OIM tasks (includes OpenCHAMI)
+- name: Upgrade OIM tasks (includes OpenCHAMI and Pulp)
ansible.builtin.import_playbook: playbooks/upgrade_oim.yml
tags: [oim]
diff --git a/src/rpm_build/README.md b/src/rpm_build/README.md
new file mode 100644
index 0000000000..527523756c
--- /dev/null
+++ b/src/rpm_build/README.md
@@ -0,0 +1,84 @@
+# Omnia RPM Build
+
+Build custom RPM packages for Omnia components. Currently supports LDMS (OVIS) RPM builds.
+
+## Quick Start
+
+```bash
+# Build LDMS RPM with Slurm metrics support
+./build_rpm.sh -u -n
+
+# Build LDMS RPM without Slurm (warning: no slurm metrics)
+./build_rpm.sh
+
+# Specify LDMS version
+./build_rpm.sh -v 4.5.2 -u -n
+```
+
+## Directory Layout
+
+```
+src/rpm_build/
+├── build_rpm.sh # Entry point — clones OVIS, dispatches build
+├── README.md # This file
+└── ldms/
+ ├── start_build_container.rockylinux10.bash # Launches Podman container for RPM build
+ ├── build_ldms.rockylinux10.bash # LDMS build script (runs inside container)
+ ├── configure.sh # LDMS configure options
+ └── rpm_postuninstall.txt # RPM post-uninstall scriptlet
+```
+
+## How It Works
+
+1. **`build_rpm.sh`** clones the [OVIS repository](https://github.com/ovis-hpc/ovis.git) at the specified version tag
+2. **`start_build_container.rockylinux10.bash`** launches a Rocky Linux 10 Podman container with the OVIS source mounted
+3. Inside the container, **`build_ldms.rockylinux10.bash`** compiles LDMS and produces the RPM
+
+The build runs inside a container to ensure a clean, reproducible environment.
+
+## Architecture Support
+
+The build script auto-detects the host architecture via `uname -m`:
+
+| Host Architecture | Podman `--arch` | Supported |
+|-------------------|----------------|-----------|
+| x86_64 / amd64 | `x86_64` | Yes |
+| aarch64 / arm64 | `aarch64` | Yes |
+
+No separate scripts are needed — the same `start_build_container.rockylinux10.bash` handles both.
+
+## Parameters
+
+| Parameter | Flag | Default | Description |
+|-----------|------|---------|-------------|
+| LDMS version | `-v`, `--version` | `4.5.2` | OVIS LDMS version tag to build |
+| Slurm repo URL | `-u`, `--url` | — | YUM repo URL for Slurm (required for Slurm metrics) |
+| Slurm repo name | `-n`, `--name` | — | YUM repo name for Slurm |
+
+```bash
+# Positional arguments also supported
+./build_rpm.sh
+```
+
+## Prerequisites
+
+- **Podman** installed on the build host
+- Internet access to clone OVIS and pull the Rocky Linux 10 base image
+- (Optional) Slurm YUM repository URL for Slurm metrics support
+
+## Output
+
+The built RPM is produced inside the container under the OVIS build directory.
+Bind-mounted paths ensure the output is accessible on the host after the build completes.
+
+## Relationship to Container Builds
+
+| What | Where | Purpose |
+|------|-------|---------|
+| Container images | `src/containers/` | Build Omnia container images (omnia_core, ldms, etc.) |
+| RPM packages | `src/rpm_build/` | Build standalone RPM packages for node installation |
+
+The LDMS **container** image (built via `src/containers/ldms/`) and the LDMS **RPM** (built here)
+serve different deployment models:
+- **Container**: LDMS aggregator running as a container on the OIM
+- **RPM**: LDMS sampler installed directly on compute nodes
diff --git a/src/rpm_build/build_rpm.sh b/src/rpm_build/build_rpm.sh
new file mode 100755
index 0000000000..5ea5a2b5c2
--- /dev/null
+++ b/src/rpm_build/build_rpm.sh
@@ -0,0 +1,112 @@
+#!/bin/bash
+
+# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+# Parse command-line inputs for SLURM repo URL and name.
+print_usage() {
+ echo "Usage: $0 -u|--url -n|--name "
+ echo " or: $0 "
+}
+
+SLURM_REPO_URL=""
+SLURM_REPO_NAME=""
+LDMS_VERSION="4.5.1"
+# Parse command-line option for LDMS version
+while [[ $# -gt 0 ]]; do
+ case "$1" in
+ -v|--version)
+ LDMS_VERSION="$2"
+ shift 2
+ ;;
+ -u|--url)
+ SLURM_REPO_URL="$2"
+ shift 2
+ ;;
+ -n|--name)
+ SLURM_REPO_NAME="$2"
+ shift 2
+ ;;
+ -h|--help)
+ print_usage
+ exit 0
+ ;;
+ *)
+ # accept positional args if flags not used
+ if [[ -z "$SLURM_REPO_URL" ]]; then
+ SLURM_REPO_URL="$1"
+ elif [[ -z "$SLURM_REPO_NAME" ]]; then
+ SLURM_REPO_NAME="$1"
+ elif [[ -z "$LDMS_VERSION" ]]; then
+ LDMS_VERSION="$1"
+ else
+ echo "Unexpected argument: $1"
+ print_usage
+ exit 1
+ fi
+ shift
+ ;;
+ esac
+done
+echo "Using LDMS_VERSION=$LDMS_VERSION"
+if [[ -z "$SLURM_REPO_URL" || -z "$SLURM_REPO_NAME" ]]; then
+ echo "Warning: SLURM_REPO_URL and SLURM_REPO_NAME are not provided, user might not be able to generate ldms slurm metrics."
+fi
+
+echo "Using SLURM_REPO_URL=$SLURM_REPO_URL"
+echo "Using SLURM_REPO_NAME=$SLURM_REPO_NAME"
+
+# Get the script directory (handles monorepo src/rpm_build/ layout)
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# Target build scripts directory
+TARGET_DIR="$SCRIPT_DIR/ldms"
+
+# Print the result
+echo "Target directory: $TARGET_DIR"
+
+# === Step 1: Clone OVIS repo if not already present ===
+REPO_URL="https://github.com/ovis-hpc/ovis.git"
+DEST_DIR="$HOME/ovis-code"
+
+mkdir -p "$DEST_DIR"
+cd "$DEST_DIR"
+
+if [ ! -d "ovis" ]; then
+ echo "Cloning OVIS repository (version $LDMS_VERSION)..."
+ git clone --branch v"$LDMS_VERSION" --depth 1 "$REPO_URL"
+else
+ echo "Repository already exists at $DEST_DIR/ovis. Skipping clone."
+fi
+
+# === Step 2: Export LDMS_REPO path ===
+export LDMS_REPO="$DEST_DIR/ovis"
+echo "LDMS_REPO set to $LDMS_REPO"
+
+# === Step 3: Start container build script ===
+# Verify target directory exists
+if [ ! -d "$TARGET_DIR" ]; then
+ echo "Error: Directory $TARGET_DIR does not exist."
+ exit 1
+fi
+
+echo "Starting container build..."
+if [[ -n "$SLURM_REPO_URL" && -n "$SLURM_REPO_NAME" ]]; then
+ echo "Starting container build with SLURM_REPO_URL=$SLURM_REPO_URL and SLURM_REPO_NAME=$SLURM_REPO_NAME"
+ bash "$TARGET_DIR/start_build_container.rockylinux10.bash" "$SLURM_REPO_URL" "$SLURM_REPO_NAME"
+else
+ echo "Warning: Starting container build without SLURM"
+ bash "$TARGET_DIR/start_build_container.rockylinux10.bash"
+fi
diff --git a/src/rpm_build/ldms/build_ldms.rockylinux10.bash b/src/rpm_build/ldms/build_ldms.rockylinux10.bash
new file mode 100755
index 0000000000..f514133eaa
--- /dev/null
+++ b/src/rpm_build/ldms/build_ldms.rockylinux10.bash
@@ -0,0 +1,215 @@
+#!/bin/bash
+
+pkg_file="ovis-ldms.tar.gz"
+ARCH=$(uname -m)
+
+echo "Install epel-release"
+dnf install -y epel-release
+dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-10.noarch.rpm
+
+# Doxygen
+#dnf config-manager -y --set-enabled powertools
+
+dnf install -y dnf-plugins-core
+dnf config-manager --set-enabled crb
+
+echo "Update"
+dnf update -y
+
+echo "Install packages"
+dnf install -y \
+ autoconf \
+ automake \
+ bison \
+ bzip2 \
+ curl \
+ doxygen \
+ flex \
+ gcc \
+ gettext \
+ git \
+ gcc-c++ \
+ gzip \
+ hostname \
+ jansson-devel \
+ jansson \
+ jq \
+ libcurl-devel \
+ libibverbs \
+ libpfm-devel \
+ librdkafka \
+ librdmacm \
+ less \
+ libtool \
+ m4 \
+ make \
+ openssl \
+ openssl-devel \
+ openssl-libs \
+ papi-devel \
+ pkg-config \
+ python3-pyverbs \
+ python3 \
+ python3-docutils \
+ python3-pip \
+ python3-pycurl \
+ python3-devel \
+ python3-Cython \
+ platform-python-devel \
+ python3-docutils \
+ python3-pip \
+ rpm-build \
+ rsync \
+ rubygems \
+ ruby \
+ ruby-devel \
+ tree \
+ vim \
+ wget \
+ which
+# Install slurm packages
+dnf install -y \
+ slurm \
+ slurm-devel
+#--------------------------------
+# CLEAN
+#--------------------------------
+echo "[>>] Clean previous install"
+if [ -f "$pkg_file" ]; then
+ rm -rf "$pkg_file"
+fi
+if [ -f "*.rpm" ]; then
+ rm -rf "*.rpm"
+fi
+find ./ -name "*.deb" -delete
+for i in \
+ /app \
+ /opt/ovis-ldms \
+ /app/etc/ldms \
+ /app/etc/systemd/system/ldmsd.kokkos.service \
+ /app/etc/systemd/system/ldmsd.aggregator.service \
+ /app/etc/systemd/system/ldmsd.sampler.service \
+ lib/etc/ld.so.conf.d/ovis-ld-so.conf \
+ $pkg_file \
+; do
+ if [ -e "$i" ]; then
+ echo "[>>] Delete $i"
+ rm -rf "$i"
+ fi
+done
+echo "[>>] Remove old rpm and tar.gz"
+rm -rf ovis-ldms.{rpm,tar.gz}
+if [ -d "/build/libserde" ]; then
+ rm -rf "/build/libserde"
+fi
+echo "[>>] Clean source tree"
+rm -rf .version
+make uninstall
+make distclean
+make clean
+make maintainer-clean
+echo "[>>] Clean isn't clean. Remove any file with an .in file"
+find ./ -name "*.in" |(while read FOO; do base="$(echo $FOO |sed 's/\.in$//g')"; echo "Remove $base"; rm -rf "$base"; done; )
+find ./ -name "*.cache" -delete
+
+#--------------------------------
+# BUILD
+#--------------------------------
+echo "[>>] Get in source dir"
+#popd
+#rpm -i datacenter-gpu-manager-2.2.3-1-x86_64.rpm
+set -xe
+echo "#define DCGM_PUBLIC_API" >> "/usr/include/dcgm_api_export.h"
+echo "[--] Build ldms"
+echo "[>>] autoreconf"
+autoreconf --install
+echo "[>>] autogen"
+./autogen.sh
+echo "[>>] configure"
+../scripts/configure.sh
+echo "[>>] make"
+make -j 10
+echo "[>>] make install"
+make install
+echo "[>>] Find python3 site packages"
+
+PYTHON_VERSION="$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')"
+PYTHON_SITE_PKGS="$(python3 -m site --user-site)"
+echo "PYTHON_VERSION=$PYTHON_VERSION"
+echo "PYTHON_SITE_PKGS=$PYTHON_SITE_PKGS"
+
+#--------------------------------
+# ASSEBLE PACKAGE TREE
+#--------------------------------
+echo "[>>] Make package tree staging area: /app"
+mkdir -p \
+ /app/etc/profile.d \
+ /app/etc/sysconfig \
+ /app/lib/systemd/system \
+ /app/opt \
+ /app/${PYTHON_SITE_PKGS} \
+ /app/usr/share
+echo "[>>] Stage /usr/share/man"
+mv /opt/ovis-ldms/share/man /app/usr/share/
+echo "[>>] Stage /opt/ovis-ldms"
+mv /opt/ovis-ldms /app/opt/
+rsync -a /app/opt/ovis-ldms/share/doc/ovis-ldms/sample_init_scripts/opt/etc/systemd /app/opt/ovis-ldms/etc/
+rsync -a /app/opt/ovis-ldms/share/doc/ovis-ldms/sample_init_scripts/opt/etc/ldms /app/opt/ovis-ldms/etc/
+echo "[>>] Stage /etc"
+ln -s /opt/ovis-ldms/etc/profile.d/set-ovis-variables.sh /app/etc/profile.d/set-ovis-variables.sh
+echo "[>>] Stage /lib/systemd/system"
+ln -s /opt/ovis-ldms/etc/systemd/system/nersc-ldmsd.sampler.service /app/lib/systemd/system/nersc-ldmsd.sampler.service
+echo "[>>] Stage ${PYTHON_SITE_PKGS}"
+ln -s /opt/ovis-ldms/lib/${PYTHON_VERSION}/ldmsd /app/${PYTHON_SITE_PKGS}/
+ln -s /opt/ovis-ldms/lib/${PYTHON_VERSION}/ovis_ldms /app/${PYTHON_SITE_PKGS}/
+
+#--------------------------------
+# PACKAGE
+#--------------------------------
+echo "[>>] Bundle"
+ls -tlr /app
+tar -C /app -czvpf "$pkg_file" .
+
+echo "[>>] Build Product"
+du -sh $pkg_file
+
+echo "[>>] install fpm"
+
+# Workaround due pleaserun hard dependency on ruby-3.0
+# REF: https://github.com/jordansissel/fpm/issues/2048
+cat > gems.rb <<'EOF'
+gem 'dotenv', '= 2.8.1'
+gem 'fpm', '= 1.15.1'
+EOF
+
+gem install --no-document --file gems.rb
+
+fpm --version
+echo "[>>] Build RPM with fpm"
+fpm \
+--input-type tar \
+--output-type rpm \
+--name ovis-ldms \
+--version $(cat .version) \
+--iteration 1 \
+--depends bash \
+--depends python3-Cython \
+--depends python3-devel \
+--directories=/opt/ovis-ldms \
+--post-uninstall ../scripts/rpm_postuninstall.txt \
+--license "GPLv2 or BSD" \
+--rpm-group root \
+--description "This package provides the LDMS commands and libraries.\n* ldmsd: the LDMS daemon, which can run as sampler or aggregator (or both).\n* ldms_ls: the tool to list metric information of an ldmsd.\n* ldmsctl: the tool to control an ldmsd." \
+--rpm-summary "LDMS - Lighweight Distributed Metric Service" \
+--package ovis-ldms-$(cat .version).${ARCH}.rpm \
+$pkg_file
+
+echo "[>>] List rpm created"
+ls -tlr *tar.gz *.rpm
+echo "[>>] List files in rpm"
+rpm -qlp "ovis-ldms-$(cat .version).${ARCH}.rpm"
+echo "[>>] Show rpm scripts"
+rpm -qp --scripts "ovis-ldms-$(cat .version).${ARCH}.rpm"
+echo "[>>] Show rpm info"
+rpm -qi "ovis-ldms-$(cat .version).${ARCH}.rpm"
+echo "[--] Success"
diff --git a/src/rpm_build/ldms/configure.sh b/src/rpm_build/ldms/configure.sh
new file mode 100755
index 0000000000..9fc37435b3
--- /dev/null
+++ b/src/rpm_build/ldms/configure.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+echo "[>>] configure"
+CFLAGS="-ggdb3 -O0" \
+./configure \
+ --prefix="/opt/ovis-ldms" \
+ --with-slurm="/usr/include/slurm" \
+ --with-libevent="/opt/ovis-ldms/lib" \
+ --disable-mmap \
+ --enable-cray-samplers \
+ --enable-doc \
+ --enable-doc-html \
+ --enable-doc-man \
+ --enable-etc \
+ --enable-genderssystemd \
+ --enable-influx \
+ --enable-jobinfo-sampler \
+ --enable-kgnilnd \
+ --enable-lustre \
+ --enable-munge \
+ --enable-papi \
+ --enable-slurm \
+ --enable-spank-plugin \
+ --enable-swig \
+ --enable-sysclassib \
+ --enable-tsampler
diff --git a/src/rpm_build/ldms/rpm_postuninstall.txt b/src/rpm_build/ldms/rpm_postuninstall.txt
new file mode 100644
index 0000000000..6c0e9da997
--- /dev/null
+++ b/src/rpm_build/ldms/rpm_postuninstall.txt
@@ -0,0 +1,6 @@
+# Remove man pages
+mandb --quiet
+# Unlink services
+systemctl daemon-reload
+systemctl reset-failed
+
diff --git a/src/rpm_build/ldms/start_build_container.rockylinux10.bash b/src/rpm_build/ldms/start_build_container.rockylinux10.bash
new file mode 100755
index 0000000000..54ae7a7c44
--- /dev/null
+++ b/src/rpm_build/ldms/start_build_container.rockylinux10.bash
@@ -0,0 +1,77 @@
+#!/bin/bash
+
+# Check LDMS_REPO variable
+if [ -z "$LDMS_REPO" ] ; then
+ echo "Set path to your clone of https://github.com/ovis-hpc/ovis.git:
+Run command: export LDMS_REPO=
+"
+ exit 1
+fi
+
+# Take inputs
+if [ $# -lt 2 ]; then
+ echo "Warning: slurm repo_url and slurm repo_name are not set"
+fi
+
+REPO_URL="$1"
+REPO_NAME="$2"
+
+export LDMS_REPO="$(readlink -f "$LDMS_REPO")"
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+# Detect or allow manual architecture selection
+ARCH=$(uname -m)
+
+# Normalize and validate architecture
+case "$ARCH" in
+ x86_64|amd64)
+ ARCH_NAME="x86_64"
+ ;;
+ aarch64|arm64)
+ ARCH_NAME="aarch64"
+ ;;
+ *)
+ echo "Unsupported architecture: $ARCH"
+ echo "Usage: $0 [x86_64|aarch64]"
+ exit 1
+ ;;
+esac
+
+echo "Detected/Selected architecture: $ARCH_NAME"
+echo
+echo "Start build container. From there: pushd /builds/ovis/ && ../scripts/build_ldms.rockylinux10.bash"
+
+if [[ -z "$REPO_URL" && -z "$REPO_NAME" ]]; then
+ echo "Warning: SLURM_REPO_URL and SLURM_REPO_NAME must be provided."
+ podman run -it --rm \
+ --arch "$ARCH_NAME" \
+ --mount type=bind,source="$LDMS_REPO",target=/builds/ovis,z \
+ --mount type=bind,source="$SCRIPT_DIR",target=/builds/scripts,z \
+ rockylinux:10.0 \
+ bash -c "
+ echo 'Running LDMS build...'
+ pushd /builds/ovis/ && ../scripts/build_ldms.rockylinux10.bash
+ "
+else
+ podman run -it --rm \
+ --arch "$ARCH_NAME" \
+ --mount type=bind,source="$LDMS_REPO",target=/builds/ovis,z \
+ --mount type=bind,source="$SCRIPT_DIR",target=/builds/scripts,z \
+ rockylinux:10.0 \
+ bash -c "
+ echo 'Configuring repo inside container...'
+ cat < /etc/yum.repos.d/${REPO_NAME}.repo
+[${REPO_NAME}]
+name=${REPO_NAME}
+baseurl=${REPO_URL}
+enabled=1
+gpgcheck=0
+EOF
+
+ dnf clean all && dnf repolist
+
+ echo 'Running LDMS build...'
+ pushd /builds/ovis/ && ../scripts/build_ldms.rockylinux10.bash
+ "
+fi