From b7e56c0f85b3cb39cba24e89c6e87c63133f4468 Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Thu, 16 Apr 2026 11:58:48 +0200 Subject: [PATCH] Adapt API v2 DTOs --- ...continuous_parallel_lvol_snapshot_clone.py | 2 - simplyblock_core/models/storage_node.py | 12 +- simplyblock_core/storage_node_ops.py | 9 +- simplyblock_web/api/v2/device.py | 4 +- simplyblock_web/api/v2/dtos.py | 259 ++++++++++++++---- 5 files changed, 213 insertions(+), 73 deletions(-) diff --git a/e2e/stress_test/continuous_parallel_lvol_snapshot_clone.py b/e2e/stress_test/continuous_parallel_lvol_snapshot_clone.py index 508141d1e..7285b2354 100644 --- a/e2e/stress_test/continuous_parallel_lvol_snapshot_clone.py +++ b/e2e/stress_test/continuous_parallel_lvol_snapshot_clone.py @@ -238,12 +238,10 @@ def _force_enqueue_deletes(self): self.logger.warning(f"[max_lvols] Forced enqueue of {added} lvol tree deletes to recover from cluster max-lvol limit") def _api(self, op: str, ctx: dict, fn, retries: int = 10, interval: int = 5): - last_exc = None for attempt in range(1, retries + 1): try: return fn() except Exception as e: - last_exc = e api_err = self._extract_api_error(e) if self._is_max_lvols_error(api_err): self._inc("failures", op if op in self._metrics["failures"] else "unknown", 1) diff --git a/simplyblock_core/models/storage_node.py b/simplyblock_core/models/storage_node.py index dba8b5f46..e95b4eedc 100644 --- a/simplyblock_core/models/storage_node.py +++ b/simplyblock_core/models/storage_node.py @@ -1,6 +1,7 @@ # coding=utf-8 import time -from typing import List +from datetime import datetime, timedelta, timezone +from typing import List, Optional from uuid import uuid4 from simplyblock_core import utils @@ -482,6 +483,13 @@ def lvol_del_sync_lock_reset(self) -> bool: time.sleep(0.250) return True + def uptime(self) -> Optional[timedelta]: + return ( + datetime.now(timezone.utc) - datetime.fromisoformat(self.online_since) + if self.online_since and self.status == StorageNode.STATUS_ONLINE + else None + ) + class NodeLVolDelLock(BaseModel): - pass \ No newline at end of file + pass diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index f155e00ab..ea64a390a 100755 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -2433,19 +2433,12 @@ def list_storage_nodes(is_json, cluster_id=None): nodes = db_controller.get_storage_nodes() data = [] output = "" - now = datetime.datetime.now(datetime.timezone.utc) for node in nodes: logger.debug(node) logger.debug("*" * 20) total_devices = len(node.nvme_devices) online_devices = 0 - uptime = "" - if node.online_since and node.status == StorageNode.STATUS_ONLINE: - try: - uptime = utils.strfdelta((now - datetime.datetime.fromisoformat(node.online_since))) - except Exception: - pass for dev in node.nvme_devices: if dev.status == NVMeDevice.STATUS_ONLINE: @@ -2459,7 +2452,7 @@ def list_storage_nodes(is_json, cluster_id=None): "LVols": f"{len(lvs)}", "Status": node.status, "Health": node.health_check, - "Up time": uptime, + "Up time": utils.strfdelta(uptime) if (uptime := node.uptime()) is not None else "", "CPU": f"{len(utils.hexa_to_cpu_list(node.spdk_cpu_mask))}", "MEM": utils.humanbytes(node.spdk_mem), "SPDK P": node.rpc_port, diff --git a/simplyblock_web/api/v2/device.py b/simplyblock_web/api/v2/device.py index f62a134fe..4635f4b6d 100644 --- a/simplyblock_web/api/v2/device.py +++ b/simplyblock_web/api/v2/device.py @@ -24,7 +24,7 @@ def list(cluster: Cluster, storage_node: StorageNode) -> List[DeviceDTO]: ret = db.get_device_stats(device, 1) if ret: stat_obj = ret[0] - data.append(DeviceDTO.from_model(device, stat_obj)) + data.append(DeviceDTO.from_model(device, storage_node.get_id(), stat_obj)) return data instance_api = APIRouter(prefix='/{device_id}') @@ -46,7 +46,7 @@ def get(cluster: Cluster, storage_node: StorageNode, device: Device) -> DeviceDT ret = db.get_device_stats(device, 1) if ret: stat_obj = ret[0] - return DeviceDTO.from_model(device, stat_obj) + return DeviceDTO.from_model(device, storage_node.get_id(), stat_obj) @instance_api.post('/remove', name='clusters:storage_nodes:devices:remove', status_code=204, responses={204: {"content": None}}) diff --git a/simplyblock_web/api/v2/dtos.py b/simplyblock_web/api/v2/dtos.py index f3056f4e5..d5b6c1bb0 100644 --- a/simplyblock_web/api/v2/dtos.py +++ b/simplyblock_web/api/v2/dtos.py @@ -1,10 +1,12 @@ +from datetime import timedelta from ipaddress import IPv4Address -from typing import List, Literal, Tuple, Optional +from typing import List, Literal, Tuple, Optional, cast from uuid import UUID from fastapi import Request from pydantic import BaseModel +from simplyblock_core.utils import hexa_to_cpu_list from simplyblock_core.models.cluster import Cluster from simplyblock_core.models.job_schedule import JobSchedule from simplyblock_core.models.lvol_model import LVol @@ -19,6 +21,56 @@ from . import util + +ClusterStatus = Literal[ + "active", + "read_only", + "inactive", + "suspended", + "degraded", + "unready", + "in_activation", + "in_expansion", +] + +StoragePoolStatus = Literal["active", "inactive"] + +StorageNodeStatus = Literal[ + "online", + "offline", + "suspended", + "in_shutdown", + "removed", + "in_restart", + "in_creation", + "unreachable", + "schedulable", + "down", +] + +TaskStatus = Literal["new", "running", "suspended", "done"] + +TaskFunctionName = Literal[ + "device_restart", + "node_restart", + "device_migration", + "failed_device_migration", + "new_device_migration", + "node_add", + "port_allow", + "balancing_on_restart", + "balancing_on_dev_rem", + "balancing_on_dev_add", + "jc_comp_resume", + "snapshot_replication", + "lvol_sync_del", + "lvol_migration", + "s3_backup", + "s3_backup_restore", + "s3_backup_merge", +] + + class CapacityStatDTO(BaseModel): date: int size_total: int @@ -43,7 +95,7 @@ class ClusterDTO(BaseModel): id: UUID name: Optional[str] nqn: str - status: Literal['active', 'read_only', 'inactive', 'suspended', 'degraded', 'unready', 'in_activation', 'in_expansion'] + status: ClusterStatus is_re_balancing: bool block_size: util.Unsigned distr_ndcs: int @@ -51,8 +103,8 @@ class ClusterDTO(BaseModel): ha: bool utliziation_critical: util.Percent utilization_warning: util.Percent - provisioned_cacacity_critical: util.Unsigned - provisioned_cacacity_warning: util.Unsigned + provisioned_capacity_critical: util.Unsigned + provisioned_capacity_warning: util.Unsigned node_affinity: bool anti_affinity: bool secret: str @@ -62,36 +114,46 @@ class ClusterDTO(BaseModel): capacity: CapacityStatDTO @staticmethod - def from_model(model: Cluster, stat_obj: Optional[StatsObject]=None): + def from_model(model: Cluster, stat_obj: Optional[StatsObject] = None): return ClusterDTO( id=UUID(model.get_id()), name=model.cluster_name, nqn=model.nqn, - status=model.status, # type: ignore + status=cast(ClusterStatus, model.status), is_re_balancing=model.is_re_balancing, block_size=model.blk_size, distr_ndcs=model.distr_ndcs, distr_npcs=model.distr_npcs, - ha=model.ha_type == 'ha', + ha=model.ha_type == "ha", utilization_warning=model.cap_warn, utliziation_critical=model.cap_crit, - provisioned_cacacity_warning=model.prov_cap_warn, - provisioned_cacacity_critical=model.prov_cap_crit, + provisioned_capacity_warning=model.prov_cap_warn, + provisioned_capacity_critical=model.prov_cap_crit, node_affinity=model.enable_node_affinity, anti_affinity=model.strict_node_anti_affinity, secret=model.secret, tls_enabled=model.tls, max_fault_tolerance=model.max_fault_tolerance, backup_enabled=bool(model.backup_config), - capacity=CapacityStatDTO.from_model(stat_obj if stat_obj else StatsObject()), + capacity=CapacityStatDTO.from_model( + stat_obj if stat_obj else StatsObject() + ), ) class DeviceDTO(BaseModel): id: UUID + cluster_id: UUID + storage_node_id: UUID + model: str + serial_number: str + nvme_controller: str + pcie_address: str status: str health_check: bool + retries_exhausted: bool size: int + cluster_device_order: util.Unsigned io_error: bool is_partition: bool nvmf_ips: List[IPv4Address] @@ -100,18 +162,28 @@ class DeviceDTO(BaseModel): capacity: CapacityStatDTO @staticmethod - def from_model(model: NVMeDevice, stat_obj: Optional[StatsObject]=None): + def from_model(model: NVMeDevice, storage_node_id: str, stat_obj: Optional[StatsObject] = None): return DeviceDTO( id=UUID(model.get_id()), + cluster_id=UUID(model.cluster_id), + storage_node_id=UUID(storage_node_id), + model=model.model_id, + serial_number=model.serial_number, + nvme_controller=model.nvme_controller, + pcie_address=model.pcie_address, status=model.status, health_check=model.health_check, + retries_exhausted=model.retries_exhausted, size=model.size, + cluster_device_order=model.cluster_device_order, io_error=model.io_error, is_partition=model.is_partition, - nvmf_ips=[IPv4Address(ip) for ip in model.nvmf_ip.split(',')], + nvmf_ips=[IPv4Address(ip) for ip in model.nvmf_ip.split(",")], nvmf_nqn=model.nvmf_nqn, nvmf_port=model.nvmf_port, - capacity=CapacityStatDTO.from_model(stat_obj if stat_obj else StatsObject()), + capacity=CapacityStatDTO.from_model( + stat_obj if stat_obj else StatsObject() + ), ) @@ -133,8 +205,9 @@ def from_model(model: MgmtNode): class StoragePoolDTO(BaseModel): id: UUID + cluster_id: UUID name: str - status: Literal['active', 'inactive'] + status: StoragePoolStatus max_size: util.Unsigned volume_max_size: util.Unsigned max_rw_iops: util.Unsigned @@ -146,11 +219,12 @@ class StoragePoolDTO(BaseModel): allowed_hosts: List[str] = [] @staticmethod - def from_model(model: Pool, stat_obj: Optional[StatsObject]=None): + def from_model(model: Pool, stat_obj: Optional[StatsObject] = None): return StoragePoolDTO( id=UUID(model.get_id()), + cluster_id=UUID(model.cluster_id), name=model.pool_name, - status=model.status, # type: ignore + status=cast(StoragePoolStatus, model.status), max_size=model.pool_max_size, volume_max_size=model.lvol_max_size, max_rw_iops=model.max_rw_ios_per_sec, @@ -159,7 +233,9 @@ def from_model(model: Pool, stat_obj: Optional[StatsObject]=None): max_w_mbytes=model.max_w_mbytes_per_sec, dhchap=getattr(model, 'dhchap', False), allowed_hosts=list(getattr(model, 'allowed_hosts', [])), - capacity=CapacityStatDTO.from_model(stat_obj if stat_obj else StatsObject()), + capacity=CapacityStatDTO.from_model( + stat_obj if stat_obj else StatsObject() + ), ) @@ -174,11 +250,16 @@ class SnapshotDTO(BaseModel): lvol: Optional[util.UrlPath] @staticmethod - def from_model(model: SnapShot, request: Request, cluster_id, pool_id, volume_id=None): + def from_model( + model: SnapShot, request: Request, cluster_id, pool_id, volume_id=None + ): from simplyblock_core.controllers import migration_controller + is_migrating = False if model.lvol is not None: - active_mig = migration_controller.get_active_migration_for_lvol(model.lvol.uuid) + active_mig = migration_controller.get_active_migration_for_lvol( + model.lvol.uuid + ) is_migrating = active_mig is not None return SnapshotDTO( @@ -189,81 +270,123 @@ def from_model(model: SnapShot, request: Request, cluster_id, pool_id, volume_id size=model.size, used_size=model.used_size, migrating=is_migrating, - lvol=str(request.url_for( - 'clusters:pools:volumes:detail', - cluster_id=cluster_id, - pool_id=pool_id, - volume_id=model.lvol.get_id(), - )) if model.lvol is not None and (volume_id == model.lvol.get_id()) else None, + lvol=str( + request.url_for( + "clusters:pools:volumes:detail", + cluster_id=cluster_id, + pool_id=pool_id, + volume_id=model.lvol.get_id(), + ) + ) + if model.lvol is not None and (volume_id == model.lvol.get_id()) + else None, ) class StorageNodeDTO(BaseModel): id: UUID - status: str + cluster_id: UUID + status: StorageNodeStatus + uptime: Optional[timedelta] hostname: str - cpu: int + host_nqn: str + cpu_total_count: util.Unsigned + cpu_spdk_count: util.Unsigned + cpu_poller_count: util.Unsigned + memory: util.Unsigned + hugepage_memory: util.Unsigned spdk_mem: int lvols: int - rpc_port: int - lvol_subsys_port: int - nvmf_port: int + lvols_max: util.Unsigned + snapshots_max: util.Unsigned + rpc_port: util.Port + lvol_subsys_port: util.Port + hublvol_port: util.Port + nvmf_port: util.Port mgmt_ip: IPv4Address health_check: bool - online_devices: str + device_count: int + online_device_count: int capacity: CapacityStatDTO @staticmethod - def from_model(model: StorageNode, stat_obj: Optional[StatsObject]=None): + def from_model(model: StorageNode, stat_obj: Optional[StatsObject] = None): return StorageNodeDTO( id=UUID(model.get_id()), - status=model.status, + cluster_id=UUID(model.cluster_id), + status=cast(StorageNodeStatus, model.status), + uptime=model.uptime(), hostname=model.hostname, - cpu=model.cpu, + host_nqn=model.host_nqn, + cpu_total_count=model.cpu, + cpu_spdk_count=len(hexa_to_cpu_list(model.spdk_cpu_mask)), + cpu_poller_count=len(model.poller_cpu_cores), + memory=model.memory, + hugepage_memory=model.hugepages, spdk_mem=model.spdk_mem, lvols=model.lvols, + lvols_max=model.max_lvol, + snapshots_max=model.max_snap, rpc_port=model.rpc_port, lvol_subsys_port=model.lvol_subsys_port, + hublvol_port=model.get_hublvol_port(), nvmf_port=model.nvmf_port, mgmt_ip=IPv4Address(model.mgmt_ip), health_check=model.health_check, - online_devices=f"{len(model.nvme_devices)}/{len([d for d in model.nvme_devices if d.status=='online'])}", - capacity=CapacityStatDTO.from_model(stat_obj if stat_obj else StatsObject()), + device_count=len(model.nvme_devices), + online_device_count=len([device for device in model.nvme_devices if device.status == "online" ]), + capacity=CapacityStatDTO.from_model( + stat_obj if stat_obj else StatsObject() + ), ) class TaskDTO(BaseModel): id: UUID - status: str + cluster_id: UUID + device_id: Optional[UUID] + storage_node_id: Optional[UUID] + status: TaskStatus canceled: bool - function_name: str + function_name: TaskFunctionName function_params: dict function_result: str + max_retry: Optional[util.Unsigned] retry: util.Unsigned - max_retry: int @staticmethod def from_model(model: JobSchedule): return TaskDTO( id=UUID(model.uuid), - status=model.status, + cluster_id=UUID(model.cluster_id), + device_id=UUID(model.device_id) if model.device_id != "" else None, + storage_node_id=UUID(model.node_id) + if model.node_id != "" + else None, + status=cast(TaskStatus, model.status), canceled=model.canceled, - function_name=model.function_name, + function_name=cast(TaskFunctionName, model.function_name), function_params=model.function_params, function_result=model.function_result, + max_retry=model.max_retry if model.max_retry >= 0 else None, retry=model.retry, - max_retry=model.max_retry, ) class VolumeDTO(BaseModel): id: UUID + cluster_id: UUID + storage_node_id: UUID name: str status: str health_check: bool + io_error: bool migrating: bool nqn: str hostname: str + priority_class: util.Unsigned + access_mode: str + namespace: str fabric: str nodes: List[util.UrlPath] port: util.Port @@ -279,7 +402,6 @@ class VolumeDTO(BaseModel): cloned_from: Optional[util.UrlPath] crypto_key: Optional[Tuple[str, str]] high_availability: bool - lvol_priority_class: util.Unsigned do_replicate: bool = False max_namespace_per_subsys: int max_rw_iops: util.Unsigned @@ -292,45 +414,63 @@ class VolumeDTO(BaseModel): rep_info: Optional[dict] = None from_source: bool = True - @staticmethod - def from_model(model: LVol, request: Request, cluster_id: str, stat_obj: Optional[StatsObject]=None, rep_info=None): + def from_model( + model: LVol, + request: Request, + cluster_id: str, + stat_obj: Optional[StatsObject] = None, + rep_info=None, + ): from simplyblock_core.controllers import migration_controller from simplyblock_core.db_controller import DBController as _DBC + active_mig = migration_controller.get_active_migration_for_lvol(model.uuid) _db = _DBC() eff_policy = _db.get_policy_for_lvol(model) return VolumeDTO( id=UUID(model.get_id()), + cluster_id=UUID(cluster_id), + storage_node_id=UUID(model.node_id), name=model.lvol_name, status=model.status, health_check=model.health_check, + io_error=model.io_error, migrating=active_mig is not None, nqn=model.nqn, hostname=model.hostname, + priority_class=model.lvol_priority_class, + namespace=model.namespace, + access_mode=model.mode, fabric=model.fabric, nodes=[ - str(request.url_for( - 'clusters:storage-nodes:detail', - cluster_id=cluster_id, - storage_node_id=node_id, - )) + str( + request.url_for( + "clusters:storage-nodes:detail", + cluster_id=cluster_id, + storage_node_id=node_id, + ) + ) for node_id in model.nodes ], port=model.subsys_port, size=model.size, - cloned_from=str(request.url_for( - 'clusters:storage-pools:snapshots:detail', - cluster_id=cluster_id, - pool_id=model.pool_uuid, - snapshot_id=model.cloned_from_snap - )) if model.cloned_from_snap else None, + cloned_from=str( + request.url_for( + "clusters:storage-pools:snapshots:detail", + cluster_id=cluster_id, + pool_id=model.pool_uuid, + snapshot_id=model.cloned_from_snap, + ) + ) + if model.cloned_from_snap + else None, crypto_key=( (model.crypto_key1, model.crypto_key2) if model.crypto_key1 and model.crypto_key2 else None ), - high_availability=model.ha_type == 'ha', + high_availability=model.ha_type == "ha", pool_uuid=model.pool_uuid, pool_name=model.pool_name, pvc_name=model.pvc_name, @@ -339,7 +479,6 @@ def from_model(model: LVol, request: Request, cluster_id: str, stat_obj: Optiona npcs=model.npcs, blobid=model.blobid, ns_id=model.ns_id, - lvol_priority_class=model.lvol_priority_class, do_replicate=model.do_replicate, max_namespace_per_subsys=model.max_namespace_per_subsys, max_rw_iops=model.rw_ios_per_sec, @@ -348,7 +487,9 @@ def from_model(model: LVol, request: Request, cluster_id: str, stat_obj: Optiona max_w_mbytes=model.w_mbytes_per_sec, allowed_hosts=[h["nqn"] for h in (model.allowed_hosts or [])], policy=eff_policy.policy_name if eff_policy else "", - capacity=CapacityStatDTO.from_model(stat_obj if stat_obj else StatsObject()), + capacity=CapacityStatDTO.from_model( + stat_obj if stat_obj else StatsObject() + ), rep_info=rep_info, from_source=model.from_source, )