Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions projects/amdsmi/amdsmi_cli/amdsmi_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

import functools
import logging
import sys
import os
Expand Down
1,283 changes: 1,274 additions & 9 deletions projects/amdsmi/amdsmi_cli/amdsmi_commands.py

Large diffs are not rendered by default.

388 changes: 386 additions & 2 deletions projects/amdsmi/amdsmi_cli/amdsmi_helpers.py

Large diffs are not rendered by default.

66 changes: 27 additions & 39 deletions projects/amdsmi/amdsmi_cli/amdsmi_init.py
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,13 @@ def check_amd_hsmp_driver():
return True
return False

def check_amd_ionic_driver():
""" Returns true if ionic is found in the list of initialized modules """
status_file = Path("/sys/module/ionic/initstate")
if status_file.exists():
if status_file.read_text(encoding="ascii").strip() == "live":
return True
return False

def amdsmi_cli_init():
""" Initializes AMDSMI Library for the CLI
Expand All @@ -94,45 +101,26 @@ def amdsmi_cli_init():
Raises:
err: AmdSmiLibraryException if not successful in initializing any drivers
"""
init_flag = amdsmi_interface.AmdSmiInitFlags.INIT_ALL_PROCESSORS
if check_amdgpu_driver() and check_amd_hsmp_driver():
init_flag = amdsmi_interface.AmdSmiInitFlags.INIT_AMD_APUS
logging.debug("Both amdgpu , amd_hsmp or hsmp_acpi driver's initstate is live")
try:
amdsmi_interface.amdsmi_init(init_flag)
except (amdsmi_interface.AmdSmiLibraryException, amdsmi_interface.AmdSmiParameterException) as e:
if e.err_code in (amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NOT_INIT,
amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_DRIVER_NOT_LOADED):
logging.error("Drivers not loaded (amdgpu, amd_hsmp or hsmp_acpi drivers not found in modules)")
sys.exit(-1)
else:
raise e
elif check_amdgpu_driver():
init_flag = amdsmi_interface.AmdSmiInitFlags.INIT_AMD_GPUS
logging.debug("amdgpu driver initstate is live")
try:
amdsmi_interface.amdsmi_init(init_flag)
except (amdsmi_interface.AmdSmiLibraryException, amdsmi_interface.AmdSmiParameterException) as e:
if e.err_code in (amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NOT_INIT,
amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_DRIVER_NOT_LOADED):
logging.error("Driver not loaded (amdgpu not found in modules)")
sys.exit(-1)
else:
raise e
logging.debug("amdgpu driver initialized successfully, but amd_hsmp or hsmp_acpi initstate was not live")
elif check_amd_hsmp_driver():
init_flag = amdsmi_interface.AmdSmiInitFlags.INIT_AMD_CPUS
logging.debug("amd_hsmp or hsmp_acpi driver initstate is live")
try:
amdsmi_interface.amdsmi_init(init_flag)
except (amdsmi_interface.AmdSmiLibraryException, amdsmi_interface.AmdSmiParameterException) as e:
if e.err_code in (amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NOT_INIT,
amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_DRIVER_NOT_LOADED):
logging.error("Driver not loaded (amd_hsmp or hsmp_acpi not found in modules)")
sys.exit(-1)
else:
raise e
logging.debug("amd_hsmp or hsmp_acpi driver initialized successfully, but amdgpu initstate was not live")
init_flag = 0
if check_amdgpu_driver():
init_flag |= amdsmi_interface.AmdSmiInitFlags.INIT_AMD_GPUS
logging.debug("amdgpu driver's initstate is live")
if check_amd_hsmp_driver():
init_flag |= amdsmi_interface.AmdSmiInitFlags.INIT_AMD_CPUS
logging.debug("hsmp driver's initstate is live")
if check_amd_ionic_driver():
logging.debug("ionic driver's initstate is live")
init_flag |= amdsmi_interface.AmdSmiInitFlags.INIT_AMD_NICS

try:
amdsmi_interface.amdsmi_init(init_flag)
except (amdsmi_interface.AmdSmiLibraryException, amdsmi_interface.AmdSmiParameterException) as e:
if e.err_code in (amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NOT_INIT,
amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_DRIVER_NOT_LOADED):
logging.error("Drivers not loaded (amdgpu, amd_hsmp, ionic, rdma drivers not found in modules)")
sys.exit(-1)
else:
raise e

logging.debug(f"AMDSMI initialized with atleast one driver successfully | init flag: {init_flag}")

Expand Down
135 changes: 134 additions & 1 deletion projects/amdsmi/amdsmi_cli/amdsmi_logger.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ def __init__(self, format='human_readable', destination='stdout', helpers=None)
self.helpers = helpers
self._cper_exit_message = True
self.store_cpu_json_output = []
self.store_nic_json_output = []
self.store_core_json_output = []
self.store_gpu_json_output = []
self.store_xgmi_metric_json_output = []
Expand Down Expand Up @@ -156,6 +157,10 @@ def _convert_json_to_tabular(self, json_object: Dict[str, any], dynamic=False):
elif key == 'gpu':
stored_gpu = string_value
table_values += string_value.rjust(3)
elif key == 'brcm_nic':
table_values += string_value.rjust(3)
elif key == 'brcm_switch':
table_values += string_value.rjust(3)
elif key == 'xcp':
stored_gpu = string_value
table_values += string_value.rjust(5)
Expand Down Expand Up @@ -188,6 +193,27 @@ def _convert_json_to_tabular(self, json_object: Dict[str, any], dynamic=False):
table_values += string_value.rjust(12)
elif key in ('pcie_replay'):
table_values += string_value.rjust(13)
#BRCM Device Metrics
#NIC
elif key == "NIC_TEMP_CURRENT":
table_values += string_value.rjust(21)
elif key == "NIC_TEMP_CRIT_ALARM":
table_values += string_value.rjust(22)
elif key == "NIC_TEMP_EMERGENCY_ALARM":
table_values += string_value.rjust(26)
elif key == "NIC_TEMP_SHUTDOWN_ALARM":
table_values += string_value.rjust(25)
elif key == "NIC_TEMP_MAX_ALARM":
table_values += string_value.rjust(20)
#SWITCH
elif key == "CURRENT_LINK_SPEED":
table_values += string_value.rjust(25)
elif key == "MAX_LINK_SPEED":
table_values += string_value.rjust(20)
elif key == "CURRENT_LINK_WIDTH":
table_values += string_value.rjust(20)
elif key == "MAX_LINK_WIDTH":
table_values += string_value.rjust(20)
# Only for handling topology tables
elif 'gpu_' in key:
table_values += string_value.ljust(13)
Expand Down Expand Up @@ -281,7 +307,7 @@ def _convert_json_to_human_readable(self, json_object: Dict[str, any]):
# Increase tabbing for device arguments by pulling them out of the main dictionary and assiging them to an empty string
tabbed_dictionary = {}
for key, value in capitalized_json.items():
if key not in ["GPU", "CPU", "CORE"]:
if key not in ["GPU", "CPU", "CORE","BRCM_NIC","BRCM_SWITCH","AI_NIC"]:
tabbed_dictionary[key] = value
# Filter out N/A values under clock
if key == "CLOCK":
Expand Down Expand Up @@ -419,6 +445,42 @@ def store_output(self, device_handle, argument, data):
"""
gpu_id = self.helpers.get_gpu_id_from_device_handle(device_handle)
self._store_output_amdsmi(gpu_id=gpu_id, argument=argument, data=data)

def store_nic_output(self, device_handle, argument, data):
""" Convert device handle to nic id and store output
params:
device_handle - device handle object to the target device output
argument (str) - key to store data
data (dict | list) - Data store against argument
return:
Nothing
"""
nic_id = self.helpers.get_nic_id_from_device_handle(device_handle)
self._store_nic_output_amdsmi(nic_id=nic_id, argument=argument, data=data)

def store_ainic_output(self, device_handle, argument, data):
""" Convert device handle to ainic id and store output
params:
device_handle - device handle object to the target device output
argument (str) - key to store data
data (dict | list) - Data store against argument
return:
Nothing
"""
nic_id = self.helpers.get_ainic_id_from_device_handle(device_handle)
self._store_ainic_output_amdsmi(nic_id=nic_id, argument=argument, data=data)

def store_switch_output(self, device_handle, argument, data):
""" Convert device handle to nic id and store output
params:
device_handle - device handle object to the target device output
argument (str) - key to store data
data (dict | list) - Data store against argument
return:
Nothing
"""
switch_id = self.helpers.get_switch_id_from_device_handle(device_handle)
self._store_switch_output_amdsmi(switch_id=switch_id, argument=argument, data=data)


def store_cpu_output(self, device_handle, argument, data):
Expand Down Expand Up @@ -511,7 +573,76 @@ def _store_output_amdsmi(self, gpu_id, argument, data):
self.output[argument] = data
else:
raise ValueError("Invalid output format: expected json, csv, or human_readable")

def _store_nic_output_amdsmi(self, nic_id, argument, data):
if argument == 'timestamp': # Make sure timestamp is the first element in the output
self.output['timestamp'] = int(time.time())

if self.is_json_format() or self.is_human_readable_format():
self.output['brcm_nic'] = int(nic_id)
if argument == 'values' and isinstance(data, dict):

self.output.update(data)
else:

self.output[argument] = data
elif self.is_csv_format():
self.output['brcm_nic'] = int(nic_id)

if argument == 'values' or isinstance(data, dict):
flat_dict = self.flatten_dict(data)
self.output.update(flat_dict)
else:
self.output[argument] = data
else:
raise ValueError("Invalid output format: expected json, csv, or human_readable")

def _store_ainic_output_amdsmi(self, nic_id, argument, data):
if argument == 'timestamp': # Make sure timestamp is the first element in the output
self.output['timestamp'] = int(time.time())

if self.is_json_format() or self.is_human_readable_format():
self.output['ai_nic'] = int(nic_id)
if argument == 'values' and isinstance(data, dict):

self.output.update(data)
else:

self.output[argument] = data
elif self.is_csv_format():
self.output['ai_nic'] = int(nic_id)

if argument == 'values' or isinstance(data, dict):
flat_dict = self.flatten_dict(data)
self.output.update(flat_dict)
else:
self.output[argument] = data
else:
raise ValueError("Invalid output format: expected json, csv, or human_readable")


def _store_switch_output_amdsmi(self, switch_id, argument, data):
if argument == 'timestamp': # Make sure timestamp is the first element in the output
self.output['timestamp'] = int(time.time())

if self.is_json_format() or self.is_human_readable_format():
self.output['brcm_switch'] = int(switch_id)
if argument == 'values' and isinstance(data, dict):

self.output.update(data)
else:

self.output[argument] = data
elif self.is_csv_format():
self.output['brcm_switch'] = int(switch_id)

if argument == 'values' or isinstance(data, dict):
flat_dict = self.flatten_dict(data)
self.output.update(flat_dict)
else:
self.output[argument] = data
else:
raise ValueError("Invalid output format: expected json, csv, or human_readable")

def store_multiple_device_output(self):
""" Store the current output into the multiple_device_output
Expand Down Expand Up @@ -606,6 +737,8 @@ def combine_arrays_to_json(self):
combined_json = {}
if self.store_cpu_json_output:
combined_json["cpu_data"] = self.store_cpu_json_output
if self.store_nic_json_output:
combined_json["nic_data"] = self.store_nic_json_output
if self.store_core_json_output:
combined_json["core_data"] = self.store_core_json_output
if self.store_gpu_json_output:
Expand Down
Loading
Loading