[Security Enhancement] Adds token auth to snode API

requirements.txt

@@ -25,4 +25,5 @@ jsonschema
 fastapi
 uvicorn
 prometheus_api_client
-paramiko
+paramiko
+flask_httpauth

simplyblock_cli/cli-reference.yaml

@@ -26,6 +26,10 @@ commands:
             type: bool
             default: false
             action: store_true
+          - name: "--access-token"
+            help: "Storage node API access token."
+            dest: access_token
+            type: str
       - name: configure
         help: "Prepare a configuration file to be used when adding the storage node."
         description: >
@@ -265,6 +269,10 @@ commands:
             dest: spdk_proxy_image
             type: str
             private: true
+          - name: "--access-token"
+            help: "Storage node API access token."
+            dest: access_token
+            type: str
       - name: delete
         help: "Deletes a storage node object from the state database."
         usage: >

simplyblock_cli/cli.py

@@ -90,6 +90,7 @@ def init_storage_node__deploy(self, subparser):
         subcommand = self.add_sub_command(subparser, 'deploy', 'Prepares a host to be used as a storage node.')
         argument = subcommand.add_argument('--ifname', help='Management interface name, e.g. eth0.', type=str, dest='ifname')
         argument = subcommand.add_argument('--isolate-cores', help='Isolates cores in kernel args for the provided CPU mask. Default: `false`.', default=False, dest='isolate_cores', action='store_true')
+        argument = subcommand.add_argument('--access-token', help='Storage node API access token.', type=str, dest='access_token')
 
     def init_storage_node__configure(self, subparser):
         subcommand = self.add_sub_command(subparser, 'configure', 'Prepare a configuration file to be used when adding the storage node.')
@@ -151,6 +152,7 @@ def init_storage_node__add_node(self, subparser):
             argument = subcommand.add_argument('--max-snap', help='The max snapshot per storage node. Default: `5000`.', type=int, default=5000, dest='max_snap')
         if self.developer_mode:
             argument = subcommand.add_argument('--spdk-proxy-image', help='The SPDK proxy image URI.', type=str, dest='spdk_proxy_image')
+        argument = subcommand.add_argument('--access-token', help='Storage node API access token.', type=str, dest='access_token')
 
     def init_storage_node__delete(self, subparser):
         subcommand = self.add_sub_command(subparser, 'delete', 'Deletes a storage node object from the state database.')

simplyblock_cli/clibase.py

@@ -79,8 +79,7 @@ def add_sub_command(self, parent_parser, command, help, usage=None):
         return parent_parser.add_parser(command, description=help, help=help, usage=usage)
 
     def storage_node__deploy(self, sub_command, args):
-        isolate_cores = args.isolate_cores
-        return storage_ops.deploy(args.ifname, isolate_cores)
+        return storage_ops.deploy(args.ifname, args.isolate_cores, args.access_token)
 
     def storage_node__configure_upgrade(self, sub_command, args):
         storage_ops.upgrade_automated_deployment_config()
@@ -195,6 +194,7 @@ def storage_node__add_node(self, sub_command, args):
                 ha_jm_count=ha_jm_count,
                 format_4k=format_4k,
                 spdk_proxy_image=getattr(args, 'spdk_proxy_image', None),
+                access_token=getattr(args, 'access_token', ""),
             )
         except Exception as e:
             print(e)

simplyblock_core/controllers/device_controller.py

@@ -9,7 +9,6 @@
 from simplyblock_core.models.storage_node import StorageNode
 from simplyblock_core.prom_client import PromClient
 from simplyblock_core.rpc_client import RPCClient
-from simplyblock_core.snode_client import SNodeClient
 
 logger = logging.getLogger()
 
@@ -260,7 +259,7 @@ def restart_device(device_id, force=False):
 
     if not snode.rpc_client().bdev_nvme_controller_list(device_obj.nvme_controller):
         try:
-            ret = SNodeClient(snode.api_endpoint, timeout=30, retry=1).bind_device_to_spdk(device_obj.pcie_address)
+            ret = snode.snode_client(timeout=30, retry=1).bind_device_to_spdk(device_obj.pcie_address)
             logger.debug(ret)
             snode.rpc_client().bdev_nvme_controller_attach(device_obj.nvme_controller, device_obj.pcie_address)
             snode.rpc_client().bdev_examine(f"{device_obj.nvme_controller}n1")
@@ -970,7 +969,7 @@ def new_device_from_failed(device_id):
 
     if not device_node.rpc_client().bdev_nvme_controller_list(device.nvme_controller):
         try:
-            ret = SNodeClient(device_node.api_endpoint, timeout=30, retry=1).bind_device_to_spdk(device.pcie_address)
+            ret = device_node.snode_client(timeout=30, retry=1).bind_device_to_spdk(device.pcie_address)
             logger.debug(ret)
             device_node.rpc_client().bdev_nvme_controller_attach(device.nvme_controller, device.pcie_address)
         except Exception as e:

simplyblock_core/controllers/health_controller.py

@@ -12,7 +12,6 @@
 from simplyblock_core.models.nvme_device import NVMeDevice, JMDevice, RemoteDevice
 from simplyblock_core.models.storage_node import StorageNode
 from simplyblock_core.rpc_client import RPCClient
-from simplyblock_core.snode_client import SNodeClient
 from simplyblock_core.controllers import device_controller
 
 logger = utils.get_logger(__name__)
@@ -117,10 +116,9 @@ def _check_node_rpc(rpc_ip, rpc_port, rpc_username, rpc_password, timeout=5, ret
     return False, False
 
 
-def _check_node_api(ip):
+def check_node_api(node):
     try:
-        snode_api = SNodeClient(f"{ip}:5000", timeout=90, retry=2)
-        logger.debug(f"Node API={ip}:5000")
+        snode_api = node.snode_client(timeout=90, retry=2)
         ret, _ = snode_api.is_live()
         logger.debug(f"snode is alive: {ret}")
         if ret:
@@ -130,14 +128,6 @@ def _check_node_api(ip):
     return False
 
 
-def _check_spdk_process_up(ip, rpc_port, cluster_id):
-    snode_api = SNodeClient(f"{ip}:5000", timeout=90, retry=2)
-    logger.debug(f"Node API={ip}:5000")
-    is_up, _ = snode_api.spdk_process_is_up(rpc_port, cluster_id)
-    logger.debug(f"SPDK is {is_up}")
-    return is_up
-
-
 def check_port_on_node(snode, port_id):
     fw_api = FirewallClient(snode, timeout=5, retry=2)
     command_output, _ = fw_api.get_firewall(snode.rpc_port)
@@ -191,9 +181,9 @@ def _check_node_ping(ip):
 
 
 def _check_ping_from_node(ip, ifname, node):
-    snodeapi = SNodeClient(node.api_endpoint, timeout=3, retry=3)
+    snode_api = node.snode_client( timeout=3, retry=3)
     try:
-        ret, _ = snodeapi.ping_ip(ip, ifname)
+        ret, _ = snode_api.ping_ip(ip, ifname)
         return bool(ret)
     except Exception as e:
         logger.error(e)
@@ -581,7 +571,7 @@ def check_node(node_id, with_devices=True):
     logger.info(f"Check: ping mgmt ip {snode.mgmt_ip} ... {ping_check}")
 
     # 2- check node API
-    node_api_check = _check_node_api(snode.mgmt_ip)
+    node_api_check = check_node_api(snode)
     logger.info(f"Check: node API {snode.mgmt_ip}:5000 ... {node_api_check}")
 
     # 3- check node RPC

simplyblock_core/controllers/lvol_controller.py

@@ -21,7 +21,6 @@
 from simplyblock_core.models.storage_node import StorageNode
 from simplyblock_core.prom_client import PromClient
 from simplyblock_core.rpc_client import RPCClient
-from simplyblock_core.snode_client import SNodeClient
 
 logger = lg.getLogger()
 
@@ -52,7 +51,7 @@ def _register_pool_dhchap_keys_on_node(pool, snode, rpc_client):
     Returns a dict with 'dhchap_key' and 'dhchap_ctrlr_key' keyring names,
     or an empty dict on failure.
     """
-    snode_api = SNodeClient(snode.api_endpoint)
+    snode_api = snode.snode_client()
     safe_pool = pool.get_id().replace("-", "_")
     key_names = {}
 
@@ -90,7 +89,7 @@ def _register_dhchap_keys_on_node(snode, host_nqn, host_entry, rpc_client):
     Returns a dict mapping key type ('dhchap_key', 'dhchap_ctrlr_key', 'psk')
     to the SPDK keyring name for use in subsystem_add_host.
     """
-    snode_api = SNodeClient(snode.api_endpoint)
+    snode_api = snode.snode_client()
     # Sanitize host NQN for use as filename
     safe_host = host_nqn.replace(":", "_").replace(".", "_")
     key_names = {}

simplyblock_core/env_var

@@ -1,5 +1,5 @@
 SIMPLY_BLOCK_COMMAND_NAME=sbcli-dev
 SIMPLY_BLOCK_VERSION=19.2.33
 
-SIMPLY_BLOCK_DOCKER_IMAGE=public.ecr.aws/simply-block/simplyblock:main
+SIMPLY_BLOCK_DOCKER_IMAGE=public.ecr.aws/simply-block/simplyblock:main-snode-auth
 SIMPLY_BLOCK_SPDK_ULTRA_IMAGE=public.ecr.aws/simply-block/ultra:main-latest

simplyblock_core/models/storage_node.py

@@ -10,6 +10,7 @@
 from simplyblock_core.models.job_schedule import JobSchedule
 from simplyblock_core.models.nvme_device import NVMeDevice, JMDevice, RemoteDevice, RemoteJMDevice
 from simplyblock_core.rpc_client import RPCClient, RPCException
+from simplyblock_core.snode_client import SNodeClient
 
 logger = utils.get_logger(__name__)
 
@@ -112,6 +113,8 @@ class StorageNode(BaseNodeObject):
     firewall_port: int = 5001
     lvol_poller_mask: str = ""
     spdk_proxy_image: str = ""
+    access_token = ""
+
 
     def get_lvol_subsys_port(self, lvs_name=None):
         """Get the client-facing NVMeoF port for a specific lvstore.
@@ -482,6 +485,11 @@ def lvol_del_sync_lock_reset(self) -> bool:
         time.sleep(0.250)
         return True
 
+    def snode_client(self, **kwargs):
+        """Return storage node api client to this node
+        """
+        return SNodeClient(self.api_endpoint, self.access_token, **kwargs)
+
 
 class NodeLVolDelLock(BaseModel):
     pass

simplyblock_core/services/health_check_service.py

@@ -67,7 +67,7 @@ def check_node(snode):
     logger.info(f"Check: ping mgmt ip {snode.mgmt_ip} ... {ping_check}")
 
     # 2- check node API
-    node_api_check = health_controller._check_node_api(snode.mgmt_ip)
+    node_api_check = health_controller.check_node_api(snode)
     logger.info(f"Check: node API {snode.mgmt_ip}:5000 ... {node_api_check}")
 
     # 3- check node RPC

simplyblock_core/services/storage_node_monitor.py

@@ -10,7 +10,6 @@
 from simplyblock_core.models.job_schedule import JobSchedule
 from simplyblock_core.models.nvme_device import NVMeDevice, JMDevice
 from simplyblock_core.models.storage_node import StorageNode
-from simplyblock_core.snode_client import SNodeClient
 
 logger = utils.get_logger(__name__)
 
@@ -464,7 +463,7 @@ def check_node(snode):
 
     # 2- check node API
     try:
-        snode_api = SNodeClient(f"{snode.mgmt_ip}:5000", timeout=10, retry=2)
+        snode_api = snode.snode_client(timeout=10, retry=2)
         ret, _ = snode_api.is_live()
         logger.info(f"Check: node API {snode.mgmt_ip}:5000 ... {ret}")
         if not ret:
@@ -478,7 +477,7 @@ def check_node(snode):
 
     # 3- check spdk process through node API
     try:
-        snode_api = SNodeClient(f"{snode.mgmt_ip}:5000", timeout=40, retry=2)
+        snode_api = snode.snode_client(timeout=40, retry=2)
         is_up, _ = snode_api.spdk_process_is_up( snode.rpc_port, snode.cluster_id)
         logger.info(f"Check: spdk process {snode.mgmt_ip}:5000 ... {bool(is_up)}")
         if not is_up:

simplyblock_core/services/tasks_runner_restart.py

@@ -176,7 +176,7 @@ def task_runner_node(task):
     # is node reachable?
     ping_check = health_controller._check_node_ping(node.mgmt_ip)
     logger.info(f"Check: ping mgmt ip {node.mgmt_ip} ... {ping_check}")
-    node_api_check = health_controller._check_node_api(node.mgmt_ip)
+    node_api_check = health_controller.check_node_api(node)
     logger.info(f"Check: node API {node.mgmt_ip}:5000 ... {node_api_check}")
     node_data_nic_ping_check = False
     for data_nic in node.data_nics:

simplyblock_core/snode_client.py

@@ -16,27 +16,27 @@ def __init__(self, message):
 
 class SNodeClient:
 
-    def __init__(self, ip_address, timeout=300, retry=5):
+    def __init__(self, ip_address, access_token="access_token", timeout=300, retry=5):
         self.ip_address = ip_address
         self.url = 'http://%s/snode/' % self.ip_address
         self.timeout = timeout
         self.session = requests.session()
         self.session.verify = False
+        self.session.auth = ("token", access_token)
         self.session.headers['Content-Type'] = "application/json"
         retries = Retry(total=retry, backoff_factor=1, connect=retry, read=retry)
         self.session.mount("http://", HTTPAdapter(max_retries=retries))
 
     def _request(self, method, path, payload=None):
         try:
-            logger.debug("Requesting path: %s, params: %s", path, payload)
             data = None
             params = None
             if payload:
                 if method == "GET" :
                     params = payload
                 else:
                     data = json.dumps(payload)
-
+            logger.debug("Requesting: %s, params: %s", self.url+path, params)
             response = self.session.request(method, self.url+path, data=data,
                                             timeout=self.timeout, params=params)
         except Exception as e: