From 910491178d69e2b5bcd90f5372cf0224a4a43474 Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Wed, 27 Oct 2021 14:14:49 +0200 Subject: [PATCH 01/67] fix nginx playbook version to stay compatible with system ansible --- requirements.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.yml b/requirements.yml index 86d23998..06febc31 100644 --- a/requirements.yml +++ b/requirements.yml @@ -1,3 +1,3 @@ --- -- nginxinc.nginx -- nginxinc.nginx_config +- nginxinc.nginx,0.21.0 +- nginxinc.nginx_config,0.3.3 From bbe00715defbb613a49fd1e292d527a2b03f83dd Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Mon, 15 Nov 2021 13:33:08 +0100 Subject: [PATCH 02/67] workaround gpg check --- roles/monitoring/tasks/main.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/roles/monitoring/tasks/main.yml b/roles/monitoring/tasks/main.yml index de1aacdc..7e851fc8 100644 --- a/roles/monitoring/tasks/main.yml +++ b/roles/monitoring/tasks/main.yml @@ -20,11 +20,16 @@ baseurl: https://repos.influxdata.com/centos/$releasever/{{ "arm64" if ansible_architecture == "aarch64" else ansible_architecture }}/stable/ gpgkey: https://repos.influxdata.com/influxdb.key +#- name: install telegraf package +# package: +# name: telegraf +# state: present +# notify: restart telegraf + - name: install telegraf package - package: + yum: name: telegraf - state: present - notify: restart telegraf + disable_gpg_check: yes - name: enable the telegraf service service: From dda92be25e696bbf0b98f558f387172081cfb457 Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Mon, 15 Nov 2021 13:44:36 +0100 Subject: [PATCH 03/67] add mpi for azure --- group_vars/compute.yml | 3 +++ group_vars/management.yml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/group_vars/compute.yml b/group_vars/compute.yml index 550ce43f..3a2f18d0 100644 --- a/group_vars/compute.yml +++ b/group_vars/compute.yml @@ -14,6 +14,9 @@ mpi_packages: google: - mpich - openmpi + azure: + - mpich + - openmpi aws: [] monitoring_role: client diff --git a/group_vars/management.yml b/group_vars/management.yml index dc96244a..dfbaa30f 100644 --- a/group_vars/management.yml +++ b/group_vars/management.yml @@ -27,6 +27,9 @@ mpi_packages: google: - openmpi-devel - mpich-devel + azure: + - openmpi-devel + - mpich-devel aws: [] monitoring_role: master From f4b096b357e064bd3dbf89adbc80acd775fda4e1 Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Tue, 16 Nov 2021 10:22:00 +0100 Subject: [PATCH 04/67] chmod /mnt/shared, since default anf will have mode 700 --- roles/finalise/tasks/main.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/roles/finalise/tasks/main.yml b/roles/finalise/tasks/main.yml index 8a022e57..f2a840d2 100644 --- a/roles/finalise/tasks/main.yml +++ b/roles/finalise/tasks/main.yml @@ -9,6 +9,12 @@ delay: 10 tags: packer +- name: update directory mode for the finalised files + file: + path: /mnt/shared + state: directory + mode: 0755 + - name: create directory for the finalised files file: path: /mnt/shared/finalised From f4b9e7b6005912ab5353e07daf12bcaddc25d8a6 Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Tue, 16 Nov 2021 17:45:57 +0100 Subject: [PATCH 05/67] add citc_cloud for azure --- roles/slurm/tasks/elastic_azure.yml | 37 +++++++++++++++++++++++++++++ roles/slurm/tasks/main.yml | 3 +++ 2 files changed, 40 insertions(+) create mode 100644 roles/slurm/tasks/elastic_azure.yml diff --git a/roles/slurm/tasks/elastic_azure.yml b/roles/slurm/tasks/elastic_azure.yml new file mode 100644 index 00000000..fa447c9d --- /dev/null +++ b/roles/slurm/tasks/elastic_azure.yml @@ -0,0 +1,37 @@ +--- +- name: install OCI tools + pip: + name: + - oci + virtualenv: /opt/cloud_sdk + +- name: create oci config directory + file: + path: "{{ slurm_elastic.azure.config_directory }}" + state: directory + owner: slurm + group: slurm + mode: 0755 + +- name: copy OCI private key + copy: + src: /home/opc/oci_api_key.pem + dest: "{{ slurm_elastic.azure.config_directory }}/oci_api_key.pem" + owner: slurm + group: slurm + mode: 0400 + +- name: configure OCI config file + copy: + src: /home/opc/config + dest: "{{ slurm_elastic.azure.config_directory }}/config" + owner: slurm + group: slurm + mode: 0400 + +- name: install startnode support module + copy: + src: citc_oci.py + dest: /opt/cloud_sdk/lib/python3.8/site-packages/citc_cloud.py + mode: u=rw,g=r,o=r + diff --git a/roles/slurm/tasks/main.yml b/roles/slurm/tasks/main.yml index debd807c..f3422cc5 100644 --- a/roles/slurm/tasks/main.yml +++ b/roles/slurm/tasks/main.yml @@ -215,6 +215,9 @@ - include_tasks: elastic_aws.yml when: slurm_role == "mgmt" and ansible_local.citc.csp == "aws" +- include_tasks: elastic_azure.yml + when: slurm_role == "mgmt" and ansible_local.citc.csp == "azure" + - name: start service slurmctld service: name: slurmctld From 48f938005309012c42c605e603b22c9e51f8dce2 Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Tue, 16 Nov 2021 17:55:59 +0100 Subject: [PATCH 06/67] add azure var --- group_vars/management.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/group_vars/management.yml b/group_vars/management.yml index dfbaa30f..5b21391a 100644 --- a/group_vars/management.yml +++ b/group_vars/management.yml @@ -12,6 +12,8 @@ slurm_role: mgmt slurm_elastic: oracle: config_directory: /home/slurm/.oci/ + azure: + config_directory: /home/slurm/.oci/ install_packages: - xorg-x11-xauth From 4c155f5026b0f483c812429d344215c70cec2907 Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Wed, 24 Nov 2021 16:24:29 +0100 Subject: [PATCH 07/67] tune elastic --- roles/slurm/tasks/elastic_azure.yml | 34 ++++++++++++++--------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/roles/slurm/tasks/elastic_azure.yml b/roles/slurm/tasks/elastic_azure.yml index fa447c9d..81a92966 100644 --- a/roles/slurm/tasks/elastic_azure.yml +++ b/roles/slurm/tasks/elastic_azure.yml @@ -5,7 +5,7 @@ - oci virtualenv: /opt/cloud_sdk -- name: create oci config directory +- name: create azure config directory file: path: "{{ slurm_elastic.azure.config_directory }}" state: directory @@ -13,25 +13,25 @@ group: slurm mode: 0755 -- name: copy OCI private key - copy: - src: /home/opc/oci_api_key.pem - dest: "{{ slurm_elastic.azure.config_directory }}/oci_api_key.pem" - owner: slurm - group: slurm - mode: 0400 + #- name: copy Azure private key + #copy: + #src: /home/opc/oci_api_key.pem + #dest: "{{ slurm_elastic.azure.config_directory }}/oci_api_key.pem" + #owner: slurm + #group: slurm + #mode: 0400 -- name: configure OCI config file - copy: - src: /home/opc/config - dest: "{{ slurm_elastic.azure.config_directory }}/config" - owner: slurm - group: slurm - mode: 0400 + #- name: configure Azure config file + #copy: + #src: /home/opc/config + ##dest: "{{ slurm_elastic.azure.config_directory }}/config" + #owner: slurm + ##group: slurm + #mode: 0400 - name: install startnode support module copy: - src: citc_oci.py + src: citc_azure.py dest: /opt/cloud_sdk/lib/python3.8/site-packages/citc_cloud.py - mode: u=rw,g=r,o=r + mode: 0755 From a8205d8b04ae337c1cf7ed6b015521bf72575a19 Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Wed, 24 Nov 2021 16:28:10 +0100 Subject: [PATCH 08/67] adding citc_azure --- roles/slurm/files/citc_azure.py | 168 ++++++++++++++++++++++++++++++++ 1 file changed, 168 insertions(+) create mode 100644 roles/slurm/files/citc_azure.py diff --git a/roles/slurm/files/citc_azure.py b/roles/slurm/files/citc_azure.py new file mode 100644 index 00000000..254f2a89 --- /dev/null +++ b/roles/slurm/files/citc_azure.py @@ -0,0 +1,168 @@ +import asyncio +import base64 +import re +import subprocess +import time +from typing import Dict, Optional, Tuple, List + +import oci # type: ignore +import yaml # type: ignore + +__all__ = ["get_nodespace", "start_node"] + + +def load_yaml(filename: str) -> dict: + with open(filename, "r") as f: + return yaml.safe_load(f) + + +def get_nodespace() -> Dict[str, str]: + """ + Get the information about the space into which we were creating nodes + This will be static for all nodes in this cluster + """ + return load_yaml("/etc/citc/startnode.yaml") + + +def get_subnet(oci_config, compartment_id: str, vcn_id: str, cluster_id: str) -> str: + """ + Get the relevant cluster subnet for a given compartment, VCN and AD + """ + return [s.id for s in oci.core.VirtualNetworkClient(oci_config).list_subnets(compartment_id, vcn_id=vcn_id).data if s.freeform_tags.get("cluster") == cluster_id][0] + + +def get_node_state(oci_config, log, compartment_id: str, hostname: str, cluster_id: str) -> str: + """ + Get the current node state of the VM for the given hostname + If there is no such VM, return "TERMINATED" + """ + matches = oci.core.ComputeClient(oci_config).list_instances(compartment_id=compartment_id, display_name=hostname).data + matches = [i for i in matches if i.freeform_tags.get("cluster") == cluster_id] + still_exist = [i for i in matches if i.lifecycle_state != "TERMINATED"] + if not still_exist: + return "TERMINATED" + if len(still_exist) > 1: + log.error(f"{hostname}: Multiple matches found for {hostname}") + return still_exist[0].lifecycle_state + + +def get_image(oci_config, compartment_id: str, cluster_id: str) -> oci.core.models.Image: + all_images = oci.pagination.list_call_get_all_results_generator(oci.core.ComputeClient(oci_config).list_images, 'record', compartment_id, operating_system="Oracle Linux") + our_images: List[oci.core.models.Image] = [i for i in all_images if i.freeform_tags.get("cluster") == cluster_id] + try: + return [i for i in our_images if "GPU" not in i.display_name][0] + except IndexError: + raise RuntimeError("Could not locate the image for the compute node") + + +def create_node_config(oci_config, hostname: str, ip: Optional[str], nodespace: Dict[str, str], ssh_keys: str) -> oci.core.models.LaunchInstanceDetails: + """ + Create the configuration needed to create ``hostname`` in ``nodespace`` with ``ssh_keys`` + """ + features = subprocess.run(["sinfo", "--Format=features:200", "--noheader", f"--nodes={hostname}"], stdout=subprocess.PIPE).stdout.decode().split(',') + ad_number = [f for f in features if f.startswith("ad=")][0].split("=")[1].strip() + ad = f"{nodespace['ad_root']}{ad_number}" + shape = [f for f in features if f.startswith("shape=")][0].split("=")[1].strip() + subnet = get_subnet(oci_config, nodespace["compartment_id"], nodespace["vcn_id"], nodespace["cluster_id"]) + image = get_image(oci_config, nodespace["compartment_id"], nodespace["cluster_id"]) + + with open("/home/slurm/bootstrap.sh", "rb") as f: + user_data = base64.b64encode(f.read()).decode() + + instance_details = oci.core.models.LaunchInstanceDetails( + compartment_id=nodespace["compartment_id"], + availability_domain=ad, + shape=shape, + subnet_id=subnet, + image_id=image.id, + display_name=hostname, + hostname_label=hostname, + create_vnic_details=oci.core.models.CreateVnicDetails(private_ip=ip, subnet_id=subnet) if ip else None, + metadata={ + "ssh_authorized_keys": ssh_keys.strip(), + "user_data": user_data, + }, + freeform_tags={ + "type": "compute", + "cluster": nodespace["cluster_id"], + }, + ) + + return instance_details + + +def get_ip(hostname: str) -> Tuple[Optional[str], Optional[str], Optional[str]]: + host_dns_match = re.match(r"(\d+\.){3}\d+", subprocess.run(["host", hostname], stdout=subprocess.PIPE).stdout.decode().split()[-1]) + dns_ip = host_dns_match.group(0) if host_dns_match else None + + slurm_dns_match = re.search(r"NodeAddr=((\d+\.){3}\d+)", subprocess.run(["scontrol", "show", "node", hostname], stdout=subprocess.PIPE).stdout.decode()) + slurm_ip = slurm_dns_match.group(1) if slurm_dns_match else None + + ip = dns_ip or slurm_ip + + return ip, dns_ip, slurm_ip + + +async def start_node(log, host: str, nodespace: Dict[str, str], ssh_keys: str) -> None: + log.info(f"{host}: Starting") + oci_config = oci.config.from_file() + + while get_node_state(oci_config, log, nodespace["compartment_id"], host, nodespace["cluster_id"]) == "TERMINATING": + log.info(f"{host}: host is currently terminating. Waiting...") + await asyncio.sleep(5) + + node_state = get_node_state(oci_config, log, nodespace["compartment_id"], host, nodespace["cluster_id"]) + if node_state != "TERMINATED": + log.warning(f"{host}: host is already running with state {node_state}") + return + + ip, _dns_ip, slurm_ip = get_ip(host) + + instance_details = create_node_config(oci_config, host, ip, nodespace, ssh_keys) + + loop = asyncio.get_event_loop() + retry_strategy_builder = oci.retry.RetryStrategyBuilder() + retry_strategy_builder.add_max_attempts(max_attempts=10).add_total_elapsed_time(total_elapsed_time_seconds=600) + retry_strategy = retry_strategy_builder.get_retry_strategy() + client = oci.core.ComputeClient(oci_config, retry_strategy=retry_strategy) + + try: + instance_result = await loop.run_in_executor(None, client.launch_instance, instance_details) + instance = instance_result.data + except oci.exceptions.ServiceError as e: + log.error(f"{host}: problem launching instance: {e}") + return + + if not slurm_ip: + node_id = instance.id + while not oci.core.ComputeClient(oci_config).list_vnic_attachments(instance_details.compartment_id, instance_id=node_id).data: + log.info(f"{host}: No VNIC attachment yet. Waiting...") + await asyncio.sleep(5) + + vnic_id = oci.core.ComputeClient(oci_config).list_vnic_attachments(instance_details.compartment_id, instance_id=node_id).data[0].vnic_id + private_ip = oci.core.VirtualNetworkClient(oci_config).get_vnic(vnic_id).data.private_ip + + log.info(f"{host}: Private IP {private_ip}") + subprocess.run(["scontrol", "update", f"NodeName={host}", f"NodeAddr={private_ip}"]) + + log.info(f"{host}: Started") + return instance + +def terminate_instance(log, hosts): + + config = oci.config.from_file() + + nodespace = get_nodespace() + for host in hosts: + log.info(f"Stopping {host}") + + try: + matching_nodes = oci.core.ComputeClient(config).list_instances(nodespace["compartment_id"], display_name=host).data + node_id = [n.id for n in matching_nodes if n.lifecycle_state not in {"TERMINATED", "TERMINATING"}][0] + + oci.core.ComputeClient(config).terminate_instance(node_id) + except Exception as e: + log.error(f" problem while stopping: {e}") + continue + + log.info(f" Stopped {host}") From 4ca54bf95e63e830902e2b48d7ae3004f0bb32bc Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Thu, 25 Nov 2021 11:35:19 +0100 Subject: [PATCH 09/67] azure python tools --- roles/slurm/tasks/elastic_azure.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/roles/slurm/tasks/elastic_azure.yml b/roles/slurm/tasks/elastic_azure.yml index 81a92966..b85836ad 100644 --- a/roles/slurm/tasks/elastic_azure.yml +++ b/roles/slurm/tasks/elastic_azure.yml @@ -1,8 +1,11 @@ --- -- name: install OCI tools +- name: install Azure tools pip: name: - - oci + - azure-mgmt-resource + - azure-mgmt-compute + - azure-mgmt-network + - azure-identity virtualenv: /opt/cloud_sdk - name: create azure config directory From aa768facbde57a1810bba9bb78783872e0b3a516 Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Thu, 25 Nov 2021 16:32:19 +0100 Subject: [PATCH 10/67] azure python tools --- roles/slurm/tasks/elastic_azure.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/slurm/tasks/elastic_azure.yml b/roles/slurm/tasks/elastic_azure.yml index b85836ad..c29bcfd5 100644 --- a/roles/slurm/tasks/elastic_azure.yml +++ b/roles/slurm/tasks/elastic_azure.yml @@ -6,6 +6,7 @@ - azure-mgmt-compute - azure-mgmt-network - azure-identity + - azure-cli virtualenv: /opt/cloud_sdk - name: create azure config directory From e6970fa083689a65805dc26e49b408299d63470f Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Thu, 25 Nov 2021 17:54:48 +0100 Subject: [PATCH 11/67] citc_azure --- roles/slurm/files/citc_azure.py | 119 ++++++++++++++++++++++---------- 1 file changed, 84 insertions(+), 35 deletions(-) diff --git a/roles/slurm/files/citc_azure.py b/roles/slurm/files/citc_azure.py index 254f2a89..862bfc62 100644 --- a/roles/slurm/files/citc_azure.py +++ b/roles/slurm/files/citc_azure.py @@ -5,8 +5,9 @@ import time from typing import Dict, Optional, Tuple, List -import oci # type: ignore import yaml # type: ignore +import os +from azure.identity import DefaultAzureCredential __all__ = ["get_nodespace", "start_node"] @@ -105,54 +106,102 @@ def get_ip(hostname: str) -> Tuple[Optional[str], Optional[str], Optional[str]]: async def start_node(log, host: str, nodespace: Dict[str, str], ssh_keys: str) -> None: log.info(f"{host}: Starting") - oci_config = oci.config.from_file() + #oci_config = oci.config.from_file() + credential = DefaultAzureCredential() + nodespace = get_nodespace() + subscription_id = nodespace["subscription"] + resource_client = ResourceManagementClient(credential, subscription_id) + compute_client = ComputeManagementClient(credential, subscription_id) + + VM_NAME = "ExampleVM" + USERNAME = "azureuser" + PASSWORD = "ChangePa$$w0rd24" + + print(f"Provisioning virtual machine {VM_NAME}; this operation might take a few minutes.") + + poller = compute_client.virtual_machines.begin_create_or_update(RESOURCE_GROUP_NAME, VM_NAME, + { + "location": LOCATION, + "storage_profile": { + "image_reference": { + "publisher": 'Canonical', + "offer": "UbuntuServer", + "sku": "16.04.0-LTS", + "version": "latest" + } + }, + "hardware_profile": { + "vm_size": "Standard_DS1_v2" + }, + "os_profile": { + "computer_name": VM_NAME, + "admin_username": USERNAME, + "admin_password": PASSWORD + }, + "network_profile": { + "network_interfaces": [{ + "id": nic_result.id, + }] + } + } + ) + vm_result = poller.result() - while get_node_state(oci_config, log, nodespace["compartment_id"], host, nodespace["cluster_id"]) == "TERMINATING": - log.info(f"{host}: host is currently terminating. Waiting...") - await asyncio.sleep(5) + print(f"Provisioned virtual machine {vm_result.name}") - node_state = get_node_state(oci_config, log, nodespace["compartment_id"], host, nodespace["cluster_id"]) - if node_state != "TERMINATED": - log.warning(f"{host}: host is already running with state {node_state}") - return + log.info(f"{host}: Started") + return vm_result - ip, _dns_ip, slurm_ip = get_ip(host) + #while get_node_state(oci_config, log, nodespace["compartment_id"], host, nodespace["cluster_id"]) == "TERMINATING": + # log.info(f"{host}: host is currently terminating. Waiting...") + # await asyncio.sleep(5) - instance_details = create_node_config(oci_config, host, ip, nodespace, ssh_keys) + #node_state = get_node_state(oci_config, log, nodespace["compartment_id"], host, nodespace["cluster_id"]) + #if node_state != "TERMINATED": + # log.warning(f"{host}: host is already running with state {node_state}") + # return - loop = asyncio.get_event_loop() - retry_strategy_builder = oci.retry.RetryStrategyBuilder() - retry_strategy_builder.add_max_attempts(max_attempts=10).add_total_elapsed_time(total_elapsed_time_seconds=600) - retry_strategy = retry_strategy_builder.get_retry_strategy() - client = oci.core.ComputeClient(oci_config, retry_strategy=retry_strategy) + #ip, _dns_ip, slurm_ip = get_ip(host) - try: - instance_result = await loop.run_in_executor(None, client.launch_instance, instance_details) - instance = instance_result.data - except oci.exceptions.ServiceError as e: - log.error(f"{host}: problem launching instance: {e}") - return + #instance_details = create_node_config(oci_config, host, ip, nodespace, ssh_keys) - if not slurm_ip: - node_id = instance.id - while not oci.core.ComputeClient(oci_config).list_vnic_attachments(instance_details.compartment_id, instance_id=node_id).data: - log.info(f"{host}: No VNIC attachment yet. Waiting...") - await asyncio.sleep(5) + #loop = asyncio.get_event_loop() + #retry_strategy_builder = oci.retry.RetryStrategyBuilder() + #retry_strategy_builder.add_max_attempts(max_attempts=10).add_total_elapsed_time(total_elapsed_time_seconds=600) + #retry_strategy = retry_strategy_builder.get_retry_strategy() + #client = oci.core.ComputeClient(oci_config, retry_strategy=retry_strategy) - vnic_id = oci.core.ComputeClient(oci_config).list_vnic_attachments(instance_details.compartment_id, instance_id=node_id).data[0].vnic_id - private_ip = oci.core.VirtualNetworkClient(oci_config).get_vnic(vnic_id).data.private_ip + #try: + # instance_result = await loop.run_in_executor(None, client.launch_instance, instance_details) + # instance = instance_result.data + #except oci.exceptions.ServiceError as e: + # log.error(f"{host}: problem launching instance: {e}") + # return - log.info(f"{host}: Private IP {private_ip}") - subprocess.run(["scontrol", "update", f"NodeName={host}", f"NodeAddr={private_ip}"]) + #if not slurm_ip: + # node_id = instance.id + # while not oci.core.ComputeClient(oci_config).list_vnic_attachments(instance_details.compartment_id, instance_id=node_id).data: + # log.info(f"{host}: No VNIC attachment yet. Waiting...") + # await asyncio.sleep(5) - log.info(f"{host}: Started") - return instance + # vnic_id = oci.core.ComputeClient(oci_config).list_vnic_attachments(instance_details.compartment_id, instance_id=node_id).data[0].vnic_id + # private_ip = oci.core.VirtualNetworkClient(oci_config).get_vnic(vnic_id).data.private_ip -def terminate_instance(log, hosts): + # log.info(f"{host}: Private IP {private_ip}") + # subprocess.run(["scontrol", "update", f"NodeName={host}", f"NodeAddr={private_ip}"]) + + #log.info(f"{host}: Started") + #return instance - config = oci.config.from_file() +def terminate_instance(log, hosts): + #config = oci.config.from_file() + credential = DefaultAzureCredential() nodespace = get_nodespace() + subscription_id = nodespace["subscription"] + resource_client = ResourceManagementClient(credential, subscription_id) + compute_client = ComputeManagementClient(credential, subscription_id) + for host in hosts: log.info(f"Stopping {host}") From eb48e81a0f3a4da705b87c6f61e68768a4167be9 Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Thu, 25 Nov 2021 20:45:16 +0100 Subject: [PATCH 12/67] citc_azure --- roles/slurm/files/citc_azure.py | 152 +++++++++++++++++--------------- 1 file changed, 81 insertions(+), 71 deletions(-) diff --git a/roles/slurm/files/citc_azure.py b/roles/slurm/files/citc_azure.py index 862bfc62..0ba844a6 100644 --- a/roles/slurm/files/citc_azure.py +++ b/roles/slurm/files/citc_azure.py @@ -25,83 +25,93 @@ def get_nodespace() -> Dict[str, str]: return load_yaml("/etc/citc/startnode.yaml") -def get_subnet(oci_config, compartment_id: str, vcn_id: str, cluster_id: str) -> str: +#def get_subnet(oci_config, compartment_id: str, vcn_id: str, cluster_id: str) -> str: +def get_subnet() -> str: """ Get the relevant cluster subnet for a given compartment, VCN and AD """ - return [s.id for s in oci.core.VirtualNetworkClient(oci_config).list_subnets(compartment_id, vcn_id=vcn_id).data if s.freeform_tags.get("cluster") == cluster_id][0] + #return [s.id for s in oci.core.VirtualNetworkClient(oci_config).list_subnets(compartment_id, vcn_id=vcn_id).data if s.freeform_tags.get("cluster") == cluster_id][0] + return "subnet" -def get_node_state(oci_config, log, compartment_id: str, hostname: str, cluster_id: str) -> str: +#def get_node_state(oci_config, log, compartment_id: str, hostname: str, cluster_id: str) -> str: +def get_node_state() -> str: """ Get the current node state of the VM for the given hostname If there is no such VM, return "TERMINATED" """ - matches = oci.core.ComputeClient(oci_config).list_instances(compartment_id=compartment_id, display_name=hostname).data - matches = [i for i in matches if i.freeform_tags.get("cluster") == cluster_id] - still_exist = [i for i in matches if i.lifecycle_state != "TERMINATED"] - if not still_exist: - return "TERMINATED" - if len(still_exist) > 1: - log.error(f"{hostname}: Multiple matches found for {hostname}") - return still_exist[0].lifecycle_state - - -def get_image(oci_config, compartment_id: str, cluster_id: str) -> oci.core.models.Image: - all_images = oci.pagination.list_call_get_all_results_generator(oci.core.ComputeClient(oci_config).list_images, 'record', compartment_id, operating_system="Oracle Linux") - our_images: List[oci.core.models.Image] = [i for i in all_images if i.freeform_tags.get("cluster") == cluster_id] + #matches = oci.core.ComputeClient(oci_config).list_instances(compartment_id=compartment_id, display_name=hostname).data + #matches = [i for i in matches if i.freeform_tags.get("cluster") == cluster_id] + #still_exist = [i for i in matches if i.lifecycle_state != "TERMINATED"] + #if not still_exist: + # return "TERMINATED" + #if len(still_exist) > 1: + # log.error(f"{hostname}: Multiple matches found for {hostname}") + #return still_exist[0].lifecycle_state + return "node_state" + + +#def get_image(oci_config, compartment_id: str, cluster_id: str) -> oci.core.models.Image: +def get_image() -> str: + #all_images = oci.pagination.list_call_get_all_results_generator(oci.core.ComputeClient(oci_config).list_images, 'record', compartment_id, operating_system="Oracle Linux") + #our_images: List[oci.core.models.Image] = [i for i in all_images if i.freeform_tags.get("cluster") == cluster_id] try: - return [i for i in our_images if "GPU" not in i.display_name][0] + #return [i for i in our_images if "GPU" not in i.display_name][0] + return "image" except IndexError: raise RuntimeError("Could not locate the image for the compute node") -def create_node_config(oci_config, hostname: str, ip: Optional[str], nodespace: Dict[str, str], ssh_keys: str) -> oci.core.models.LaunchInstanceDetails: +#def create_node_config(oci_config, hostname: str, ip: Optional[str], nodespace: Dict[str, str], ssh_keys: str) -> oci.core.models.LaunchInstanceDetails: +def create_node_config() -> str: """ Create the configuration needed to create ``hostname`` in ``nodespace`` with ``ssh_keys`` """ - features = subprocess.run(["sinfo", "--Format=features:200", "--noheader", f"--nodes={hostname}"], stdout=subprocess.PIPE).stdout.decode().split(',') - ad_number = [f for f in features if f.startswith("ad=")][0].split("=")[1].strip() - ad = f"{nodespace['ad_root']}{ad_number}" - shape = [f for f in features if f.startswith("shape=")][0].split("=")[1].strip() - subnet = get_subnet(oci_config, nodespace["compartment_id"], nodespace["vcn_id"], nodespace["cluster_id"]) - image = get_image(oci_config, nodespace["compartment_id"], nodespace["cluster_id"]) - - with open("/home/slurm/bootstrap.sh", "rb") as f: - user_data = base64.b64encode(f.read()).decode() - - instance_details = oci.core.models.LaunchInstanceDetails( - compartment_id=nodespace["compartment_id"], - availability_domain=ad, - shape=shape, - subnet_id=subnet, - image_id=image.id, - display_name=hostname, - hostname_label=hostname, - create_vnic_details=oci.core.models.CreateVnicDetails(private_ip=ip, subnet_id=subnet) if ip else None, - metadata={ - "ssh_authorized_keys": ssh_keys.strip(), - "user_data": user_data, - }, - freeform_tags={ - "type": "compute", - "cluster": nodespace["cluster_id"], - }, - ) - - return instance_details - - -def get_ip(hostname: str) -> Tuple[Optional[str], Optional[str], Optional[str]]: - host_dns_match = re.match(r"(\d+\.){3}\d+", subprocess.run(["host", hostname], stdout=subprocess.PIPE).stdout.decode().split()[-1]) - dns_ip = host_dns_match.group(0) if host_dns_match else None - - slurm_dns_match = re.search(r"NodeAddr=((\d+\.){3}\d+)", subprocess.run(["scontrol", "show", "node", hostname], stdout=subprocess.PIPE).stdout.decode()) - slurm_ip = slurm_dns_match.group(1) if slurm_dns_match else None - - ip = dns_ip or slurm_ip - - return ip, dns_ip, slurm_ip + #features = subprocess.run(["sinfo", "--Format=features:200", "--noheader", f"--nodes={hostname}"], stdout=subprocess.PIPE).stdout.decode().split(',') + #ad_number = [f for f in features if f.startswith("ad=")][0].split("=")[1].strip() + #ad = f"{nodespace['ad_root']}{ad_number}" + #shape = [f for f in features if f.startswith("shape=")][0].split("=")[1].strip() + #subnet = get_subnet(oci_config, nodespace["compartment_id"], nodespace["vcn_id"], nodespace["cluster_id"]) + #image = get_image(oci_config, nodespace["compartment_id"], nodespace["cluster_id"]) + + #with open("/home/slurm/bootstrap.sh", "rb") as f: + # user_data = base64.b64encode(f.read()).decode() + + #instance_details = oci.core.models.LaunchInstanceDetails( + # compartment_id=nodespace["compartment_id"], + # availability_domain=ad, + # shape=shape, + # subnet_id=subnet, + # image_id=image.id, + # display_name=hostname, + # hostname_label=hostname, + # create_vnic_details=oci.core.models.CreateVnicDetails(private_ip=ip, subnet_id=subnet) if ip else None, + # metadata={ + # "ssh_authorized_keys": ssh_keys.strip(), + # "user_data": user_data, + # }, + # freeform_tags={ + # "type": "compute", + # "cluster": nodespace["cluster_id"], + # }, + #) + + #return instance_details + return "instance_details" + + +#def get_ip(hostname: str) -> Tuple[Optional[str], Optional[str], Optional[str]]: +def get_ip() -> str: + #host_dns_match = re.match(r"(\d+\.){3}\d+", subprocess.run(["host", hostname], stdout=subprocess.PIPE).stdout.decode().split()[-1]) + #dns_ip = host_dns_match.group(0) if host_dns_match else None + + #slurm_dns_match = re.search(r"NodeAddr=((\d+\.){3}\d+)", subprocess.run(["scontrol", "show", "node", hostname], stdout=subprocess.PIPE).stdout.decode()) + #slurm_ip = slurm_dns_match.group(1) if slurm_dns_match else None + + #ip = dns_ip or slurm_ip + + #return ip, dns_ip, slurm_ip + return "ip" async def start_node(log, host: str, nodespace: Dict[str, str], ssh_keys: str) -> None: @@ -202,16 +212,16 @@ def terminate_instance(log, hosts): resource_client = ResourceManagementClient(credential, subscription_id) compute_client = ComputeManagementClient(credential, subscription_id) - for host in hosts: - log.info(f"Stopping {host}") - - try: - matching_nodes = oci.core.ComputeClient(config).list_instances(nodespace["compartment_id"], display_name=host).data - node_id = [n.id for n in matching_nodes if n.lifecycle_state not in {"TERMINATED", "TERMINATING"}][0] - - oci.core.ComputeClient(config).terminate_instance(node_id) - except Exception as e: - log.error(f" problem while stopping: {e}") - continue + #for host in hosts: + # log.info(f"Stopping {host}") +# +# try: +# matching_nodes = oci.core.ComputeClient(config).list_instances(nodespace["compartment_id"], display_name=host).data +# node_id = [n.id for n in matching_nodes if n.lifecycle_state not in {"TERMINATED", "TERMINATING"}][0] +# +# oci.core.ComputeClient(config).terminate_instance(node_id) +# except Exception as e: +# log.error(f" problem while stopping: {e}") +# continue log.info(f" Stopped {host}") From 363755207e81a69006b7cfcb36951eff520611be Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Thu, 25 Nov 2021 20:47:36 +0100 Subject: [PATCH 13/67] citc_azure --- roles/slurm/files/citc_azure.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/roles/slurm/files/citc_azure.py b/roles/slurm/files/citc_azure.py index 0ba844a6..0a009148 100644 --- a/roles/slurm/files/citc_azure.py +++ b/roles/slurm/files/citc_azure.py @@ -8,6 +8,9 @@ import yaml # type: ignore import os from azure.identity import DefaultAzureCredential +from azure.mgmt.resource import ResourceManagementClient +from azure.mgmt.network import NetworkManagementClient +from azure.mgmt.compute import ComputeManagementClient __all__ = ["get_nodespace", "start_node"] From 9add1b11c3ec6230faf6a32c9a4c192e8f2e6c00 Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Thu, 25 Nov 2021 21:00:45 +0100 Subject: [PATCH 14/67] citc_azure --- roles/slurm/files/citc_azure.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/roles/slurm/files/citc_azure.py b/roles/slurm/files/citc_azure.py index 0a009148..243f6ab3 100644 --- a/roles/slurm/files/citc_azure.py +++ b/roles/slurm/files/citc_azure.py @@ -123,8 +123,24 @@ async def start_node(log, host: str, nodespace: Dict[str, str], ssh_keys: str) - credential = DefaultAzureCredential() nodespace = get_nodespace() subscription_id = nodespace["subscription"] + resource_group = nodespace["resource_group"] + region = nodespace["region"] + subnet = nodespace["subnet"] resource_client = ResourceManagementClient(credential, subscription_id) + network_client = NetworkManagementClient(credential, subscription_id) compute_client = ComputeManagementClient(credential, subscription_id) + + poller = network_client.network_interfaces.begin_create_or_update(RESOURCE_GROUP_NAME,NIC_NAME, + { + "location": region, + "ip_configurations": [ { + "name": "mynic", + "subnet": { "id": subnet }, + }] + } + ) + + nic_result = poller.result() VM_NAME = "ExampleVM" USERNAME = "azureuser" @@ -132,9 +148,9 @@ async def start_node(log, host: str, nodespace: Dict[str, str], ssh_keys: str) - print(f"Provisioning virtual machine {VM_NAME}; this operation might take a few minutes.") - poller = compute_client.virtual_machines.begin_create_or_update(RESOURCE_GROUP_NAME, VM_NAME, + poller = compute_client.virtual_machines.begin_create_or_update(resource_group, VM_NAME, { - "location": LOCATION, + "location": region, "storage_profile": { "image_reference": { "publisher": 'Canonical', From 0be676439498be35f84b6b8b2aee97202b9773f4 Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Thu, 25 Nov 2021 21:02:18 +0100 Subject: [PATCH 15/67] citc_azure --- roles/slurm/files/citc_azure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/slurm/files/citc_azure.py b/roles/slurm/files/citc_azure.py index 243f6ab3..b0420ace 100644 --- a/roles/slurm/files/citc_azure.py +++ b/roles/slurm/files/citc_azure.py @@ -130,7 +130,7 @@ async def start_node(log, host: str, nodespace: Dict[str, str], ssh_keys: str) - network_client = NetworkManagementClient(credential, subscription_id) compute_client = ComputeManagementClient(credential, subscription_id) - poller = network_client.network_interfaces.begin_create_or_update(RESOURCE_GROUP_NAME,NIC_NAME, + poller = network_client.network_interfaces.begin_create_or_update(resource_group,"mynic" { "location": region, "ip_configurations": [ { From f7850df3d0f491b20b2076b9badc3903bc49373a Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Thu, 25 Nov 2021 21:08:26 +0100 Subject: [PATCH 16/67] citc_azure --- roles/slurm/files/citc_azure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/slurm/files/citc_azure.py b/roles/slurm/files/citc_azure.py index b0420ace..90c25f1a 100644 --- a/roles/slurm/files/citc_azure.py +++ b/roles/slurm/files/citc_azure.py @@ -130,7 +130,7 @@ async def start_node(log, host: str, nodespace: Dict[str, str], ssh_keys: str) - network_client = NetworkManagementClient(credential, subscription_id) compute_client = ComputeManagementClient(credential, subscription_id) - poller = network_client.network_interfaces.begin_create_or_update(resource_group,"mynic" + poller = network_client.network_interfaces.begin_create_or_update(resource_group,"mynic", { "location": region, "ip_configurations": [ { From 2471d1c83f7d5931be34a59a18a78111a612df81 Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Fri, 26 Nov 2021 10:15:41 +0100 Subject: [PATCH 17/67] citc_azure --- roles/slurm/files/citc_azure.py | 35 ++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/roles/slurm/files/citc_azure.py b/roles/slurm/files/citc_azure.py index 90c25f1a..f3514596 100644 --- a/roles/slurm/files/citc_azure.py +++ b/roles/slurm/files/citc_azure.py @@ -38,11 +38,13 @@ def get_subnet() -> str: #def get_node_state(oci_config, log, compartment_id: str, hostname: str, cluster_id: str) -> str: -def get_node_state() -> str: +def get_node_state(compute_client, log, hostname: str, resource_group: str) -> str: """ Get the current node state of the VM for the given hostname If there is no such VM, return "TERMINATED" """ + matches = compute_client.virtual_machines.list(resource_group) + print(matches) #matches = oci.core.ComputeClient(oci_config).list_instances(compartment_id=compartment_id, display_name=hostname).data #matches = [i for i in matches if i.freeform_tags.get("cluster") == cluster_id] #still_exist = [i for i in matches if i.lifecycle_state != "TERMINATED"] @@ -130,11 +132,15 @@ async def start_node(log, host: str, nodespace: Dict[str, str], ssh_keys: str) - network_client = NetworkManagementClient(credential, subscription_id) compute_client = ComputeManagementClient(credential, subscription_id) - poller = network_client.network_interfaces.begin_create_or_update(resource_group,"mynic", + while get_node_state(compute_client, log, host, resource_group) == "TERMINATING": + log.info(f"{host}: host is currently terminating. Waiting...") + await asyncio.sleep(5) + + poller = network_client.network_interfaces.begin_create_or_update(resource_group,host-"nic", { "location": region, "ip_configurations": [ { - "name": "mynic", + "name": host-"nic", "subnet": { "id": subnet }, }] } @@ -142,29 +148,30 @@ async def start_node(log, host: str, nodespace: Dict[str, str], ssh_keys: str) - nic_result = poller.result() - VM_NAME = "ExampleVM" - USERNAME = "azureuser" - PASSWORD = "ChangePa$$w0rd24" - print(f"Provisioning virtual machine {VM_NAME}; this operation might take a few minutes.") - poller = compute_client.virtual_machines.begin_create_or_update(resource_group, VM_NAME, + poller = compute_client.virtual_machines.begin_create_or_update(resource_group, host, { "location": region, "storage_profile": { "image_reference": { - "publisher": 'Canonical', - "offer": "UbuntuServer", - "sku": "16.04.0-LTS", + "publisher": "OpenLogic", + "offer": "CentOS", + "sku": "8_4-gen2", "version": "latest" } }, "hardware_profile": { - "vm_size": "Standard_DS1_v2" + "vm_size": "Standard_D4s_v3" }, "os_profile": { - "computer_name": VM_NAME, - "admin_username": USERNAME, + "computer_name": host, + "admin_username": "centos", + "linux_configuration": { + "ssh": { + "public_keys" : [ ssh_keys ] + } + } "admin_password": PASSWORD }, "network_profile": { From e0e5c08624bb2bff2699478130b84481736fb2aa Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Fri, 26 Nov 2021 15:56:23 +0100 Subject: [PATCH 18/67] citc_azure --- roles/slurm/files/citc_azure.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/roles/slurm/files/citc_azure.py b/roles/slurm/files/citc_azure.py index f3514596..d974ae2e 100644 --- a/roles/slurm/files/citc_azure.py +++ b/roles/slurm/files/citc_azure.py @@ -136,11 +136,11 @@ async def start_node(log, host: str, nodespace: Dict[str, str], ssh_keys: str) - log.info(f"{host}: host is currently terminating. Waiting...") await asyncio.sleep(5) - poller = network_client.network_interfaces.begin_create_or_update(resource_group,host-"nic", + poller = network_client.network_interfaces.begin_create_or_update(resource_group,host+"-nic", { "location": region, "ip_configurations": [ { - "name": host-"nic", + "name": host+"-nic", "subnet": { "id": subnet }, }] } @@ -172,7 +172,6 @@ async def start_node(log, host: str, nodespace: Dict[str, str], ssh_keys: str) - "public_keys" : [ ssh_keys ] } } - "admin_password": PASSWORD }, "network_profile": { "network_interfaces": [{ From 25486110c7b8c69ee7286cc021c6ac2a6e701c44 Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Fri, 26 Nov 2021 16:16:36 +0100 Subject: [PATCH 19/67] update_config and citc_azure --- roles/slurm/files/citc_azure.py | 7 +++++-- roles/slurm/files/update_config.py | 4 ++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/roles/slurm/files/citc_azure.py b/roles/slurm/files/citc_azure.py index d974ae2e..8f033b83 100644 --- a/roles/slurm/files/citc_azure.py +++ b/roles/slurm/files/citc_azure.py @@ -148,7 +148,7 @@ async def start_node(log, host: str, nodespace: Dict[str, str], ssh_keys: str) - nic_result = poller.result() - print(f"Provisioning virtual machine {VM_NAME}; this operation might take a few minutes.") + print(f"Provisioning virtual machine {host}; this operation might take a few minutes.") poller = compute_client.virtual_machines.begin_create_or_update(resource_group, host, { @@ -169,7 +169,10 @@ async def start_node(log, host: str, nodespace: Dict[str, str], ssh_keys: str) - "admin_username": "centos", "linux_configuration": { "ssh": { - "public_keys" : [ ssh_keys ] + "public_keys" : [ { + "path": "/home/centos/.ssh/authorized_keys", + "key_data": ssh_keys + } ] } } }, diff --git a/roles/slurm/files/update_config.py b/roles/slurm/files/update_config.py index 2ac46142..8ecea15b 100644 --- a/roles/slurm/files/update_config.py +++ b/roles/slurm/files/update_config.py @@ -40,9 +40,9 @@ def get_nodespace() -> Dict[str, Dict[str, str]]: def encode_nodename(shape_name: str, node_number: int, cluster_id: str, ad: Optional[int] = None) -> str: if ad is not None: - return "{}-ad{}-{:0>4}".format(shape_name.lower().replace(".", "-"), ad, node_number) + return "{}-ad{}-{:0>4}".format(shape_name.lower().replace(".", "-").replace("_", "-"), ad, node_number) else: - return "{}-{}-{:0>4}".format(cluster_id, shape_name.lower().replace(".", "-"), node_number) + return "{}-{}-{:0>4}".format(cluster_id, shape_name.lower().replace(".", "-").replace("_", "-"), node_number) def create_slurmconf_line(number: int, shape_info: Dict, shape: str, cluster_id, ad: Optional[int] = None): From f7199d555d542e0000cbb38fe402350c7d2fdd45 Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Fri, 26 Nov 2021 16:25:34 +0100 Subject: [PATCH 20/67] add user_data --- roles/slurm/files/citc_azure.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/roles/slurm/files/citc_azure.py b/roles/slurm/files/citc_azure.py index 8f033b83..a0bb24f8 100644 --- a/roles/slurm/files/citc_azure.py +++ b/roles/slurm/files/citc_azure.py @@ -132,6 +132,9 @@ async def start_node(log, host: str, nodespace: Dict[str, str], ssh_keys: str) - network_client = NetworkManagementClient(credential, subscription_id) compute_client = ComputeManagementClient(credential, subscription_id) + with open("/home/slurm/bootstrap.sh", "rb") as f: + user_data = base64.b64encode(f.read()).decode() + while get_node_state(compute_client, log, host, resource_group) == "TERMINATING": log.info(f"{host}: host is currently terminating. Waiting...") await asyncio.sleep(5) @@ -180,7 +183,8 @@ async def start_node(log, host: str, nodespace: Dict[str, str], ssh_keys: str) - "network_interfaces": [{ "id": nic_result.id, }] - } + }, + "user_data": user_data, } ) vm_result = poller.result() From 4a47e349124127578c4d51d709c01708b958bd14 Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Mon, 29 Nov 2021 09:43:13 +0100 Subject: [PATCH 21/67] add user_data --- roles/slurm/files/citc_azure.py | 73 +++++++++++++++++++++------------ 1 file changed, 47 insertions(+), 26 deletions(-) diff --git a/roles/slurm/files/citc_azure.py b/roles/slurm/files/citc_azure.py index a0bb24f8..6c6dc8fb 100644 --- a/roles/slurm/files/citc_azure.py +++ b/roles/slurm/files/citc_azure.py @@ -67,8 +67,8 @@ def get_image() -> str: raise RuntimeError("Could not locate the image for the compute node") -#def create_node_config(oci_config, hostname: str, ip: Optional[str], nodespace: Dict[str, str], ssh_keys: str) -> oci.core.models.LaunchInstanceDetails: -def create_node_config() -> str: +def create_node_config(hostname: str, ip: Optional[str], nodespace: Dict[str, str], ssh_keys: str) -> instance_details: +#def create_node_config() -> str: """ Create the configuration needed to create ``hostname`` in ``nodespace`` with ``ssh_keys`` """ @@ -79,30 +79,48 @@ def create_node_config() -> str: #subnet = get_subnet(oci_config, nodespace["compartment_id"], nodespace["vcn_id"], nodespace["cluster_id"]) #image = get_image(oci_config, nodespace["compartment_id"], nodespace["cluster_id"]) - #with open("/home/slurm/bootstrap.sh", "rb") as f: - # user_data = base64.b64encode(f.read()).decode() - - #instance_details = oci.core.models.LaunchInstanceDetails( - # compartment_id=nodespace["compartment_id"], - # availability_domain=ad, - # shape=shape, - # subnet_id=subnet, - # image_id=image.id, - # display_name=hostname, - # hostname_label=hostname, - # create_vnic_details=oci.core.models.CreateVnicDetails(private_ip=ip, subnet_id=subnet) if ip else None, - # metadata={ - # "ssh_authorized_keys": ssh_keys.strip(), - # "user_data": user_data, - # }, - # freeform_tags={ - # "type": "compute", - # "cluster": nodespace["cluster_id"], - # }, - #) - - #return instance_details - return "instance_details" + with open("/home/slurm/bootstrap.sh", "rb") as f: + user_data = base64.b64encode(f.read()).decode() + + subscription_id = nodespace["subscription"] + resource_group = nodespace["resource_group"] + region = nodespace["region"] + subnet = nodespace["subnet"] + + instance_details = '{ + "location": region, + "storage_profile": { + "image_reference": { + "publisher": "OpenLogic", + "offer": "CentOS", + "sku": "8_4-gen2", + "version": "latest" + } + }, + "hardware_profile": { + "vm_size": "Standard_D4s_v3" + }, + "os_profile": { + "computer_name": host, + "admin_username": "centos", + "linux_configuration": { + "ssh": { + "public_keys" : [ { + "path": "/home/centos/.ssh/authorized_keys", + "key_data": ssh_keys + } ] + } + } + }, + "network_profile": { + "network_interfaces": [{ + "id": nic_result.id, + }] + }, + "user_data": user_data, + }' + + return instance_details #def get_ip(hostname: str) -> Tuple[Optional[str], Optional[str], Optional[str]]: @@ -139,6 +157,9 @@ async def start_node(log, host: str, nodespace: Dict[str, str], ssh_keys: str) - log.info(f"{host}: host is currently terminating. Waiting...") await asyncio.sleep(5) + instance_details = create_node_config(host, ip, nodespace, ssh_keys) + print(instance_details) + poller = network_client.network_interfaces.begin_create_or_update(resource_group,host+"-nic", { "location": region, From 4d471cad6b20970a0b1d88e5ad59cba16cac4dc7 Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Mon, 29 Nov 2021 09:46:15 +0100 Subject: [PATCH 22/67] add user_data --- roles/slurm/files/citc_azure.py | 46 ++++++++++++++++----------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/roles/slurm/files/citc_azure.py b/roles/slurm/files/citc_azure.py index 6c6dc8fb..deccdc0a 100644 --- a/roles/slurm/files/citc_azure.py +++ b/roles/slurm/files/citc_azure.py @@ -87,38 +87,38 @@ def create_node_config(hostname: str, ip: Optional[str], nodespace: Dict[str, st region = nodespace["region"] subnet = nodespace["subnet"] - instance_details = '{ - "location": region, - "storage_profile": { - "image_reference": { - "publisher": "OpenLogic", - "offer": "CentOS", - "sku": "8_4-gen2", - "version": "latest" + instance_details = "{ + \"location\": region, + \"storage_profile\": { + \"image_reference\": { + \"publisher\": \"OpenLogic\", + \"offer\": \"CentOS\", + \"sku\": \"8_4-gen2\", + \"version\": \"latest\" } }, - "hardware_profile": { - "vm_size": "Standard_D4s_v3" + \"hardware_profile\": { + \"vm_size\": \"Standard_D4s_v3\" }, - "os_profile": { - "computer_name": host, - "admin_username": "centos", - "linux_configuration": { - "ssh": { - "public_keys" : [ { - "path": "/home/centos/.ssh/authorized_keys", - "key_data": ssh_keys + \"os_profile\": { + \"computer_name\": host, + \"admin_username\": \"centos\", + \"linux_configuration\": { + \"ssh\": { + \"public_keys\" : [ { + \"path\": \"/home/centos/.ssh/authorized_keys\", + \"key_data\": ssh_keys } ] } } }, - "network_profile": { - "network_interfaces": [{ - "id": nic_result.id, + \"network_profile\": { + \"network_interfaces\": [{ + \"id\": nic_result.id, }] }, - "user_data": user_data, - }' + \"user_data\": user_data, + }" return instance_details From e58725835679dcbe3f716cb58f771e794ee85c31 Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Mon, 29 Nov 2021 09:47:49 +0100 Subject: [PATCH 23/67] add user_data --- roles/slurm/files/citc_azure.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/slurm/files/citc_azure.py b/roles/slurm/files/citc_azure.py index deccdc0a..f0251503 100644 --- a/roles/slurm/files/citc_azure.py +++ b/roles/slurm/files/citc_azure.py @@ -87,7 +87,7 @@ def create_node_config(hostname: str, ip: Optional[str], nodespace: Dict[str, st region = nodespace["region"] subnet = nodespace["subnet"] - instance_details = "{ + instance_details = """{ \"location\": region, \"storage_profile\": { \"image_reference\": { @@ -118,7 +118,7 @@ def create_node_config(hostname: str, ip: Optional[str], nodespace: Dict[str, st }] }, \"user_data\": user_data, - }" + }""" return instance_details From f6ec63e88eb58022621c9a51c24365586e39a773 Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Mon, 29 Nov 2021 10:13:12 +0100 Subject: [PATCH 24/67] add user_data --- roles/slurm/files/citc_azure.py | 56 ++++++++++++++------------------- 1 file changed, 24 insertions(+), 32 deletions(-) diff --git a/roles/slurm/files/citc_azure.py b/roles/slurm/files/citc_azure.py index f0251503..d16b59b8 100644 --- a/roles/slurm/files/citc_azure.py +++ b/roles/slurm/files/citc_azure.py @@ -87,38 +87,29 @@ def create_node_config(hostname: str, ip: Optional[str], nodespace: Dict[str, st region = nodespace["region"] subnet = nodespace["subnet"] - instance_details = """{ - \"location\": region, - \"storage_profile\": { - \"image_reference\": { - \"publisher\": \"OpenLogic\", - \"offer\": \"CentOS\", - \"sku\": \"8_4-gen2\", - \"version\": \"latest\" - } - }, - \"hardware_profile\": { - \"vm_size\": \"Standard_D4s_v3\" - }, - \"os_profile\": { - \"computer_name\": host, - \"admin_username\": \"centos\", - \"linux_configuration\": { - \"ssh\": { - \"public_keys\" : [ { - \"path\": \"/home/centos/.ssh/authorized_keys\", - \"key_data\": ssh_keys - } ] - } - } - }, - \"network_profile\": { - \"network_interfaces\": [{ - \"id\": nic_result.id, - }] - }, - \"user_data\": user_data, - }""" + instance_details = "{ \"location\": "+region+"," \ + "\"storage_profile\": {"+ \ + "\"image_reference\": {"+ \ + "\"publisher\": \"OpenLogic\","+ \ + "\"offer\": \"CentOS\","+ \ + "\"sku\": \"8_4-gen2\","+ \ + "\"version\": \"latest\" } },"+ \ + "\"hardware_profile\": {"+ \ + "\"vm_size\": \"Standard_D4s_v3\"},"+ \ + "\"os_profile\": {"+ \ + "\"computer_name\":"+ host +","+ \ + "\"admin_username\": \"centos\","+ \ + "\"linux_configuration\": {" + \ + "\"ssh\": { "+ \ + "\"public_keys\" : [ { "+ \ + "\"path\": \"/home/centos/.ssh/authorized_keys\","+ \ + "\"key_data\":"+ ssh_keys +\ + "} ]}}},"+ \ + "\"network_profile\": {"+ \ + "\"network_interfaces\": [{"+ \ + "\"id\":"+ nic_result.id +","+ \ + "}]},"+ \ + "\"user_data\":"+ user_data + ",}" return instance_details @@ -146,6 +137,7 @@ async def start_node(log, host: str, nodespace: Dict[str, str], ssh_keys: str) - resource_group = nodespace["resource_group"] region = nodespace["region"] subnet = nodespace["subnet"] + ip = "ip" resource_client = ResourceManagementClient(credential, subscription_id) network_client = NetworkManagementClient(credential, subscription_id) compute_client = ComputeManagementClient(credential, subscription_id) From 4d41c59c1ac86e773987941bcdec4356bcf27b1e Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Mon, 29 Nov 2021 10:14:19 +0100 Subject: [PATCH 25/67] add user_data --- roles/slurm/files/citc_azure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/slurm/files/citc_azure.py b/roles/slurm/files/citc_azure.py index d16b59b8..f949bf74 100644 --- a/roles/slurm/files/citc_azure.py +++ b/roles/slurm/files/citc_azure.py @@ -67,7 +67,7 @@ def get_image() -> str: raise RuntimeError("Could not locate the image for the compute node") -def create_node_config(hostname: str, ip: Optional[str], nodespace: Dict[str, str], ssh_keys: str) -> instance_details: +def create_node_config(hostname: str, ip: Optional[str], nodespace: Dict[str, str], ssh_keys: str): #def create_node_config() -> str: """ Create the configuration needed to create ``hostname`` in ``nodespace`` with ``ssh_keys`` From 367d5fe74925d4931989c07bc9e489889c924dcd Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Mon, 29 Nov 2021 10:15:46 +0100 Subject: [PATCH 26/67] add user_data --- roles/slurm/files/citc_azure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/slurm/files/citc_azure.py b/roles/slurm/files/citc_azure.py index f949bf74..e250e60a 100644 --- a/roles/slurm/files/citc_azure.py +++ b/roles/slurm/files/citc_azure.py @@ -67,7 +67,7 @@ def get_image() -> str: raise RuntimeError("Could not locate the image for the compute node") -def create_node_config(hostname: str, ip: Optional[str], nodespace: Dict[str, str], ssh_keys: str): +def create_node_config(host: str, ip: Optional[str], nodespace: Dict[str, str], ssh_keys: str): #def create_node_config() -> str: """ Create the configuration needed to create ``hostname`` in ``nodespace`` with ``ssh_keys`` From 0a6e767deba10467004c9403e2cdfa745fa7a1c9 Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Mon, 29 Nov 2021 10:21:54 +0100 Subject: [PATCH 27/67] add user_data --- roles/slurm/files/citc_azure.py | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/roles/slurm/files/citc_azure.py b/roles/slurm/files/citc_azure.py index e250e60a..bfb1877d 100644 --- a/roles/slurm/files/citc_azure.py +++ b/roles/slurm/files/citc_azure.py @@ -149,8 +149,6 @@ async def start_node(log, host: str, nodespace: Dict[str, str], ssh_keys: str) - log.info(f"{host}: host is currently terminating. Waiting...") await asyncio.sleep(5) - instance_details = create_node_config(host, ip, nodespace, ssh_keys) - print(instance_details) poller = network_client.network_interfaces.begin_create_or_update(resource_group,host+"-nic", { @@ -164,6 +162,31 @@ async def start_node(log, host: str, nodespace: Dict[str, str], ssh_keys: str) - nic_result = poller.result() + instance_details = "{ \"location\": "+region+"," \ + "\"storage_profile\": {"+ \ + "\"image_reference\": {"+ \ + "\"publisher\": \"OpenLogic\","+ \ + "\"offer\": \"CentOS\","+ \ + "\"sku\": \"8_4-gen2\","+ \ + "\"version\": \"latest\" } },"+ \ + "\"hardware_profile\": {"+ \ + "\"vm_size\": \"Standard_D4s_v3\"},"+ \ + "\"os_profile\": {"+ \ + "\"computer_name\":"+ host +","+ \ + "\"admin_username\": \"centos\","+ \ + "\"linux_configuration\": {" + \ + "\"ssh\": { "+ \ + "\"public_keys\" : [ { "+ \ + "\"path\": \"/home/centos/.ssh/authorized_keys\","+ \ + "\"key_data\":"+ ssh_keys +\ + "} ]}}},"+ \ + "\"network_profile\": {"+ \ + "\"network_interfaces\": [{"+ \ + "\"id\":"+ nic_result.id +","+ \ + "}]},"+ \ + "\"user_data\":"+ user_data + ",}" + + print(instance_details) print(f"Provisioning virtual machine {host}; this operation might take a few minutes.") poller = compute_client.virtual_machines.begin_create_or_update(resource_group, host, From 4ef2dbfb9187fe56b3dacc21d5dcc60741afb99e Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Mon, 29 Nov 2021 10:30:42 +0100 Subject: [PATCH 28/67] add user_data --- roles/slurm/files/citc_azure.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/roles/slurm/files/citc_azure.py b/roles/slurm/files/citc_azure.py index bfb1877d..70a5234e 100644 --- a/roles/slurm/files/citc_azure.py +++ b/roles/slurm/files/citc_azure.py @@ -189,8 +189,8 @@ async def start_node(log, host: str, nodespace: Dict[str, str], ssh_keys: str) - print(instance_details) print(f"Provisioning virtual machine {host}; this operation might take a few minutes.") - poller = compute_client.virtual_machines.begin_create_or_update(resource_group, host, - { + poller = compute_client.virtual_machines.begin_create_or_update(resource_group, host, instance_details) + bla = """ { "location": region, "storage_profile": { "image_reference": { @@ -221,8 +221,8 @@ async def start_node(log, host: str, nodespace: Dict[str, str], ssh_keys: str) - }] }, "user_data": user_data, - } - ) + }""" + vm_result = poller.result() print(f"Provisioned virtual machine {vm_result.name}") From 4675ab66f7fb0e403225eeb0fe2e1b2caf4997b4 Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Mon, 29 Nov 2021 10:51:48 +0100 Subject: [PATCH 29/67] packer --- roles/packer/files/all.pkr.hcl | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/roles/packer/files/all.pkr.hcl b/roles/packer/files/all.pkr.hcl index f92c810d..3bb8760f 100644 --- a/roles/packer/files/all.pkr.hcl +++ b/roles/packer/files/all.pkr.hcl @@ -10,6 +10,10 @@ variable "aws_region" {} variable "aws_instance_type" {} variable "aws_arch" {} +variable "azure_region" {} +variable "azure_instance_type" {} +variable "azure_resource_group" {} + variable "oracle_availability_domain" {} variable "oracle_base_image_ocid" {} variable "oracle_compartment_ocid" {} @@ -91,6 +95,13 @@ source "amazon-ebs" "aws" { } } +source "azure-arm" "azure" { + image_name = "${var.destination_image_name}-${var.cluster}-v{{timestamp}}" + ssh_username = var.ssh_username + location = var.azure_region +} + + source "oracle-oci" "oracle" { image_name = "${var.destination_image_name}-${var.cluster}-v{{timestamp}}" availability_domain = var.oracle_availability_domain From 7a2c352ddf1e6732cdde5a5262fc0bccc97dce3e Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Mon, 29 Nov 2021 10:56:36 +0100 Subject: [PATCH 30/67] packer --- roles/packer/templates/variables.pkrvars.hcl.j2 | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/roles/packer/templates/variables.pkrvars.hcl.j2 b/roles/packer/templates/variables.pkrvars.hcl.j2 index 02d22f0e..e64a8839 100644 --- a/roles/packer/templates/variables.pkrvars.hcl.j2 +++ b/roles/packer/templates/variables.pkrvars.hcl.j2 @@ -1,12 +1,16 @@ ca_cert = "{{ ca_cert }}" cluster = "{{ startnode_config.cluster_id }}" destination_image_name = "citc-slurm-compute" -ssh_username = "{%- if ansible_local.citc.csp in ["aws", "google"] -%}centos{%- else -%}opc{%- endif -%}" +ssh_username = "{%- if ansible_local.citc.csp in ["aws", "google", "azure"] -%}centos{%- else -%}opc{%- endif -%}" aws_arch = "x86_64" aws_region = "{%- if startnode_config.region is defined -%}{{ startnode_config.region }}{%- endif -%}" aws_instance_type = "t2.nano" +azure_region = "{%- if startnode_config.region is defined -%}{{ startnode_config.region }}{%- endif -%}" +azure_recource_group = "{%- if startnode_config.recource_group is defined -%}{{ startnode_config.recource_group }}{%- endif -%}" +azure_instance_type = "Standard_D4s_v3" + google_destination_image_family = "citc-slurm-compute" google_network = "{%- if startnode_config.network_name is defined -%}{{ startnode_config.network_name }}{%- endif -%}" google_source_image_family = "centos-8" From f43c5e01d8e34bfd1fe0300b43d1b87ef9010256 Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Mon, 29 Nov 2021 12:42:31 +0100 Subject: [PATCH 31/67] packer --- roles/packer/files/all.pkr.hcl | 8 +++++++- roles/packer/templates/variables.pkrvars.hcl.j2 | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/roles/packer/files/all.pkr.hcl b/roles/packer/files/all.pkr.hcl index 3bb8760f..1aa744f6 100644 --- a/roles/packer/files/all.pkr.hcl +++ b/roles/packer/files/all.pkr.hcl @@ -96,9 +96,14 @@ source "amazon-ebs" "aws" { } source "azure-arm" "azure" { - image_name = "${var.destination_image_name}-${var.cluster}-v{{timestamp}}" + managed_image_name = "${var.destination_image_name}-${var.cluster}-v{{timestamp}}" + managed_image_resource_group_name = var.azure_resource_group ssh_username = var.ssh_username location = var.azure_region + os_type = "Linux" + image_publisher = "OpenLogic" + image_offer = "CentOS" + image_sku = "8_4-gen2" } @@ -122,6 +127,7 @@ build { "source.googlecompute.google", "source.amazon-ebs.aws", "source.oracle-oci.oracle", + "source.azure-arm.azure", ] provisioner "file" { diff --git a/roles/packer/templates/variables.pkrvars.hcl.j2 b/roles/packer/templates/variables.pkrvars.hcl.j2 index e64a8839..ebbbce7f 100644 --- a/roles/packer/templates/variables.pkrvars.hcl.j2 +++ b/roles/packer/templates/variables.pkrvars.hcl.j2 @@ -8,7 +8,7 @@ aws_region = "{%- if startnode_config.region is defined -%}{{ startnode_config.r aws_instance_type = "t2.nano" azure_region = "{%- if startnode_config.region is defined -%}{{ startnode_config.region }}{%- endif -%}" -azure_recource_group = "{%- if startnode_config.recource_group is defined -%}{{ startnode_config.recource_group }}{%- endif -%}" +azure_resource_group = "{%- if startnode_config.resource_group is defined -%}{{ startnode_config.resource_group }}{%- endif -%}" azure_instance_type = "Standard_D4s_v3" google_destination_image_family = "citc-slurm-compute" From 1bc5f6ea16a9dcdbc1586130cb6bd3cd9d4875ab Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Mon, 29 Nov 2021 13:45:30 +0100 Subject: [PATCH 32/67] packer --- roles/packer/files/all.pkr.hcl | 4 +++- roles/packer/templates/prepare_ansible.sh.j2 | 2 +- roles/packer/templates/variables.pkrvars.hcl.j2 | 2 ++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/roles/packer/files/all.pkr.hcl b/roles/packer/files/all.pkr.hcl index 1aa744f6..74084d28 100644 --- a/roles/packer/files/all.pkr.hcl +++ b/roles/packer/files/all.pkr.hcl @@ -98,8 +98,10 @@ source "amazon-ebs" "aws" { source "azure-arm" "azure" { managed_image_name = "${var.destination_image_name}-${var.cluster}-v{{timestamp}}" managed_image_resource_group_name = var.azure_resource_group + build_resource_group_name = var.azure_resource_group + virtual_network_name = var.azure_virtual_network + virtual_network_subnet_name = var.azure_virtual_network_subnet ssh_username = var.ssh_username - location = var.azure_region os_type = "Linux" image_publisher = "OpenLogic" image_offer = "CentOS" diff --git a/roles/packer/templates/prepare_ansible.sh.j2 b/roles/packer/templates/prepare_ansible.sh.j2 index 0637bb52..3a404f3f 100644 --- a/roles/packer/templates/prepare_ansible.sh.j2 +++ b/roles/packer/templates/prepare_ansible.sh.j2 @@ -9,7 +9,7 @@ $(hostname) cluster_id={{ startnode_config.cluster_id }} packer_run=yes EOF' -{% if ansible_local.citc.csp in ["aws", "google"] %} +{% if ansible_local.citc.csp in ["aws", "google", "azure"] %} sudo yum install -y epel-release sudo dnf config-manager --set-enabled powertools {% elif ansible_local.citc.csp == "oracle" %} diff --git a/roles/packer/templates/variables.pkrvars.hcl.j2 b/roles/packer/templates/variables.pkrvars.hcl.j2 index ebbbce7f..f26deac5 100644 --- a/roles/packer/templates/variables.pkrvars.hcl.j2 +++ b/roles/packer/templates/variables.pkrvars.hcl.j2 @@ -9,6 +9,8 @@ aws_instance_type = "t2.nano" azure_region = "{%- if startnode_config.region is defined -%}{{ startnode_config.region }}{%- endif -%}" azure_resource_group = "{%- if startnode_config.resource_group is defined -%}{{ startnode_config.resource_group }}{%- endif -%}" +azure_virtual_network = "{%- if startnode_config.virtual_network is defined -%}{{ startnode_config.virtual_network }}{%- endif -%}" +azure_virtual_network_subnet = "{%- if startnode_config.virtual_network_subnet is defined -%}{{ startnode_config.virtual_network_subnet }}{%- endif -%}" azure_instance_type = "Standard_D4s_v3" google_destination_image_family = "citc-slurm-compute" From abe1cab58b6bd17c90470a859fd46a04bdfd40f6 Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Mon, 29 Nov 2021 18:46:09 +0100 Subject: [PATCH 33/67] dns zone compute --- roles/filesystem/tasks/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/filesystem/tasks/main.yml b/roles/filesystem/tasks/main.yml index 7bbdcf5d..6e7c05fc 100644 --- a/roles/filesystem/tasks/main.yml +++ b/roles/filesystem/tasks/main.yml @@ -6,11 +6,12 @@ - name: ensure fileserver name exists lineinfile: dest: /etc/hosts - line: "{{ ansible_local.citc.fileserver_ip }} fileserver" + line: "{{ ansible_local.citc.fileserver_ip }} fileserver fileserver.test.zone" state: present when: - ansible_local.citc is defined - ansible_local.citc.csp == "google" + - ansible_local.citc.csp == "azure" - name: Wait for fileserver to be ready checking nfs tcp port open (critical path - avoids race condition) wait_for: From 22bd6f89ca5c36ef897321fa570f54a35a9b099d Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Tue, 30 Nov 2021 09:34:31 +0100 Subject: [PATCH 34/67] packer --- roles/packer/files/all.pkr.hcl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/packer/files/all.pkr.hcl b/roles/packer/files/all.pkr.hcl index 74084d28..2060b65f 100644 --- a/roles/packer/files/all.pkr.hcl +++ b/roles/packer/files/all.pkr.hcl @@ -101,6 +101,8 @@ source "azure-arm" "azure" { build_resource_group_name = var.azure_resource_group virtual_network_name = var.azure_virtual_network virtual_network_subnet_name = var.azure_virtual_network_subnet + virtual_network_resource_group_name = var.azure_resource_group + vm_size = var.azure_instance_type ssh_username = var.ssh_username os_type = "Linux" image_publisher = "OpenLogic" From 4686f255b47c384beee068c4a6a22baa838b3e95 Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Tue, 30 Nov 2021 09:58:58 +0100 Subject: [PATCH 35/67] packer --- roles/packer/files/all.pkr.hcl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/packer/files/all.pkr.hcl b/roles/packer/files/all.pkr.hcl index 2060b65f..e99d46f0 100644 --- a/roles/packer/files/all.pkr.hcl +++ b/roles/packer/files/all.pkr.hcl @@ -13,6 +13,8 @@ variable "aws_arch" {} variable "azure_region" {} variable "azure_instance_type" {} variable "azure_resource_group" {} +variable "azure_virtual_network" {} +variable "azure_virtual_network_subnet" {} variable "oracle_availability_domain" {} variable "oracle_base_image_ocid" {} From 99076ed11593275795bc23a62900d0523412efca Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Tue, 30 Nov 2021 12:30:35 +0100 Subject: [PATCH 36/67] packer --- roles/packer/files/all.pkr.hcl | 4 ++++ roles/packer/files/compute_image_finalize.sh | 3 +++ roles/packer/tasks/main.yml | 8 ++++++++ 3 files changed, 15 insertions(+) create mode 100644 roles/packer/files/compute_image_finalize.sh diff --git a/roles/packer/files/all.pkr.hcl b/roles/packer/files/all.pkr.hcl index e99d46f0..ccd6c95f 100644 --- a/roles/packer/files/all.pkr.hcl +++ b/roles/packer/files/all.pkr.hcl @@ -184,4 +184,8 @@ build { provisioner "shell" { script = "/home/citc/compute_image_extra.sh" } + + provisioner "shell" { + script = "/home/citc/compute_image_finalize.sh" + } } diff --git a/roles/packer/files/compute_image_finalize.sh b/roles/packer/files/compute_image_finalize.sh new file mode 100644 index 00000000..c1a5c890 --- /dev/null +++ b/roles/packer/files/compute_image_finalize.sh @@ -0,0 +1,3 @@ +#! /bin/bash + +/usr/sbin/waagent -force -deprovision+user && export HISTSIZE=0 && sync diff --git a/roles/packer/tasks/main.yml b/roles/packer/tasks/main.yml index f05ac85a..4494157d 100644 --- a/roles/packer/tasks/main.yml +++ b/roles/packer/tasks/main.yml @@ -65,6 +65,14 @@ group: citc mode: u=rw,g=rw,o= +- name: copy in packer finalize run script template + copy: + src: compute_image_finalize.sh + dest: /home/citc/compute_image_finalize.sh + owner: citc + group: citc + mode: u=rw,g=rw,o= + - name: copy in EESSI install script copy: src: install_cvmfs_eessi.sh From 4c3d79215943eb742f00b07d69d4088e23a3d652 Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Tue, 30 Nov 2021 13:02:26 +0100 Subject: [PATCH 37/67] packer --- roles/slurm/files/citc_azure.py | 38 ++++++--------------------------- 1 file changed, 6 insertions(+), 32 deletions(-) diff --git a/roles/slurm/files/citc_azure.py b/roles/slurm/files/citc_azure.py index 70a5234e..752bb874 100644 --- a/roles/slurm/files/citc_azure.py +++ b/roles/slurm/files/citc_azure.py @@ -149,6 +149,9 @@ async def start_node(log, host: str, nodespace: Dict[str, str], ssh_keys: str) - log.info(f"{host}: host is currently terminating. Waiting...") await asyncio.sleep(5) + images = compute_client.images.list_by_resource_group(resource_group) + for image in images: + vm_image = str(image.id) poller = network_client.network_interfaces.begin_create_or_update(resource_group,host+"-nic", { @@ -162,42 +165,13 @@ async def start_node(log, host: str, nodespace: Dict[str, str], ssh_keys: str) - nic_result = poller.result() - instance_details = "{ \"location\": "+region+"," \ - "\"storage_profile\": {"+ \ - "\"image_reference\": {"+ \ - "\"publisher\": \"OpenLogic\","+ \ - "\"offer\": \"CentOS\","+ \ - "\"sku\": \"8_4-gen2\","+ \ - "\"version\": \"latest\" } },"+ \ - "\"hardware_profile\": {"+ \ - "\"vm_size\": \"Standard_D4s_v3\"},"+ \ - "\"os_profile\": {"+ \ - "\"computer_name\":"+ host +","+ \ - "\"admin_username\": \"centos\","+ \ - "\"linux_configuration\": {" + \ - "\"ssh\": { "+ \ - "\"public_keys\" : [ { "+ \ - "\"path\": \"/home/centos/.ssh/authorized_keys\","+ \ - "\"key_data\":"+ ssh_keys +\ - "} ]}}},"+ \ - "\"network_profile\": {"+ \ - "\"network_interfaces\": [{"+ \ - "\"id\":"+ nic_result.id +","+ \ - "}]},"+ \ - "\"user_data\":"+ user_data + ",}" - - print(instance_details) print(f"Provisioning virtual machine {host}; this operation might take a few minutes.") - poller = compute_client.virtual_machines.begin_create_or_update(resource_group, host, instance_details) - bla = """ { + poller = compute_client.virtual_machines.begin_create_or_update(resource_group, host, { "location": region, "storage_profile": { "image_reference": { - "publisher": "OpenLogic", - "offer": "CentOS", - "sku": "8_4-gen2", - "version": "latest" + "id": vm_image, } }, "hardware_profile": { @@ -221,7 +195,7 @@ async def start_node(log, host: str, nodespace: Dict[str, str], ssh_keys: str) - }] }, "user_data": user_data, - }""" + }) vm_result = poller.result() From cc304e8fc423396b695c312c96d369f80ab68dd2 Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Tue, 30 Nov 2021 14:20:18 +0100 Subject: [PATCH 38/67] packer --- roles/filesystem/tasks/main.yml | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/roles/filesystem/tasks/main.yml b/roles/filesystem/tasks/main.yml index 6e7c05fc..620e7e62 100644 --- a/roles/filesystem/tasks/main.yml +++ b/roles/filesystem/tasks/main.yml @@ -6,12 +6,11 @@ - name: ensure fileserver name exists lineinfile: dest: /etc/hosts - line: "{{ ansible_local.citc.fileserver_ip }} fileserver fileserver.test.zone" + line: "{{ ansible_local.citc.fileserver_ip }} fileserver" state: present when: - ansible_local.citc is defined - ansible_local.citc.csp == "google" - - ansible_local.citc.csp == "azure" - name: Wait for fileserver to be ready checking nfs tcp port open (critical path - avoids race condition) wait_for: @@ -27,7 +26,18 @@ opts: defaults,nofail,nosuid state: mounted when: - ansible_local.citc.csp != "aws" + - ansible_local.citc.csp != "aws" + - ansible_local.citc.csp != "azure" + +- name: Mount shared file system now that fileserver is ready + mount: + path: /mnt/{{ filesystem_mount_point }} + src: "{{ ansible_local.citc.fileserver_ip }}:{{ filesystem_mount_point }}" + fstype: nfs + opts: defaults,nofail,nosuid + state: mounted + when: + ansible_local.citc.csp == "azure" - name: Mount shared file system mount: From 393793a5cfe3c686729dd7eb9da47fbb996fe42d Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Tue, 30 Nov 2021 15:15:54 +0100 Subject: [PATCH 39/67] packer --- roles/slurm/tasks/elastic.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/slurm/tasks/elastic.yml b/roles/slurm/tasks/elastic.yml index b6a038d6..8c55d780 100644 --- a/roles/slurm/tasks/elastic.yml +++ b/roles/slurm/tasks/elastic.yml @@ -125,7 +125,7 @@ {% endif %} # Ensure that slurmd is running at this point - systemctl start slurmd + #systemctl start slurmd date dest: /home/slurm/bootstrap.sh From dac6dfd695398bcfd73aa2599b8d4f1ff04b1025 Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Tue, 30 Nov 2021 15:17:40 +0100 Subject: [PATCH 40/67] packer --- roles/packer/files/compute_image_finalize.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/packer/files/compute_image_finalize.sh b/roles/packer/files/compute_image_finalize.sh index c1a5c890..cd3d5b29 100644 --- a/roles/packer/files/compute_image_finalize.sh +++ b/roles/packer/files/compute_image_finalize.sh @@ -1,3 +1,3 @@ #! /bin/bash -/usr/sbin/waagent -force -deprovision+user && export HISTSIZE=0 && sync +/usr/sbin/waagent -force -deprovision && export HISTSIZE=0 && sync From 2eda4d1792d3d1c3274128a8899296570cddf43f Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Thu, 2 Dec 2021 12:58:39 +0100 Subject: [PATCH 41/67] autoscaling --- roles/filesystem/tasks/main.yml | 20 ++++++++++---------- roles/slurm/tasks/elastic_azure.yml | 16 ---------------- 2 files changed, 10 insertions(+), 26 deletions(-) diff --git a/roles/filesystem/tasks/main.yml b/roles/filesystem/tasks/main.yml index 620e7e62..d5fb7e44 100644 --- a/roles/filesystem/tasks/main.yml +++ b/roles/filesystem/tasks/main.yml @@ -27,17 +27,17 @@ state: mounted when: - ansible_local.citc.csp != "aws" - - ansible_local.citc.csp != "azure" + # - ansible_local.citc.csp != "azure" -- name: Mount shared file system now that fileserver is ready - mount: - path: /mnt/{{ filesystem_mount_point }} - src: "{{ ansible_local.citc.fileserver_ip }}:{{ filesystem_mount_point }}" - fstype: nfs - opts: defaults,nofail,nosuid - state: mounted - when: - ansible_local.citc.csp == "azure" + #- name: Mount shared file system now that fileserver is ready + #mount: + #path: /mnt/{{ filesystem_mount_point }} + #src: "{{ ansible_local.citc.fileserver_ip }}:{{ filesystem_mount_point }}" + #fstype: nfs + #opts: defaults,nofail,nosuid + #state: mounted + #when: + #ansible_local.citc.csp == "azure" - name: Mount shared file system mount: diff --git a/roles/slurm/tasks/elastic_azure.yml b/roles/slurm/tasks/elastic_azure.yml index c29bcfd5..a49e7c78 100644 --- a/roles/slurm/tasks/elastic_azure.yml +++ b/roles/slurm/tasks/elastic_azure.yml @@ -17,22 +17,6 @@ group: slurm mode: 0755 - #- name: copy Azure private key - #copy: - #src: /home/opc/oci_api_key.pem - #dest: "{{ slurm_elastic.azure.config_directory }}/oci_api_key.pem" - #owner: slurm - #group: slurm - #mode: 0400 - - #- name: configure Azure config file - #copy: - #src: /home/opc/config - ##dest: "{{ slurm_elastic.azure.config_directory }}/config" - #owner: slurm - ##group: slurm - #mode: 0400 - - name: install startnode support module copy: src: citc_azure.py From 994a0aa28ce11bf8e192c1ca828f7500fec77a86 Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Thu, 2 Dec 2021 13:17:07 +0100 Subject: [PATCH 42/67] packer --- roles/packer/files/all.pkr.hcl | 2 ++ roles/packer/templates/variables.pkrvars.hcl.j2 | 1 + 2 files changed, 3 insertions(+) diff --git a/roles/packer/files/all.pkr.hcl b/roles/packer/files/all.pkr.hcl index ccd6c95f..d356033c 100644 --- a/roles/packer/files/all.pkr.hcl +++ b/roles/packer/files/all.pkr.hcl @@ -15,6 +15,7 @@ variable "azure_instance_type" {} variable "azure_resource_group" {} variable "azure_virtual_network" {} variable "azure_virtual_network_subnet" {} +variable "azure_dns_zone" {} variable "oracle_availability_domain" {} variable "oracle_base_image_ocid" {} @@ -104,6 +105,7 @@ source "azure-arm" "azure" { virtual_network_name = var.azure_virtual_network virtual_network_subnet_name = var.azure_virtual_network_subnet virtual_network_resource_group_name = var.azure_resource_group + temp_compute_name = "packer.${var.azure_dns_zone}" vm_size = var.azure_instance_type ssh_username = var.ssh_username os_type = "Linux" diff --git a/roles/packer/templates/variables.pkrvars.hcl.j2 b/roles/packer/templates/variables.pkrvars.hcl.j2 index f26deac5..b7059663 100644 --- a/roles/packer/templates/variables.pkrvars.hcl.j2 +++ b/roles/packer/templates/variables.pkrvars.hcl.j2 @@ -11,6 +11,7 @@ azure_region = "{%- if startnode_config.region is defined -%}{{ startnode_config azure_resource_group = "{%- if startnode_config.resource_group is defined -%}{{ startnode_config.resource_group }}{%- endif -%}" azure_virtual_network = "{%- if startnode_config.virtual_network is defined -%}{{ startnode_config.virtual_network }}{%- endif -%}" azure_virtual_network_subnet = "{%- if startnode_config.virtual_network_subnet is defined -%}{{ startnode_config.virtual_network_subnet }}{%- endif -%}" +azure_dns_zone = "{%- if startnode_config.dns_zone is defined -%}{{ startnode_config.dns_zone }}{%- endif -%}" azure_instance_type = "Standard_D4s_v3" google_destination_image_family = "citc-slurm-compute" From 3b4a386a51fdec5f82b581853bf9652ba1a0b17a Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Fri, 3 Dec 2021 15:44:30 +0100 Subject: [PATCH 43/67] dns_zone --- roles/slurm/files/citc_azure.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/slurm/files/citc_azure.py b/roles/slurm/files/citc_azure.py index 752bb874..a1e982a4 100644 --- a/roles/slurm/files/citc_azure.py +++ b/roles/slurm/files/citc_azure.py @@ -137,6 +137,7 @@ async def start_node(log, host: str, nodespace: Dict[str, str], ssh_keys: str) - resource_group = nodespace["resource_group"] region = nodespace["region"] subnet = nodespace["subnet"] + dns_zone = "."+nodespace["dns_zone"] ip = "ip" resource_client = ResourceManagementClient(credential, subscription_id) network_client = NetworkManagementClient(credential, subscription_id) @@ -178,7 +179,7 @@ async def start_node(log, host: str, nodespace: Dict[str, str], ssh_keys: str) - "vm_size": "Standard_D4s_v3" }, "os_profile": { - "computer_name": host, + "computer_name": host+dns_zone, "admin_username": "centos", "linux_configuration": { "ssh": { From d663eb9e30be025197bd9bac24ecaf80a9783a96 Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Wed, 8 Dec 2021 11:13:11 +0100 Subject: [PATCH 44/67] fixing boot --- compute.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/compute.yml b/compute.yml index 74414fd6..f3db0cad 100644 --- a/compute.yml +++ b/compute.yml @@ -12,13 +12,13 @@ become: yes roles: - citc_user - - filesystem - - ssh - #- security_updates - - ntp - - sssd - - lmod - - packages - - mpi - - slurm - - monitoring + #- filesystem + #- ssh + ##- security_updates + #- ntp + #- sssd + #- lmod + #- packages + #- mpi + #- slurm + #- monitoring From 1d2a976c3386e154ccfff000a2628acb8292423f Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Wed, 8 Dec 2021 11:47:59 +0100 Subject: [PATCH 45/67] fixing boot --- compute.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/compute.yml b/compute.yml index f3db0cad..f2bcdc20 100644 --- a/compute.yml +++ b/compute.yml @@ -12,13 +12,13 @@ become: yes roles: - citc_user - #- filesystem - #- ssh - ##- security_updates - #- ntp - #- sssd - #- lmod - #- packages - #- mpi - #- slurm - #- monitoring + - filesystem + #- ssh + ##- security_updates + #- ntp + #- sssd + #- lmod + #- packages + #- mpi + #- slurm + #- monitoring From ea9c7398d082691487e585cafab641ab9b0c7d81 Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Wed, 8 Dec 2021 14:29:21 +0100 Subject: [PATCH 46/67] azure --- roles/slurm/files/citc_azure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/slurm/files/citc_azure.py b/roles/slurm/files/citc_azure.py index a1e982a4..79a47011 100644 --- a/roles/slurm/files/citc_azure.py +++ b/roles/slurm/files/citc_azure.py @@ -179,7 +179,7 @@ async def start_node(log, host: str, nodespace: Dict[str, str], ssh_keys: str) - "vm_size": "Standard_D4s_v3" }, "os_profile": { - "computer_name": host+dns_zone, + "computer_name": host, "admin_username": "centos", "linux_configuration": { "ssh": { From fa396c3ca0fdd509285e1565cfe3579a685989a1 Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Wed, 8 Dec 2021 15:03:17 +0100 Subject: [PATCH 47/67] anf --- roles/filesystem/tasks/main.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/roles/filesystem/tasks/main.yml b/roles/filesystem/tasks/main.yml index d5fb7e44..eae9eb4d 100644 --- a/roles/filesystem/tasks/main.yml +++ b/roles/filesystem/tasks/main.yml @@ -27,17 +27,17 @@ state: mounted when: - ansible_local.citc.csp != "aws" - # - ansible_local.citc.csp != "azure" + - ansible_local.citc.csp != "azure" - #- name: Mount shared file system now that fileserver is ready - #mount: - #path: /mnt/{{ filesystem_mount_point }} - #src: "{{ ansible_local.citc.fileserver_ip }}:{{ filesystem_mount_point }}" - #fstype: nfs - #opts: defaults,nofail,nosuid - #state: mounted - #when: - #ansible_local.citc.csp == "azure" +- name: Mount shared file system now that fileserver is ready + mount: + path: /mnt/{{ filesystem_mount_point }} + src: "{{ filesystem_target_address }}:{{ filesystem_mount_point }}" + fstype: nfs + opts: rw,hard,rsize=1048576,wsize=1048576,vers=3,tcp,_netdev + state: mounted + when: + - ansible_local.citc.csp == "azure" - name: Mount shared file system mount: From d4bba71c79d8f5584055c8af8b6477f92eb22abc Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Wed, 8 Dec 2021 15:29:50 +0100 Subject: [PATCH 48/67] packer --- roles/packer/files/all.pkr.hcl | 1 - 1 file changed, 1 deletion(-) diff --git a/roles/packer/files/all.pkr.hcl b/roles/packer/files/all.pkr.hcl index d356033c..2884e2cc 100644 --- a/roles/packer/files/all.pkr.hcl +++ b/roles/packer/files/all.pkr.hcl @@ -105,7 +105,6 @@ source "azure-arm" "azure" { virtual_network_name = var.azure_virtual_network virtual_network_subnet_name = var.azure_virtual_network_subnet virtual_network_resource_group_name = var.azure_resource_group - temp_compute_name = "packer.${var.azure_dns_zone}" vm_size = var.azure_instance_type ssh_username = var.ssh_username os_type = "Linux" From 4aef1b2113f83368319a88b4ab8a9ab3a19f1427 Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Wed, 8 Dec 2021 15:42:05 +0100 Subject: [PATCH 49/67] azure dns --- roles/packer/templates/prepare_ansible.sh.j2 | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/roles/packer/templates/prepare_ansible.sh.j2 b/roles/packer/templates/prepare_ansible.sh.j2 index 3a404f3f..3f5f8c55 100644 --- a/roles/packer/templates/prepare_ansible.sh.j2 +++ b/roles/packer/templates/prepare_ansible.sh.j2 @@ -17,6 +17,11 @@ sudo dnf install -y oracle-epel-release-el8 sudo dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm sudo dnf config-manager --set-enabled ol8_codeready_builder {% endif %} +{% if ansible_local.citc.csp in ["azure"] %} +echo "[Main]" | sudo tee -a /etc/NetworkManager/conf.d/dhclient.conf +echo "dhcp=dhclient" | sudo tee -a /etc/NetworkManager/conf.d/dhclient.conf +echo "append domain-name \"{{ dns_zone }}\";" | sudo tee -s /etc/dhcp/dhclient.conf +{% endif %} sudo dnf install -y ansible git sudo cat /tmp/hosts sudo mkdir -p /etc/ansible/facts.d/ From e23bc1bd8674c317ce6a7503c76a925de0dde04c Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Wed, 8 Dec 2021 15:45:04 +0100 Subject: [PATCH 50/67] azure dns --- roles/packer/templates/prepare_ansible.sh.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/packer/templates/prepare_ansible.sh.j2 b/roles/packer/templates/prepare_ansible.sh.j2 index 3f5f8c55..cb07b631 100644 --- a/roles/packer/templates/prepare_ansible.sh.j2 +++ b/roles/packer/templates/prepare_ansible.sh.j2 @@ -20,7 +20,7 @@ sudo dnf config-manager --set-enabled ol8_codeready_builder {% if ansible_local.citc.csp in ["azure"] %} echo "[Main]" | sudo tee -a /etc/NetworkManager/conf.d/dhclient.conf echo "dhcp=dhclient" | sudo tee -a /etc/NetworkManager/conf.d/dhclient.conf -echo "append domain-name \"{{ dns_zone }}\";" | sudo tee -s /etc/dhcp/dhclient.conf +echo "append domain-name \"{{ ansible_local.citc.dns_zone }}\";" | sudo tee -s /etc/dhcp/dhclient.conf {% endif %} sudo dnf install -y ansible git sudo cat /tmp/hosts From 901660928a99a3f81244ba34248228426bb93567 Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Wed, 8 Dec 2021 16:04:36 +0100 Subject: [PATCH 51/67] azure dns --- roles/packer/templates/prepare_ansible.sh.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/packer/templates/prepare_ansible.sh.j2 b/roles/packer/templates/prepare_ansible.sh.j2 index cb07b631..993d3ab6 100644 --- a/roles/packer/templates/prepare_ansible.sh.j2 +++ b/roles/packer/templates/prepare_ansible.sh.j2 @@ -20,7 +20,7 @@ sudo dnf config-manager --set-enabled ol8_codeready_builder {% if ansible_local.citc.csp in ["azure"] %} echo "[Main]" | sudo tee -a /etc/NetworkManager/conf.d/dhclient.conf echo "dhcp=dhclient" | sudo tee -a /etc/NetworkManager/conf.d/dhclient.conf -echo "append domain-name \"{{ ansible_local.citc.dns_zone }}\";" | sudo tee -s /etc/dhcp/dhclient.conf +echo "append domain-name \"{{ ansible_local.citc.dns_zone }}\";" | sudo tee -a /etc/dhcp/dhclient.conf {% endif %} sudo dnf install -y ansible git sudo cat /tmp/hosts From 96c56960bfced749696618f66ae7e91ef6ac785e Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Wed, 8 Dec 2021 16:07:04 +0100 Subject: [PATCH 52/67] azure dns --- roles/packer/templates/prepare_ansible.sh.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/packer/templates/prepare_ansible.sh.j2 b/roles/packer/templates/prepare_ansible.sh.j2 index 993d3ab6..dffdfd4f 100644 --- a/roles/packer/templates/prepare_ansible.sh.j2 +++ b/roles/packer/templates/prepare_ansible.sh.j2 @@ -20,7 +20,7 @@ sudo dnf config-manager --set-enabled ol8_codeready_builder {% if ansible_local.citc.csp in ["azure"] %} echo "[Main]" | sudo tee -a /etc/NetworkManager/conf.d/dhclient.conf echo "dhcp=dhclient" | sudo tee -a /etc/NetworkManager/conf.d/dhclient.conf -echo "append domain-name \"{{ ansible_local.citc.dns_zone }}\";" | sudo tee -a /etc/dhcp/dhclient.conf +echo "append domain-name \"{{ startnode_config.dns_zone }}\";" | sudo tee -a /etc/dhcp/dhclient.conf {% endif %} sudo dnf install -y ansible git sudo cat /tmp/hosts From a6a726a2fc7fdd68ec1b6dec7119ce5af0bdebe3 Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Wed, 8 Dec 2021 17:16:03 +0100 Subject: [PATCH 53/67] azure dns --- roles/packer/templates/prepare_ansible.sh.j2 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/packer/templates/prepare_ansible.sh.j2 b/roles/packer/templates/prepare_ansible.sh.j2 index dffdfd4f..5e26f6be 100644 --- a/roles/packer/templates/prepare_ansible.sh.j2 +++ b/roles/packer/templates/prepare_ansible.sh.j2 @@ -20,7 +20,8 @@ sudo dnf config-manager --set-enabled ol8_codeready_builder {% if ansible_local.citc.csp in ["azure"] %} echo "[Main]" | sudo tee -a /etc/NetworkManager/conf.d/dhclient.conf echo "dhcp=dhclient" | sudo tee -a /etc/NetworkManager/conf.d/dhclient.conf -echo "append domain-name \"{{ startnode_config.dns_zone }}\";" | sudo tee -a /etc/dhcp/dhclient.conf +echo "append domain-name \" {{ startnode_config.dns_zone }}\";" | sudo tee -a /etc/dhcp/dhclient.conf +cat /etc/resolv.conf {% endif %} sudo dnf install -y ansible git sudo cat /tmp/hosts From b190d9d9782b60864549adf5a7941c54e2374d16 Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Thu, 9 Dec 2021 09:49:00 +0100 Subject: [PATCH 54/67] azure dns --- roles/packer/templates/prepare_ansible.sh.j2 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/roles/packer/templates/prepare_ansible.sh.j2 b/roles/packer/templates/prepare_ansible.sh.j2 index 5e26f6be..68a00ef2 100644 --- a/roles/packer/templates/prepare_ansible.sh.j2 +++ b/roles/packer/templates/prepare_ansible.sh.j2 @@ -18,9 +18,10 @@ sudo dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8. sudo dnf config-manager --set-enabled ol8_codeready_builder {% endif %} {% if ansible_local.citc.csp in ["azure"] %} -echo "[Main]" | sudo tee -a /etc/NetworkManager/conf.d/dhclient.conf +echo "[main]" | sudo tee -a /etc/NetworkManager/conf.d/dhclient.conf echo "dhcp=dhclient" | sudo tee -a /etc/NetworkManager/conf.d/dhclient.conf -echo "append domain-name \" {{ startnode_config.dns_zone }}\";" | sudo tee -a /etc/dhcp/dhclient.conf +echo "append domain-search \" {{ startnode_config.dns_zone }}\";" | sudo tee -a /etc/dhcp/dhclient.conf +sudo systemctl restart NetworkManager cat /etc/resolv.conf {% endif %} sudo dnf install -y ansible git From 986e573499b495b8e96a158cacefebd1d15eb992 Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Thu, 9 Dec 2021 09:51:08 +0100 Subject: [PATCH 55/67] azure dns --- compute.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/compute.yml b/compute.yml index f2bcdc20..ad38308f 100644 --- a/compute.yml +++ b/compute.yml @@ -13,12 +13,12 @@ roles: - citc_user - filesystem - #- ssh + - ssh ##- security_updates - #- ntp - #- sssd - #- lmod - #- packages - #- mpi - #- slurm - #- monitoring + - ntp + - sssd + - lmod + - packages + - mpi + - slurm + - monitoring From bcc1b61ed8ccdb4322d947eea9f53848e5adf86f Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Thu, 9 Dec 2021 10:30:40 +0100 Subject: [PATCH 56/67] azure dns --- roles/packer/templates/prepare_ansible.sh.j2 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/roles/packer/templates/prepare_ansible.sh.j2 b/roles/packer/templates/prepare_ansible.sh.j2 index 68a00ef2..8ac998d1 100644 --- a/roles/packer/templates/prepare_ansible.sh.j2 +++ b/roles/packer/templates/prepare_ansible.sh.j2 @@ -20,7 +20,10 @@ sudo dnf config-manager --set-enabled ol8_codeready_builder {% if ansible_local.citc.csp in ["azure"] %} echo "[main]" | sudo tee -a /etc/NetworkManager/conf.d/dhclient.conf echo "dhcp=dhclient" | sudo tee -a /etc/NetworkManager/conf.d/dhclient.conf +cat /etc/NetworkManager/conf.d/dhclient.conf +echo "append domain-name \" {{ startnode_config.dns_zone }}\";" | sudo tee -a /etc/dhcp/dhclient.conf echo "append domain-search \" {{ startnode_config.dns_zone }}\";" | sudo tee -a /etc/dhcp/dhclient.conf +sudo cat /etc/dhcp/dhclient.conf sudo systemctl restart NetworkManager cat /etc/resolv.conf {% endif %} From 1510b29742ac19fa1c7e0826ffe9e5f7f60f573c Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Fri, 10 Dec 2021 11:43:12 +0100 Subject: [PATCH 57/67] bootstrap and custom data --- roles/filesystem/tasks/main.yml | 2 +- roles/slurm/files/citc_azure.py | 8 ++++---- roles/slurm/tasks/elastic.yml | 4 ++++ 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/roles/filesystem/tasks/main.yml b/roles/filesystem/tasks/main.yml index eae9eb4d..84f43e87 100644 --- a/roles/filesystem/tasks/main.yml +++ b/roles/filesystem/tasks/main.yml @@ -34,7 +34,7 @@ path: /mnt/{{ filesystem_mount_point }} src: "{{ filesystem_target_address }}:{{ filesystem_mount_point }}" fstype: nfs - opts: rw,hard,rsize=1048576,wsize=1048576,vers=3,tcp,_netdev + opts: rw,hard,rsize=1048576,wsize=1048576,vers=3,tcp,_netdev,noauto state: mounted when: - ansible_local.citc.csp == "azure" diff --git a/roles/slurm/files/citc_azure.py b/roles/slurm/files/citc_azure.py index 79a47011..19d42477 100644 --- a/roles/slurm/files/citc_azure.py +++ b/roles/slurm/files/citc_azure.py @@ -144,7 +144,7 @@ async def start_node(log, host: str, nodespace: Dict[str, str], ssh_keys: str) - compute_client = ComputeManagementClient(credential, subscription_id) with open("/home/slurm/bootstrap.sh", "rb") as f: - user_data = base64.b64encode(f.read()).decode() + custom_data = base64.b64encode(f.read()).decode() while get_node_state(compute_client, log, host, resource_group) == "TERMINATING": log.info(f"{host}: host is currently terminating. Waiting...") @@ -188,14 +188,14 @@ async def start_node(log, host: str, nodespace: Dict[str, str], ssh_keys: str) - "key_data": ssh_keys } ] } - } + }, + "custom_data": custom_data, }, "network_profile": { "network_interfaces": [{ "id": nic_result.id, }] - }, - "user_data": user_data, + } }) vm_result = poller.result() diff --git a/roles/slurm/tasks/elastic.yml b/roles/slurm/tasks/elastic.yml index 8c55d780..08fc13cd 100644 --- a/roles/slurm/tasks/elastic.yml +++ b/roles/slurm/tasks/elastic.yml @@ -123,6 +123,10 @@ --output text --query 'Tags[?Key==`Name`].Value') hostnamectl set-hostname ${name} {% endif %} + {% if ansible_local.citc.csp == "azure" %} + sudo mount /mnt/shared + sudo systemctl start slurmd + {% endif %} # Ensure that slurmd is running at this point #systemctl start slurmd From a97bd9bb8f279ad306589658fd8ecb8588189d29 Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Fri, 10 Dec 2021 14:42:51 +0100 Subject: [PATCH 58/67] adding node delete --- roles/slurm/files/citc_azure.py | 130 +++----------------------------- 1 file changed, 10 insertions(+), 120 deletions(-) diff --git a/roles/slurm/files/citc_azure.py b/roles/slurm/files/citc_azure.py index 19d42477..39c72afb 100644 --- a/roles/slurm/files/citc_azure.py +++ b/roles/slurm/files/citc_azure.py @@ -28,15 +28,6 @@ def get_nodespace() -> Dict[str, str]: return load_yaml("/etc/citc/startnode.yaml") -#def get_subnet(oci_config, compartment_id: str, vcn_id: str, cluster_id: str) -> str: -def get_subnet() -> str: - """ - Get the relevant cluster subnet for a given compartment, VCN and AD - """ - #return [s.id for s in oci.core.VirtualNetworkClient(oci_config).list_subnets(compartment_id, vcn_id=vcn_id).data if s.freeform_tags.get("cluster") == cluster_id][0] - return "subnet" - - #def get_node_state(oci_config, log, compartment_id: str, hostname: str, cluster_id: str) -> str: def get_node_state(compute_client, log, hostname: str, resource_group: str) -> str: """ @@ -56,64 +47,6 @@ def get_node_state(compute_client, log, hostname: str, resource_group: str) -> s return "node_state" -#def get_image(oci_config, compartment_id: str, cluster_id: str) -> oci.core.models.Image: -def get_image() -> str: - #all_images = oci.pagination.list_call_get_all_results_generator(oci.core.ComputeClient(oci_config).list_images, 'record', compartment_id, operating_system="Oracle Linux") - #our_images: List[oci.core.models.Image] = [i for i in all_images if i.freeform_tags.get("cluster") == cluster_id] - try: - #return [i for i in our_images if "GPU" not in i.display_name][0] - return "image" - except IndexError: - raise RuntimeError("Could not locate the image for the compute node") - - -def create_node_config(host: str, ip: Optional[str], nodespace: Dict[str, str], ssh_keys: str): -#def create_node_config() -> str: - """ - Create the configuration needed to create ``hostname`` in ``nodespace`` with ``ssh_keys`` - """ - #features = subprocess.run(["sinfo", "--Format=features:200", "--noheader", f"--nodes={hostname}"], stdout=subprocess.PIPE).stdout.decode().split(',') - #ad_number = [f for f in features if f.startswith("ad=")][0].split("=")[1].strip() - #ad = f"{nodespace['ad_root']}{ad_number}" - #shape = [f for f in features if f.startswith("shape=")][0].split("=")[1].strip() - #subnet = get_subnet(oci_config, nodespace["compartment_id"], nodespace["vcn_id"], nodespace["cluster_id"]) - #image = get_image(oci_config, nodespace["compartment_id"], nodespace["cluster_id"]) - - with open("/home/slurm/bootstrap.sh", "rb") as f: - user_data = base64.b64encode(f.read()).decode() - - subscription_id = nodespace["subscription"] - resource_group = nodespace["resource_group"] - region = nodespace["region"] - subnet = nodespace["subnet"] - - instance_details = "{ \"location\": "+region+"," \ - "\"storage_profile\": {"+ \ - "\"image_reference\": {"+ \ - "\"publisher\": \"OpenLogic\","+ \ - "\"offer\": \"CentOS\","+ \ - "\"sku\": \"8_4-gen2\","+ \ - "\"version\": \"latest\" } },"+ \ - "\"hardware_profile\": {"+ \ - "\"vm_size\": \"Standard_D4s_v3\"},"+ \ - "\"os_profile\": {"+ \ - "\"computer_name\":"+ host +","+ \ - "\"admin_username\": \"centos\","+ \ - "\"linux_configuration\": {" + \ - "\"ssh\": { "+ \ - "\"public_keys\" : [ { "+ \ - "\"path\": \"/home/centos/.ssh/authorized_keys\","+ \ - "\"key_data\":"+ ssh_keys +\ - "} ]}}},"+ \ - "\"network_profile\": {"+ \ - "\"network_interfaces\": [{"+ \ - "\"id\":"+ nic_result.id +","+ \ - "}]},"+ \ - "\"user_data\":"+ user_data + ",}" - - return instance_details - - #def get_ip(hostname: str) -> Tuple[Optional[str], Optional[str], Optional[str]]: def get_ip() -> str: #host_dns_match = re.match(r"(\d+\.){3}\d+", subprocess.run(["host", hostname], stdout=subprocess.PIPE).stdout.decode().split()[-1]) @@ -130,7 +63,6 @@ def get_ip() -> str: async def start_node(log, host: str, nodespace: Dict[str, str], ssh_keys: str) -> None: log.info(f"{host}: Starting") - #oci_config = oci.config.from_file() credential = DefaultAzureCredential() nodespace = get_nodespace() subscription_id = nodespace["subscription"] @@ -205,66 +137,24 @@ async def start_node(log, host: str, nodespace: Dict[str, str], ssh_keys: str) - log.info(f"{host}: Started") return vm_result - #while get_node_state(oci_config, log, nodespace["compartment_id"], host, nodespace["cluster_id"]) == "TERMINATING": - # log.info(f"{host}: host is currently terminating. Waiting...") - # await asyncio.sleep(5) - - #node_state = get_node_state(oci_config, log, nodespace["compartment_id"], host, nodespace["cluster_id"]) - #if node_state != "TERMINATED": - # log.warning(f"{host}: host is already running with state {node_state}") - # return - - #ip, _dns_ip, slurm_ip = get_ip(host) - - #instance_details = create_node_config(oci_config, host, ip, nodespace, ssh_keys) - - #loop = asyncio.get_event_loop() - #retry_strategy_builder = oci.retry.RetryStrategyBuilder() - #retry_strategy_builder.add_max_attempts(max_attempts=10).add_total_elapsed_time(total_elapsed_time_seconds=600) - #retry_strategy = retry_strategy_builder.get_retry_strategy() - #client = oci.core.ComputeClient(oci_config, retry_strategy=retry_strategy) - - #try: - # instance_result = await loop.run_in_executor(None, client.launch_instance, instance_details) - # instance = instance_result.data - #except oci.exceptions.ServiceError as e: - # log.error(f"{host}: problem launching instance: {e}") - # return - - #if not slurm_ip: - # node_id = instance.id - # while not oci.core.ComputeClient(oci_config).list_vnic_attachments(instance_details.compartment_id, instance_id=node_id).data: - # log.info(f"{host}: No VNIC attachment yet. Waiting...") - # await asyncio.sleep(5) - - # vnic_id = oci.core.ComputeClient(oci_config).list_vnic_attachments(instance_details.compartment_id, instance_id=node_id).data[0].vnic_id - # private_ip = oci.core.VirtualNetworkClient(oci_config).get_vnic(vnic_id).data.private_ip - - # log.info(f"{host}: Private IP {private_ip}") - # subprocess.run(["scontrol", "update", f"NodeName={host}", f"NodeAddr={private_ip}"]) - - #log.info(f"{host}: Started") - #return instance def terminate_instance(log, hosts): - #config = oci.config.from_file() credential = DefaultAzureCredential() nodespace = get_nodespace() subscription_id = nodespace["subscription"] resource_client = ResourceManagementClient(credential, subscription_id) compute_client = ComputeManagementClient(credential, subscription_id) - #for host in hosts: - # log.info(f"Stopping {host}") -# -# try: -# matching_nodes = oci.core.ComputeClient(config).list_instances(nodespace["compartment_id"], display_name=host).data -# node_id = [n.id for n in matching_nodes if n.lifecycle_state not in {"TERMINATED", "TERMINATING"}][0] -# -# oci.core.ComputeClient(config).terminate_instance(node_id) -# except Exception as e: -# log.error(f" problem while stopping: {e}") -# continue + for host in hosts: + log.info(f"Stopping {host}") + + try: + vm = compute_client.virtual_machines.get(resource_group, host, expand='instanceView') + for stat in vm.instance_view.statuses: + if stat.code == "PowerState/running": + poller = compute_client.virtual_machines.begin_delete(resource_group, host) + vm_result = poller.result() + print(f"Deleted virtual machine {vm_result.name}") log.info(f" Stopped {host}") From 0812dea3a1b4e474cb0f9b5c645fb1e85f5a4df4 Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Fri, 10 Dec 2021 15:58:02 +0100 Subject: [PATCH 59/67] stop node --- roles/slurm/files/citc_azure.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/roles/slurm/files/citc_azure.py b/roles/slurm/files/citc_azure.py index 39c72afb..e58c9464 100644 --- a/roles/slurm/files/citc_azure.py +++ b/roles/slurm/files/citc_azure.py @@ -34,8 +34,8 @@ def get_node_state(compute_client, log, hostname: str, resource_group: str) -> s Get the current node state of the VM for the given hostname If there is no such VM, return "TERMINATED" """ - matches = compute_client.virtual_machines.list(resource_group) - print(matches) + #matches = compute_client.virtual_machines.list(resource_group) + #print(matches) #matches = oci.core.ComputeClient(oci_config).list_instances(compartment_id=compartment_id, display_name=hostname).data #matches = [i for i in matches if i.freeform_tags.get("cluster") == cluster_id] #still_exist = [i for i in matches if i.lifecycle_state != "TERMINATED"] @@ -156,5 +156,6 @@ def terminate_instance(log, hosts): poller = compute_client.virtual_machines.begin_delete(resource_group, host) vm_result = poller.result() print(f"Deleted virtual machine {vm_result.name}") - - log.info(f" Stopped {host}") + except: + print("An exception occurred") + log.info(f" Stopped {host}") From 4788f3e6289773891c1079718dfb6aab6accb97f Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Fri, 10 Dec 2021 16:26:04 +0100 Subject: [PATCH 60/67] stop node --- roles/slurm/files/citc_azure.py | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/slurm/files/citc_azure.py b/roles/slurm/files/citc_azure.py index e58c9464..3af1e990 100644 --- a/roles/slurm/files/citc_azure.py +++ b/roles/slurm/files/citc_azure.py @@ -143,6 +143,7 @@ def terminate_instance(log, hosts): credential = DefaultAzureCredential() nodespace = get_nodespace() subscription_id = nodespace["subscription"] + resource_group = nodespace["resource_group"] resource_client = ResourceManagementClient(credential, subscription_id) compute_client = ComputeManagementClient(credential, subscription_id) From 7710ba9fad89bdcb82eeb2fd65a920e2893e3842 Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Tue, 4 Jan 2022 09:25:03 +0100 Subject: [PATCH 61/67] create vm based on shape --- roles/slurm/files/citc_azure.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/roles/slurm/files/citc_azure.py b/roles/slurm/files/citc_azure.py index 3af1e990..33056bf9 100644 --- a/roles/slurm/files/citc_azure.py +++ b/roles/slurm/files/citc_azure.py @@ -28,6 +28,14 @@ def get_nodespace() -> Dict[str, str]: return load_yaml("/etc/citc/startnode.yaml") +def get_node_features(hostname): + features = subprocess.run( + ["sinfo", "--Format=features:200", "--noheader", f"--nodes={hostname}"], + stdout=subprocess.PIPE + ).stdout.decode().strip().split(',') + features = {f.split("=")[0]: f.split("=")[1] for f in features} + return features + #def get_node_state(oci_config, log, compartment_id: str, hostname: str, cluster_id: str) -> str: def get_node_state(compute_client, log, hostname: str, resource_group: str) -> str: """ @@ -86,6 +94,9 @@ async def start_node(log, host: str, nodespace: Dict[str, str], ssh_keys: str) - for image in images: vm_image = str(image.id) + features = get_node_features(hostname) + shape = features["shape"] + poller = network_client.network_interfaces.begin_create_or_update(resource_group,host+"-nic", { "location": region, @@ -108,7 +119,7 @@ async def start_node(log, host: str, nodespace: Dict[str, str], ssh_keys: str) - } }, "hardware_profile": { - "vm_size": "Standard_D4s_v3" + "vm_size": shape }, "os_profile": { "computer_name": host, From 814d8a7bca36e9190cafd3d953367f67a7f31724 Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Tue, 4 Jan 2022 10:07:23 +0100 Subject: [PATCH 62/67] create vm based on shape --- roles/slurm/files/citc_azure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/slurm/files/citc_azure.py b/roles/slurm/files/citc_azure.py index 33056bf9..f5c0103b 100644 --- a/roles/slurm/files/citc_azure.py +++ b/roles/slurm/files/citc_azure.py @@ -94,7 +94,7 @@ async def start_node(log, host: str, nodespace: Dict[str, str], ssh_keys: str) - for image in images: vm_image = str(image.id) - features = get_node_features(hostname) + features = get_node_features(host) shape = features["shape"] poller = network_client.network_interfaces.begin_create_or_update(resource_group,host+"-nic", From 4c66c4018d5c94841dc10afcc208abd16d731cf5 Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Tue, 4 Jan 2022 18:20:49 +0100 Subject: [PATCH 63/67] add avset --- roles/slurm/files/citc_azure.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/roles/slurm/files/citc_azure.py b/roles/slurm/files/citc_azure.py index f5c0103b..57858c91 100644 --- a/roles/slurm/files/citc_azure.py +++ b/roles/slurm/files/citc_azure.py @@ -97,6 +97,19 @@ async def start_node(log, host: str, nodespace: Dict[str, str], ssh_keys: str) - features = get_node_features(host) shape = features["shape"] + + poller = compute_client.availability_sets.create_or_update(resource_group,shape, + { + "location": region, + "platform_update_domain_count": 1, + "platform_fault_domain_count": 1, + "sku": { "name": "Aligned" }, + } + ) + avset_id = poller.id + print(f"Provisioning avset {avset_id}.") + + poller = network_client.network_interfaces.begin_create_or_update(resource_group,host+"-nic", { "location": region, @@ -138,6 +151,9 @@ async def start_node(log, host: str, nodespace: Dict[str, str], ssh_keys: str) - "network_interfaces": [{ "id": nic_result.id, }] + }, + "availability_set": { + "id": avset_id, } }) From 4a218c24bda1ec94077d2dcf54e4139c903334c4 Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Wed, 5 Jan 2022 09:17:00 +0100 Subject: [PATCH 64/67] add avset --- roles/slurm/files/citc_azure.py | 69 +++++++++++++++++++++++---------- 1 file changed, 49 insertions(+), 20 deletions(-) diff --git a/roles/slurm/files/citc_azure.py b/roles/slurm/files/citc_azure.py index 57858c91..5bba89fc 100644 --- a/roles/slurm/files/citc_azure.py +++ b/roles/slurm/files/citc_azure.py @@ -94,22 +94,7 @@ async def start_node(log, host: str, nodespace: Dict[str, str], ssh_keys: str) - for image in images: vm_image = str(image.id) - features = get_node_features(host) - shape = features["shape"] - - - poller = compute_client.availability_sets.create_or_update(resource_group,shape, - { - "location": region, - "platform_update_domain_count": 1, - "platform_fault_domain_count": 1, - "sku": { "name": "Aligned" }, - } - ) - avset_id = poller.id - print(f"Provisioning avset {avset_id}.") - - + print(f"Provisioning nic for {host}....") poller = network_client.network_interfaces.begin_create_or_update(resource_group,host+"-nic", { "location": region, @@ -122,9 +107,22 @@ async def start_node(log, host: str, nodespace: Dict[str, str], ssh_keys: str) - nic_result = poller.result() - print(f"Provisioning virtual machine {host}; this operation might take a few minutes.") + features = get_node_features(host) + shape = features["shape"] - poller = compute_client.virtual_machines.begin_create_or_update(resource_group, host, { + if shape == "Standard_HC44rs" : + print(f"Provisioning availability set for {host}....") + poller = compute_client.availability_sets.create_or_update(resource_group,shape, + { + "location": region, + "platform_update_domain_count": 1, + "platform_fault_domain_count": 1, + "sku": { "name": "Aligned" }, + } + ) + avset_id = poller.id + print(f"Provisioning virtual machine {host}; this operation might take a few minutes.") + poller = compute_client.virtual_machines.begin_create_or_update(resource_group, host, { "location": region, "storage_profile": { "image_reference": { @@ -156,8 +154,39 @@ async def start_node(log, host: str, nodespace: Dict[str, str], ssh_keys: str) - "id": avset_id, } }) - - vm_result = poller.result() + vm_result = poller.result() + else : + print(f"Provisioning virtual machine {host}; this operation might take a few minutes.") + poller = compute_client.virtual_machines.begin_create_or_update(resource_group, host, { + "location": region, + "storage_profile": { + "image_reference": { + "id": vm_image, + } + }, + "hardware_profile": { + "vm_size": shape + }, + "os_profile": { + "computer_name": host, + "admin_username": "centos", + "linux_configuration": { + "ssh": { + "public_keys" : [ { + "path": "/home/centos/.ssh/authorized_keys", + "key_data": ssh_keys + } ] + } + }, + "custom_data": custom_data, + }, + "network_profile": { + "network_interfaces": [{ + "id": nic_result.id, + }] + }, + }) + vm_result = poller.result() print(f"Provisioned virtual machine {vm_result.name}") From 659a71fac10005b8d9c2018cae04f35578679f30 Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Thu, 6 Jan 2022 15:11:11 +0100 Subject: [PATCH 65/67] add cvmfs --- roles/packer/files/all.pkr.hcl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/roles/packer/files/all.pkr.hcl b/roles/packer/files/all.pkr.hcl index 2884e2cc..21f85e03 100644 --- a/roles/packer/files/all.pkr.hcl +++ b/roles/packer/files/all.pkr.hcl @@ -186,6 +186,10 @@ build { script = "/home/citc/compute_image_extra.sh" } + provisioner "shell" { + script = "/home/citc/install_cvmfs_eessi.sh" + } + provisioner "shell" { script = "/home/citc/compute_image_finalize.sh" } From 66ae716ed28aa8cb0aac05299e61906469c89a2b Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Thu, 6 Jan 2022 16:05:36 +0100 Subject: [PATCH 66/67] tuning cvmfs --- roles/packer/files/compute_image_extra.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/packer/files/compute_image_extra.sh b/roles/packer/files/compute_image_extra.sh index 27bd331b..e4f7dd0c 100644 --- a/roles/packer/files/compute_image_extra.sh +++ b/roles/packer/files/compute_image_extra.sh @@ -8,4 +8,4 @@ # sudo yum -y install cmake gcc-gfortran # to install CernVM-FS and configure access to EESSI, uncomment the line below: -# /home/citc/install_cvmfs_eessi.sh +#/home/citc/install_cvmfs_eessi.sh From ea3e906247bf9401da05843e449ade220d0c14f1 Mon Sep 17 00:00:00 2001 From: hugo meiland Date: Mon, 17 Jan 2022 09:40:56 +0100 Subject: [PATCH 67/67] enable rdma --- roles/slurm/tasks/elastic.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/slurm/tasks/elastic.yml b/roles/slurm/tasks/elastic.yml index 08fc13cd..9e36eb51 100644 --- a/roles/slurm/tasks/elastic.yml +++ b/roles/slurm/tasks/elastic.yml @@ -124,6 +124,8 @@ hostnamectl set-hostname ${name} {% endif %} {% if ansible_local.citc.csp == "azure" %} + sudo sed -i -e 's/# OS.EnableRDMA=./OS.EnableRDMA=y/g' /etc/waagent.conf + sudo systemctl restart waagent sudo mount /mnt/shared sudo systemctl start slurmd {% endif %}