diff --git a/config.example/maas-inventory.yml b/config.example/maas-inventory.yml new file mode 100644 index 000000000..6f7d62498 --- /dev/null +++ b/config.example/maas-inventory.yml @@ -0,0 +1,28 @@ +# MAAS Dynamic Inventory Configuration +# +# This file is copied to config/maas-inventory.yml by scripts/setup.sh. +# Edit your private copy there (config/ is gitignored). +# +# You can also set MAAS_INVENTORY_CONFIG=/path/to/config.yml or use +# environment variables (MAAS_API_URL, MAAS_API_KEY, etc.). +# Environment variables take precedence over config file values. + +# MAAS API endpoint (required) +api_url: "http://:5240/MAAS/api/2.0" + +# MAAS API key in consumer_key:token_key:token_secret format (required) +# Generate with: sudo maas apikey --username= +# Or find in MAAS web UI: your profile -> API keys +api_key: "CONSUMER_KEY:TOKEN_KEY:TOKEN_SECRET" + +# Default SSH user for all hosts (default: ubuntu) +ssh_user: "ubuntu" + +# Preferred network prefix for selecting ansible_host IP (optional) +# Machines may have multiple IPs; this selects the one on your target network +#network: "10.0.0" + +# SSH bastion/jumpbox for ProxyJump (optional) +# Only needed if target machines are on a private network that requires +# jumping through a bastion host. Leave commented out for direct access. +#ssh_bastion: "user@bastion-host" diff --git a/docs/pxe/maas.md b/docs/pxe/maas.md index f14af775e..ca95671c9 100644 --- a/docs/pxe/maas.md +++ b/docs/pxe/maas.md @@ -18,6 +18,7 @@ OS Provisioning with MAAS - [Releasing and reinstalling the machine](#releasing-and-reinstalling-the-machine) - [Scaling up](#scaling-up) - [Creating a DGX OS image installable by MAAS](#creating-a-dgx-os-image-installable-by-maas) + - [Dynamic Inventory](#dynamic-inventory) ## Introduction @@ -27,7 +28,7 @@ By default, this playbook will set up a single-node MAAS install which can then In most cases, you will use MAAS to provision the OS on multiple bare-metal servers. However, MAAS can also be used to provision VMs, and this guide will walk through an example using two virtual machines for ease of demonstration. -This guide was written using MAAS 2.8. +This guide was originally written using MAAS 2.8; current MAAS releases are 3.x. MAAS has a lot of different configuration options which are outside the scope of this guide. For the best reference on how to use MAAS in general, see the [documentation on maas.io](https://maas.io/docs). @@ -36,7 +37,7 @@ For the best reference on how to use MAAS in general, see the [documentation on In order to set up and use MAAS, you should at minimum have the following components: -- An Ubuntu 18.04 server which you can use to run MAAS +- An Ubuntu 22.04 or 24.04 server which you can use to run MAAS - One or more servers which you will manage using MAAS - A network connection between all the servers on which you can safely run DHCP. This is needed so that MAAS can provision IP addresses to the nodes it manages. - A network connection which you can use to log into the MAAS server. This may be the same network as the inter-node network, or it may be a separate network. @@ -47,7 +48,7 @@ In this example, we will use: - An internal (VM-only) network on which we'll use the subnet `192.168.1.0/24` - An external network connection on which we'll use the subnet `192.168.122.0/24` -- `maas-vm`, a pre-installed Ubuntu 18.04 virtual machine +- `maas-vm`, a pre-installed Ubuntu 22.04 (or 24.04) virtual machine - `maas-vm` has IP `192.168.1.1` on the internal network, and `192.168.122.90` on the external network - `test-vm`, a "blank" virtual machine with no OS, on the same VM host - `test-vm` has a connection only to the internal network, which is not configured yet @@ -72,7 +73,7 @@ Please consult your hypervisor documentation for instructions on doing this. maas_dns_domain: 'deepops.local' maas_region_controller: '192.168.1.1' maas_region_controller_url: 'http://{{ maas_region_controller }}:5240/MAAS' - maas_repo: 'ppa:maas/2.8' + maas_repo: 'ppa:maas/3.5' ``` 1. Run the Ansible playbook to install: ```bash @@ -104,7 +105,7 @@ After entering SSH keys, you will be redirected to an intro configuration page o - The source and versions of Ubuntu which will be downloaded by MAAS to install In this example, I'm leaving all these parameters at their default values. -This will enable us to install Ubuntu 18.04 on the VM we are installing. +This will enable us to install Ubuntu on the VM we are provisioning. MAAS will sync the necessary package repositories for the installation to your provisioning server (`maas-vm`), which may take some time. If the repositories are still syncing, the "Continue" button at the bottom will be grayed out. @@ -201,3 +202,125 @@ then commission them and provision them by selecting them as a group in the Mach The official NVIDIA DGX OS version 5.0 and higher is installable by MAAS by creating a custom OS image. The code is located in the `submodules/packer-maas` directory. For more information, see: https://github.com/DeepOps/packer-maas/tree/master/dgxos5 + +## Dynamic Inventory + +DeepOps includes a dynamic inventory script that queries your MAAS server's +API to automatically discover deployed machines and map them to Ansible groups +using MAAS tags. This eliminates the need to manually edit `config/inventory` +when machines are provisioned, released, or reassigned. + +### Prerequisites + +- A MAAS server with the REST API enabled (default port 5240) +- A MAAS API key (generate one from the MAAS web UI under your profile, + or run `sudo maas apikey --username=` on the MAAS server) +- Python 3 on the Ansible control machine (no additional packages required) + +### Setup + +1. Run `scripts/setup.sh` (or manually copy `config.example/` to `config/`). + +2. Edit `config/maas-inventory.yml` with your MAAS server details: + + ```yaml + api_url: "http://maas-server:5240/MAAS/api/2.0" + api_key: "consumer_key:token_key:token_secret" + ssh_user: "ubuntu" + ``` + +3. Optionally set `network` to prefer a specific subnet when machines have + multiple IPs, and `ssh_bastion` if your machines are behind a jumpbox: + + ```yaml + network: "10.0.0" + ssh_bastion: "user@bastion-host" + ``` + +4. Test the inventory: + + ```bash + ./scripts/maas_inventory.py --list + ansible -i scripts/maas_inventory.py all -m ping + ``` + +### Tag-Based Group Assignment + +The script maps MAAS tags directly to Ansible groups. To assign a machine to +the `[slurm-master]` group, tag it `slurm-master` in MAAS. A machine can +have multiple tags and will appear in all corresponding groups. + +DeepOps parent groups (`slurm-cluster`, `k8s-cluster`, etc.) are +automatically created with the correct `children` relationships, so you +only need to tag leaf groups. + +**Recommended tags** (matching DeepOps inventory groups): + +| Tag | Ansible Group | Used By | +|-----|--------------|---------| +| `kube-master` | `[kube-master]` | K8s control plane | +| `kube-node` | `[kube-node]` | K8s worker nodes | +| `slurm-master` | `[slurm-master]` | Slurm head node | +| `slurm-node` | `[slurm-node]` | Slurm compute nodes | +| `slurm-nfs` | `[slurm-nfs]` | Slurm NFS server | + +**Example: switching between Slurm and K8s testing:** + +```bash +# Tag machines for Slurm +maas admin tag update-nodes slurm-master add= +maas admin tag update-nodes slurm-node add= add= + +# Run Slurm deployment +ansible-playbook -i scripts/maas_inventory.py playbooks/slurm-cluster.yml + +# Later, retag for K8s +maas admin tag update-nodes slurm-master remove= +maas admin tag update-nodes kube-master add= +maas admin tag update-nodes kube-node add= add= + +# Run K8s deployment +ansible-playbook -i scripts/maas_inventory.py playbooks/k8s-cluster.yml +``` + +### Configuration Reference + +Configuration is loaded from environment variables or `config/maas-inventory.yml`. +Environment variables take precedence. + +| Config Key | Env Variable | Required | Description | +|-----------|-------------|----------|-------------| +| `api_url` | `MAAS_API_URL` | Yes | MAAS API endpoint | +| `api_key` | `MAAS_API_KEY` | Yes | OAuth1 API key (`consumer:token:secret`) | +| `ssh_user` | `MAAS_SSH_USER` | No | SSH user (default: `ubuntu`) | +| `network` | `MAAS_NETWORK` | No | Preferred IP network prefix | +| `ssh_bastion` | `MAAS_SSH_BASTION` | No | SSH bastion for ProxyJump | + +### Host Variables + +The script exposes MAAS metadata as Ansible host variables: + +| Variable | Example | Description | +|----------|---------|-------------| +| `maas_system_id` | `4fcb8q` | MAAS machine ID | +| `maas_fqdn` | `node01.maas` | Fully qualified domain name | +| `maas_os` | `ubuntu` | Operating system | +| `maas_distro` | `noble` | Distribution series | +| `maas_tags` | `["slurm-master", "virtual"]` | All tags on the machine | +| `maas_cpus` | `4` | CPU count | +| `maas_memory_mb` | `8192` | Memory in MB | +| `maas_arch` | `amd64/generic` | Architecture | +| `maas_zone` | `default` | MAAS availability zone | +| `maas_pool` | `default` | MAAS resource pool | + +### Static vs Dynamic Inventory + +You can use either approach: + +- **Static** (`config/inventory`): Manually list hosts and groups. Simpler + for fixed environments. This is the default set up by `scripts/setup.sh`. +- **Dynamic** (`scripts/maas_inventory.py`): Auto-discovers machines from + MAAS. Better for environments where machines are frequently provisioned + or reassigned. + +Both can be combined by passing multiple `-i` flags to `ansible-playbook`. diff --git a/scripts/maas_inventory.py b/scripts/maas_inventory.py new file mode 100755 index 000000000..3c6653b7d --- /dev/null +++ b/scripts/maas_inventory.py @@ -0,0 +1,256 @@ +#!/usr/bin/env python3 +"""MAAS Dynamic Inventory Script for Ansible. + +Queries a Canonical MAAS server's REST API and generates Ansible inventory +based on machine tags. Machines tagged with Ansible group names (e.g., +"slurm-master", "kube-node") are placed into those groups automatically. + +Only machines in the "Deployed" state (status=6) are included. + +Configuration is loaded from environment variables or a YAML config file. + +Environment variables: + MAAS_API_URL MAAS API endpoint (e.g., http://maas-server:5240/MAAS/api/2.0) + MAAS_API_KEY OAuth1 key in consumer_key:token_key:token_secret format + MAAS_SSH_USER Default SSH user for all hosts (default: ubuntu) + MAAS_NETWORK Preferred network prefix for ansible_host IP (optional) + MAAS_SSH_BASTION SSH bastion for ProxyJump (optional, for private networks) + +Config file (checked in order): + 1. Path in MAAS_INVENTORY_CONFIG env var + 2. config/maas-inventory.yml (repo-relative, created by setup.sh) + 3. ~/.config/maas-inventory.yml + +Usage: + ./scripts/maas_inventory.py --list + ./scripts/maas_inventory.py --host + ansible-playbook -i scripts/maas_inventory.py playbooks/slurm-cluster.yml +""" + +import argparse +import json +import os +import sys +import time +import urllib.request +import urllib.error +import uuid +from pathlib import Path + +# DeepOps group hierarchy: parent -> list of child groups. +# Tags in MAAS should match "leaf" group names (e.g., slurm-master, kube-node). +# These parent groups are auto-created using Ansible's "children" mechanism. +GROUP_CHILDREN = { + "k8s-cluster": ["kube-master", "kube-node"], + "slurm-cluster": [ + "slurm-master", "slurm-node", "slurm-cache", + "slurm-nfs", "slurm-metric", "slurm-login", + ], + "slurm-cache": ["slurm-master"], + "slurm-nfs-client": ["slurm-node"], + "slurm-metric": ["slurm-master"], + "slurm-login": ["slurm-master"], +} + + +def build_oauth1_header(api_key): + """Build OAuth1 Authorization header for MAAS API.""" + parts = api_key.split(":") + if len(parts) != 3: + raise ValueError( + "MAAS_API_KEY must be in format consumer_key:token_key:token_secret" + ) + consumer_key, token_key, token_secret = parts + return ( + f'OAuth oauth_version="1.0", ' + f'oauth_signature_method="PLAINTEXT", ' + f'oauth_consumer_key="{consumer_key}", ' + f'oauth_token="{token_key}", ' + f'oauth_signature="&{token_secret}", ' + f'oauth_nonce="{uuid.uuid4().hex}", ' + f'oauth_timestamp="{int(time.time())}"' + ) + + +def load_config(): + """Load configuration from env vars and/or YAML config file.""" + config = { + "api_url": os.environ.get("MAAS_API_URL", ""), + "api_key": os.environ.get("MAAS_API_KEY", ""), + "ssh_bastion": os.environ.get("MAAS_SSH_BASTION", ""), + "ssh_user": os.environ.get("MAAS_SSH_USER", ""), + "network": os.environ.get("MAAS_NETWORK", ""), + } + + # Try YAML config file if env vars are incomplete + if not (config["api_url"] and config["api_key"]): + config_paths = [] + if os.environ.get("MAAS_INVENTORY_CONFIG"): + config_paths.append(Path(os.environ["MAAS_INVENTORY_CONFIG"])) + # Repo-relative: config/maas-inventory.yml (created by setup.sh) + repo_root = Path(__file__).parent.parent + config_paths.append(repo_root / "config" / "maas-inventory.yml") + config_paths.append(Path.home() / ".config" / "maas-inventory.yml") + + for path in config_paths: + if path.exists(): + try: + import yaml + with open(path) as f: + file_config = yaml.safe_load(f) or {} + for key in config: + if not config[key] and key in file_config: + config[key] = str(file_config[key]) + break + except ImportError: + # PyYAML not available; try simple key: value parsing + with open(path) as f: + for line in f: + line = line.strip() + if line.startswith("#") or ":" not in line: + continue + k, v = line.split(":", 1) + k, v = k.strip(), v.strip().strip('"').strip("'") + if k in config and not config[k]: + config[k] = v + break + + if not config["api_url"]: + print("Error: MAAS_API_URL not configured", file=sys.stderr) + sys.exit(1) + if not config["api_key"]: + print("Error: MAAS_API_KEY not configured", file=sys.stderr) + sys.exit(1) + + # Defaults + if not config["ssh_user"]: + config["ssh_user"] = "ubuntu" + + return config + + +def maas_api_get(url, api_key): + """Make an authenticated GET request to the MAAS API.""" + auth_header = build_oauth1_header(api_key) + req = urllib.request.Request(url) + req.add_header("Authorization", auth_header) + req.add_header("Accept", "application/json") + + try: + with urllib.request.urlopen(req, timeout=30) as response: + return json.loads(response.read().decode()) + except urllib.error.HTTPError as e: + print(f"MAAS API error: {e.code} {e.reason}", file=sys.stderr) + if e.code == 401: + print("Check your MAAS_API_KEY value.", file=sys.stderr) + sys.exit(1) + except urllib.error.URLError as e: + print(f"MAAS connection error: {e.reason}", file=sys.stderr) + print("Check MAAS_API_URL and network connectivity.", file=sys.stderr) + sys.exit(1) + + +def get_preferred_ip(machine, network_filter): + """Select the best IP for a machine, preferring the filtered network.""" + ips = machine.get("ip_addresses", []) + if not ips: + return "" + if network_filter: + for ip in ips: + if ip.startswith(network_filter): + return ip + return ips[0] + + +def build_inventory(config): + """Query MAAS API and build Ansible inventory.""" + api_url = config["api_url"].rstrip("/") + machines_url = f"{api_url}/machines/" + + machines = maas_api_get(machines_url, config["api_key"]) + + inventory = { + "_meta": {"hostvars": {}}, + "all": {"hosts": [], "vars": {}}, + } + + # Pre-create parent groups with children relationships + for parent, children in GROUP_CHILDREN.items(): + inventory[parent] = {"children": children, "hosts": []} + + for machine in machines: + # Only include Deployed machines + if machine.get("status") != 6: + continue + + hostname = machine.get("hostname", "") + if not hostname: + continue + + ip = get_preferred_ip(machine, config.get("network", "")) + tags = machine.get("tag_names", []) + + # Build hostvars + hostvars = { + "ansible_python_interpreter": "/usr/bin/python3", + } + if ip: + hostvars["ansible_host"] = ip + if config.get("ssh_user"): + hostvars["ansible_user"] = config["ssh_user"] + if config.get("ssh_bastion"): + hostvars["ansible_ssh_common_args"] = ( + f'-o ProxyJump="{config["ssh_bastion"]}"' + ) + + # MAAS metadata + hostvars["maas_system_id"] = machine.get("system_id", "") + hostvars["maas_fqdn"] = machine.get("fqdn", "") + hostvars["maas_status"] = machine.get("status_name", "") + hostvars["maas_zone"] = machine.get("zone", {}).get("name", "") + hostvars["maas_pool"] = machine.get("pool", {}).get("name", "") + hostvars["maas_tags"] = tags + hostvars["maas_arch"] = machine.get("architecture", "") + hostvars["maas_os"] = machine.get("osystem", "") + hostvars["maas_distro"] = machine.get("distro_series", "") + hostvars["maas_cpus"] = machine.get("cpu_count", 0) + hostvars["maas_memory_mb"] = machine.get("memory", 0) + hostvars["maas_power_state"] = machine.get("power_state", "") + + inventory["_meta"]["hostvars"][hostname] = hostvars + inventory["all"]["hosts"].append(hostname) + + # Map tags to Ansible groups + for tag in tags: + if tag not in inventory: + inventory[tag] = {"hosts": [], "vars": {}} + elif "hosts" not in inventory[tag]: + inventory[tag]["hosts"] = [] + inventory[tag]["hosts"].append(hostname) + + return inventory + + +def main(): + parser = argparse.ArgumentParser( + description="MAAS Dynamic Inventory for DeepOps" + ) + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("--list", action="store_true", help="List all hosts") + group.add_argument("--host", help="Get variables for a specific host") + args = parser.parse_args() + + config = load_config() + + if args.list: + inventory = build_inventory(config) + print(json.dumps(inventory, indent=2)) + elif args.host: + # Ansible uses hostvars from _meta in --list and typically does + # not call --host when _meta is provided. Return empty dict to + # avoid an unnecessary MAAS API call. + print(json.dumps({})) + + +if __name__ == "__main__": + main()