diff --git a/CLAUDE.md b/CLAUDE.md index c40e0a900..158fd399c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -264,7 +264,7 @@ Handler source is grouped under `src/components/core/`: `getComputeResult` (+ `getComputeStreamableLogs`) → `stopCompute`. Paid compute settles via the `Escrow` component; `serviceResourceMatching.ts` maps requested cpu/ram/disk/gpu against environment pools (dual-gate: per-env ceiling + engine-wide pool; GPUs tracked globally). - See `docs/compute-pricing.md`, `docs/GPU.md`. + See `docs/compute.md`. - **database/** — `Database.init()` factory (`index.ts`, `DatabaseFactory.ts`). The metadata DB backend is pluggable: **Typesense or Elasticsearch** (chosen by `DB_TYPE`) for DDOs, indexer state, logs, orders, ddoState, access lists, escrow events — behind the @@ -317,5 +317,5 @@ can talk to `/var/run/docker.sock`. Deployment options (Docker, local Docker bui `Arhitecture.md` (note the spelling), `API.md` (full HTTP API reference — very large, plus a Postman collection), `env.md` (authoritative env-var reference), `database.md`, `Storage.md` / `persistentStorage.md`, `KeyManager.md`, `PolicyServer.md`, `services.md` -(Service-on-Demand), `compute-pricing.md` / `GPU.md` (C2D), `networking.md`, `Logs.md`, +(Service-on-Demand), `compute.md` (C2D configuration: resources, GPUs, constraints, pricing), `networking.md`, `Logs.md`, `Publishing.md`, `testing.md`, `dockerDeployment.md`. diff --git a/README.md b/README.md index 7e211fe6b..6a5233b23 100644 --- a/README.md +++ b/README.md @@ -154,6 +154,5 @@ Your node is now running. To start additional nodes, repeat these steps in a new - [Network Configuration](docs/networking.md) - [Logging & accessing logs](docs/networking.md) - [Docker Deployment Guide](docs/dockerDeployment.md) -- [C2D GPU Guide](docs/GPU.md) -- [Compute pricing](docs/compute-pricing.md) +- [Compute (C2D) Configuration — resources, GPUs, constraints, pricing](docs/compute.md) - [Services (Service-on-Demand)](docs/services.md) diff --git a/docs/GPU.md b/docs/GPU.md deleted file mode 100644 index e67af2a85..000000000 --- a/docs/GPU.md +++ /dev/null @@ -1,562 +0,0 @@ -Supporting GPUs for c2d jobs comes down to: - -- define each GPU as a named resource at the **connection level** (same level as `socketPath`) -- pass docker device args inside each GPU's `init` block -- reference the GPU by id in the environment's `resources` list -- set a price for each GPU in the environment's `fees` (see [Compute pricing](compute-pricing.md)) - -## Key rules - -- Each physical GPU is its own resource with a unique id and exactly **one** `DeviceID`. -- `kind: "discrete"` (non-fungible): only one job at a time can use the device. This is the default when `init` is present. -- `cpu`, `ram`, and `disk` are **auto-detected** from the host — you do not need to declare them unless you want to cap their totals. -- Environment `resources` are **lightweight refs** (`id` + optional `total`/`min`/`max`/`constraints`). Hardware details (`init`, `driverVersion`, `platform`, etc.) live only at connection level. - -> **Security note**: `init.advanced` entries (`Binds`, `CapAdd`, `Devices`, `SecurityOpt`) apply to every job in every environment that references the resource. Review them carefully before adding to production configs. - ---- - -## NVIDIA GPU Example - -Install nvidia cuda drivers (https://docs.nvidia.com/cuda/cuda-installation-guide-linux/) and nvidia container toolkit (https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html). - -Check your GPU details: - -``` -root@gpu-1:/repos/ocean/ocean-node# nvidia-smi -Fri Apr 25 06:00:34 2025 -+-----------------------------------------------------------------------------------------+ -| NVIDIA-SMI 550.163.01 Driver Version: 550.163.01 CUDA Version: 12.4 | -|-----------------------------------------+------------------------+----------------------+ -| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC | -| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. | -| | | N/A | -|=========================================+========================+======================| -| 0 NVIDIA GeForce GTX 1060 3GB Off | 00000000:01:00.0 Off | N/A | -| 0% 39C P8 6W / 120W | 2MiB / 3072MiB | 0% Default | -| | | N/A | -+-----------------------------------------+------------------------+----------------------+ - -+-----------------------------------------------------------------------------------------+ -| Processes: | -| GPU GI CI PID Type Process name GPU Memory | -| ID ID Usage | -|=========================================================================================| -| No running processes found | -+-----------------------------------------------------------------------------------------+ -``` - -Get the GPU UUID: - -```bash -root@gpu-1:/repos/ocean/ocean-node# nvidia-smi --query-gpu=name,uuid,driver_version,memory.total --format=csv -name, uuid, driver version, memory total -NVIDIA GeForce GTX 1060 3GB, GPU-294c6802-bb2f-fedb-f9e0-a26b9142dd81, 570.195.03, 3072 MiB -``` - -Full `DOCKER_COMPUTE_ENVIRONMENTS` configuration: - -```json -[ - { - "socketPath": "/var/run/docker.sock", - - "resources": [ - { - "id": "gpu0", - "kind": "discrete", - "type": "gpu", - "total": 1, - "description": "NVIDIA GeForce GTX 1060 3GB", - "platform": "nvidia", - "driverVersion": "570.195.03", - "memoryTotal": "3072 MiB", - "init": { - "deviceRequests": { - "Driver": "nvidia", - "DeviceIDs": ["GPU-294c6802-bb2f-fedb-f9e0-a26b9142dd81"], - "Capabilities": [["gpu"]] - } - }, - "constraints": [ - { "id": "ram", "min": 2 }, - { "id": "cpu", "min": 1 } - ] - } - ], - - "environments": [ - { - "id": "gpu-env", - "description": "NVIDIA GPU environment", - "storageExpiry": 604800, - "maxJobDuration": 3600, - "minJobDuration": 60, - "enableNetwork": false, - "resources": [ - { "id": "cpu", "min": 1, "max": 4 }, - { "id": "ram", "min": 1, "max": 8 }, - { "id": "disk", "min": 1, "max": 50 }, - { "id": "gpu0" } - ], - "access": { "addresses": [], "accessLists": null }, - "fees": { - "1": [ - { - "feeToken": "0x967da4048cD07aB37855c090aAF366e4ce1b9F48", - "prices": [ - { "id": "cpu", "price": 1 }, - { "id": "gpu0", "price": 3 } - ] - } - ] - }, - "free": { - "maxJobDuration": 60, - "minJobDuration": 10, - "maxJobs": 3, - "access": { "addresses": [], "accessLists": null }, - "resources": [ - { "id": "cpu", "max": 1 }, - { "id": "ram", "max": 2 }, - { "id": "disk", "max": 5 }, - { "id": "gpu0" } - ] - } - } - ] - } -] -``` - -Verify: - -```bash -curl http://localhost:8000/api/services/computeEnvironments -``` - -The response includes `resources` with the GPU fully resolved (including `init`, `driverVersion`, etc.) and `inUse` counters. - -Start a free GPU job: - -```json -{ - "command": "freeStartCompute", - "algorithm": { - "meta": { - "container": { - "image": "tensorflow/tensorflow", - "tag": "2.17.0-gpu", - "entrypoint": "python $ALGO" - }, - "rawcode": "import tensorflow as tf\nprint('Num GPUs Available:', len(tf.config.list_physical_devices('GPU')))" - } - }, - "consumerAddress": "0x00", - "signature": "123", - "nonce": 1, - "environment": "", - "resources": [ - { "id": "cpu", "amount": 1 }, - { "id": "gpu0", "amount": 1 } - ] -} -``` - ---- - -## AMD Radeon GPU Example - -Install ROCm (https://rocm.docs.amd.com/projects/radeon/en/latest/docs/install/wsl/install-radeon.html). - -```json -[ - { - "socketPath": "/var/run/docker.sock", - - "resources": [ - { - "id": "gpu0", - "kind": "discrete", - "type": "gpu", - "total": 1, - "description": "AMD Radeon RX 9070 XT", - "driverVersion": "26.2.2", - "memoryTotal": "16384 MiB", - "init": { - "advanced": { - "IpcMode": "host", - "ShmSize": 8589934592, - "CapAdd": ["SYS_PTRACE"], - "Devices": ["/dev/dxg", "/dev/dri/card0"], - "Binds": [ - "/usr/lib/wsl/lib/libdxcore.so:/usr/lib/libdxcore.so", - "/opt/rocm/lib/libhsa-runtime64.so.1:/opt/rocm/lib/libhsa-runtime64.so.1" - ], - "SecurityOpt": { "seccomp": "unconfined" } - } - }, - "constraints": [ - { "id": "ram", "min": 4 }, - { "id": "cpu", "min": 2 } - ] - } - ], - - "environments": [ - { - "id": "amd-gpu-env", - "description": "AMD Radeon GPU environment", - "storageExpiry": 604800, - "maxJobDuration": 3600, - "minJobDuration": 60, - "enableNetwork": false, - "resources": [ - { "id": "cpu", "min": 1, "max": 4 }, - { "id": "ram", "min": 1, "max": 16 }, - { "id": "disk", "min": 1, "max": 50 }, - { "id": "gpu0" } - ], - "access": { "addresses": [], "accessLists": null }, - "fees": { - "1": [ - { - "feeToken": "0x967da4048cD07aB37855c090aAF366e4ce1b9F48", - "prices": [ - { "id": "cpu", "price": 1 }, - { "id": "gpu0", "price": 3 } - ] - } - ] - }, - "free": { - "maxJobDuration": 60, - "minJobDuration": 10, - "maxJobs": 3, - "access": { "addresses": [], "accessLists": null }, - "resources": [ - { "id": "cpu", "max": 1 }, - { "id": "ram", "max": 4 }, - { "id": "disk", "max": 5 }, - { "id": "gpu0" } - ] - } - } - ] - } -] -``` - -Start a free job: - -```json -{ - "command": "freeStartCompute", - "algorithm": { - "meta": { - "container": { - "image": "rocm/tensorflow", - "tag": "rocm6.4-py3.12-tf2.18-dev", - "entrypoint": "python $ALGO" - }, - "rawcode": "import tensorflow as tf\nprint('Num GPUs Available:', len(tf.config.list_physical_devices('GPU')))" - } - }, - "consumerAddress": "0x00", - "signature": "123", - "nonce": 1, - "environment": "", - "resources": [ - { "id": "cpu", "amount": 1 }, - { "id": "gpu0", "amount": 1 } - ] -} -``` - ---- - -## Intel Arc GPU Example - -Install Intel GPU drivers (https://dgpu-docs.intel.com/driver/installation.html). - -```bash -root@gpu-1:/repos/ocean/ocean-node# clinfo -Number of platforms: 1 - Platform #0: Intel(R) OpenCL Graphics - Number of devices: 1 - Device #0: Intel(R) Arc(TM) A770M Graphics - Board name: Intel Arc Graphics - Vendor ID: 0x8086 - Device ID: 0x56a0 - Device Topology (NV12): PCI[ B#3 D#0 F#0 ] - Max compute units: 32 - Max clock frequency: 2400 MHz -``` - -```json -[ - { - "socketPath": "/var/run/docker.sock", - - "resources": [ - { - "id": "gpu0", - "kind": "discrete", - "type": "gpu", - "total": 1, - "description": "Intel Arc A770M Graphics", - "driverVersion": "32.0.101.8531", - "memoryTotal": "16384 MiB", - "init": { - "advanced": { - "Devices": ["/dev/dri/renderD128", "/dev/dri/card0"], - "GroupAdd": ["video", "render"], - "CapAdd": ["SYS_ADMIN"] - } - }, - "constraints": [ - { "id": "ram", "min": 2 }, - { "id": "cpu", "min": 1 } - ] - } - ], - - "environments": [ - { - "id": "intel-gpu-env", - "description": "Intel Arc GPU environment", - "storageExpiry": 604800, - "maxJobDuration": 3600, - "minJobDuration": 60, - "enableNetwork": false, - "resources": [ - { "id": "cpu", "min": 1, "max": 4 }, - { "id": "ram", "min": 1, "max": 8 }, - { "id": "disk", "min": 1, "max": 50 }, - { "id": "gpu0" } - ], - "access": { "addresses": [], "accessLists": null }, - "fees": { - "1": [ - { - "feeToken": "0x967da4048cD07aB37855c090aAF366e4ce1b9F48", - "prices": [ - { "id": "cpu", "price": 1 }, - { "id": "gpu0", "price": 2 } - ] - } - ] - }, - "free": { - "maxJobDuration": 60, - "minJobDuration": 10, - "maxJobs": 3, - "access": { "addresses": [], "accessLists": null }, - "resources": [ - { "id": "cpu", "max": 1 }, - { "id": "ram", "max": 2 }, - { "id": "disk", "max": 5 }, - { "id": "gpu0" } - ] - } - } - ] - } -] -``` - ---- - -## Multiple GPUs — Shared Between Environments - -Each physical GPU is its own resource. Both environments can reference both GPUs; the engine tracks usage globally so no GPU is ever double-allocated. - -```json -[ - { - "socketPath": "/var/run/docker.sock", - - "resources": [ - { - "id": "gpu0", - "kind": "discrete", - "type": "gpu", - "total": 1, - "description": "NVIDIA A100 40GB (slot 0)", - "platform": "nvidia", - "driverVersion": "570.195.03", - "memoryTotal": "40960 MiB", - "init": { - "deviceRequests": { - "Driver": "nvidia", - "DeviceIDs": ["GPU-uuid-a"], - "Capabilities": [["gpu"]] - } - }, - "constraints": [ - { "id": "ram", "min": 8 }, - { "id": "cpu", "min": 2 } - ] - }, - { - "id": "gpu1", - "kind": "discrete", - "type": "gpu", - "total": 1, - "description": "NVIDIA A100 40GB (slot 1)", - "platform": "nvidia", - "driverVersion": "570.195.03", - "memoryTotal": "40960 MiB", - "init": { - "deviceRequests": { - "Driver": "nvidia", - "DeviceIDs": ["GPU-uuid-b"], - "Capabilities": [["gpu"]] - } - }, - "constraints": [ - { "id": "ram", "min": 8 }, - { "id": "cpu", "min": 2 } - ] - } - ], - - "environments": [ - { - "id": "premium", - "description": "Full GPU access", - "storageExpiry": 604800, - "maxJobDuration": 3600, - "minJobDuration": 60, - "enableNetwork": true, - "resources": [ - { "id": "cpu", "total": 16, "min": 1, "max": 8 }, - { "id": "ram", "total": 60, "min": 1, "max": 32 }, - { "id": "disk", "total": 200, "min": 1, "max": 100 }, - { "id": "gpu0" }, - { "id": "gpu1" } - ], - "access": { "addresses": [], "accessLists": null }, - "fees": { - "1": [ - { - "feeToken": "0x967da4048cD07aB37855c090aAF366e4ce1b9F48", - "prices": [ - { "id": "cpu", "price": 1 }, - { "id": "ram", "price": 0.5 }, - { "id": "gpu0", "price": 10 }, - { "id": "gpu1", "price": 10 } - ] - } - ] - } - }, - { - "id": "standard", - "description": "CPU only", - "storageExpiry": 604800, - "maxJobDuration": 1800, - "minJobDuration": 60, - "enableNetwork": false, - "resources": [ - { "id": "cpu", "total": 8, "min": 1, "max": 4 }, - { "id": "ram", "total": 16, "min": 1, "max": 8 }, - { "id": "disk", "total": 50, "min": 1, "max": 50 } - ], - "access": { "addresses": [], "accessLists": null }, - "fees": { - "1": [ - { - "feeToken": "0x967da4048cD07aB37855c090aAF366e4ce1b9F48", - "prices": [ - { "id": "cpu", "price": 0.5 }, - { "id": "ram", "price": 0.2 } - ] - } - ] - }, - "free": { - "maxJobDuration": 300, - "minJobDuration": 10, - "maxJobs": 3, - "access": { "addresses": [], "accessLists": null }, - "resources": [ - { "id": "cpu", "max": 1 }, - { "id": "ram", "max": 2 }, - { "id": "disk", "max": 5 } - ] - } - } - ] - } -] -``` - ---- - -## Shareable Devices (NIC, TPM, HSM) - -Use `kind: "discrete"` and `shareable: true` for devices that multiple jobs may use simultaneously. The engine tracks `inUse` for visibility but never blocks allocation. - -```json -{ - "id": "nic0", - "kind": "discrete", - "shareable": true, - "type": "network", - "total": 1, - "description": "SR-IOV NIC", - "init": { - "advanced": { - "Devices": [{ "PathOnHost": "/dev/net/tun", "PathInContainer": "/dev/net/tun" }] - } - } -} -``` - -> `shareable: true` is **not** allowed on `type: "gpu"` or `type: "fpga"` — the node will refuse to start. GPUs and FPGAs require exclusive per-job access. - ---- - -## Resource Constraints - -Constraints on a GPU resource define minimum companion resources required per job. When a user requests the GPU, the engine automatically allocates at least the constrained amounts. This also prevents the GPU from being scheduled when fungible resources (RAM, CPU) are exhausted. - -```json -{ - "id": "gpu0", - "kind": "discrete", - "constraints": [ - { "id": "ram", "min": 8 }, - { "id": "cpu", "min": 2 }, - { "id": "disk", "min": 10 } - ] -} -``` - -Environments can override pool-level constraints via the `EnvironmentResourceRef`: - -```json -{ "id": "gpu0", "constraints": [{ "id": "ram", "min": 16 }, { "id": "cpu", "min": 4 }] } -``` - -Set `"constraints": []` to remove all constraints for a specific environment. - ---- - -## Migration from old format - -The old format placed hardware resources (`init`, `driverVersion`, etc.) inside environments. This is now a startup error. - -**Old (rejected):** -```json -"environments": [{ "resources": [{ "id": "myGPU", "total": 1, "init": {...} }] }] -``` - -**New:** -```json -"resources": [{ "id": "myGPU", "kind": "discrete", "total": 1, "init": {...} }], -"environments": [{ "resources": [{ "id": "myGPU" }] }] -``` - -Move all `init`, `description`, `driverVersion`, `platform`, `memoryTotal`, `type`, `kind`, and `constraints` fields to the connection-level `resources` array. Each environment's `resources` keeps only `id` and optionally `total`/`min`/`max`/`constraints`. diff --git a/docs/compute-pricing.md b/docs/compute-pricing.md deleted file mode 100644 index fc070654e..000000000 --- a/docs/compute-pricing.md +++ /dev/null @@ -1,344 +0,0 @@ -# Compute Environment Configuration and Pricing - -This guide explains how to configure your node's Docker compute environments and how to set prices for each resource. It covers the `DOCKER_COMPUTE_ENVIRONMENTS` variable (or equivalent config), the fee structure, pricing units, and examples for CPU, RAM, disk, and GPU. - -## Overview - -- **Configuration**: Define compute environments via the `DOCKER_COMPUTE_ENVIRONMENTS` environment variable (JSON) or via `config.json` under `dockerComputeEnvironments`. -- **Two-level layout**: Resources are defined at the **Docker-connection level** (`socketPath`) and referenced by each environment. This lets multiple environments share the same hardware (e.g. both a paid and a free environment can use the same GPU). -- **Auto-detection**: `cpu` and `ram` are automatically detected from the host at startup. `disk` is measured via `statfs`. You only need to declare them in `resources` if you want to cap/override the detected value. -- **Resources**: The connection-level `resources` array holds full hardware definitions. Each environment's `resources` array holds lightweight refs (`{ id, total?, min?, max? }`) pointing to those pool entries. -- **Dual-gate tracking** (fungible resources like CPU/RAM/disk): Gate 1 enforces the per-environment ceiling; Gate 2 enforces the engine-wide physical pool ceiling. Both must pass for a job to be admitted. -- **Pricing**: For each chain and fee token, you set a `price` per resource. Cost is computed as **price × amount × duration (in minutes, rounded up)**. -- **Free tier**: Environments can have a `free` block that permits jobs with no payment, but with tighter resource limits. -- **Image building**: Free jobs cannot build images (Dockerfiles are not allowed). For paid jobs, image build time counts toward billable duration. - -## Pricing Units - -| Resource | Unit of `amount` | Price meaning | Cost formula | -| -------- | ---------------- | ------------------ | ------------------------------------ | -| **CPU** | Number of CPUs | Per CPU per minute | `price × cpus × ceil(duration/60)` | -| **RAM** | Gigabytes (GB) | Per GB per minute | `price × ramGB × ceil(duration/60)` | -| **Disk** | Gigabytes (GB) | Per GB per minute | `price × diskGB × ceil(duration/60)` | -| **GPU** | Number of GPUs | Per GPU per minute | `price × gpus × ceil(duration/60)` | - -Duration is always in seconds; it is converted to minutes with **ceil(duration / 60)** (e.g. 61 seconds → 2 minutes). - ---- - -## Where to Configure - -1. **Environment variable** - Set `DOCKER_COMPUTE_ENVIRONMENTS` to a JSON string (array of Docker-connection objects). - `export DOCKER_COMPUTE_ENVIRONMENTS='[{"socketPath":"/var/run/docker.sock",...}]'` - -2. **Config file** - Put the same array in your JSON config under the key `dockerComputeEnvironments`, and point the node to that file (e.g. via `CONFIG_PATH`). - -If both are set, the environment variable overrides the config. See [env.md](env.md) for all available fields. - ---- - -## Configuration Layout - -``` -DOCKER_COMPUTE_ENVIRONMENTS -└── [ Docker connection ] ← socketPath, resources[], environments[] - ├── resources[] ← full hardware definitions (CPU, RAM, disk, GPU, …) - │ ├── { id: "cpu", total: 6 } (optional: caps auto-detected value) - │ ├── { id: "ram", total: 28 } (optional: caps auto-detected value) - │ ├── { id: "disk", total: 80 } (optional: caps auto-detected value) - │ └── { id: "gpu0", kind: "discrete", … } (required for custom hardware) - └── environments[] ← one or more compute environments - └── { id, fees, resources[], free? } - └── resources[] ← lightweight refs to the pool above - ├── { id: "cpu", total: 4, min: 1, max: 4 } - ├── { id: "ram", total: 16, min: 1, max: 8 } - ├── { id: "disk", max: 20 } - └── { id: "gpu0" } -``` - -`cpu`, `ram`, and `disk` are **auto-detected** at startup — you do not need to declare them in the connection-level `resources` array. Include them only to cap the detected value (e.g. limit an 8-core host to 6 cores for compute). Custom hardware (GPUs, NICs) must always be declared. - ---- - -## Connection-level Resource Fields - -These fields go in the `resources` array at the Docker-connection level: - -| Field | Description | -|---|---| -| `id` | Unique identifier used in env refs and `fees.prices[].id` (e.g. `"cpu"`, `"gpu0"`) | -| `total` | Maximum units available in the pool. For `cpu`/`ram`/`disk`, caps the auto-detected value. | -| `kind` | `"fungible"` (CPU, RAM, disk — interchangeable units) or `"discrete"` (GPU, FPGA — named device). Auto-inferred if omitted: `"discrete"` when `init` is present, `"fungible"` otherwise. | -| `shareable` | `discrete` only. `true` → multiple jobs may use the device simultaneously (e.g. NIC, TPM). `false` (default) → exclusive per job (GPU, FPGA). | -| `min` | Minimum units per job request | -| `max` | Maximum units per job request (defaults to `total`) | -| `type` | Hint string: `"cpu"`, `"ram"`, `"disk"`, `"gpu"` | -| `description` | Human-readable label shown in `getComputeEnvironments` | -| `driverVersion` | GPU driver version string | -| `memoryTotal` | GPU VRAM string (e.g. `"40960 MiB"`) | -| `platform` | GPU vendor: `"nvidia"`, `"amd"`, `"intel"` | -| `init` | Docker container configuration (`deviceRequests` for NVIDIA, `advanced` for AMD/Intel). Makes `kind` default to `"discrete"`. | -| `constraints` | `[{ id, min?, max? }]` — companion resource requirements. When a job rents this resource, linked resources are auto-bumped to their minimums. | - -## Environment-level Resource Ref Fields - -These fields go in each environment's `resources` array (lightweight refs to the pool): - -| Field | Description | -|---|---| -| `id` | Must match a connection-level resource `id`. `cpu`, `ram`, `disk` are always valid (auto-detected). | -| `total` | Environment aggregate ceiling: maximum units all jobs in this environment can use simultaneously. Omit to default to the pool total. | -| `min` | Per-job minimum override for this environment | -| `max` | Per-job maximum override (capped to `total` if both are set) | -| `constraints` | Per-env override: replaces the pool resource's constraints entirely for this environment. Omit to inherit pool constraints. Set `[]` to remove all constraints for this env. | - ---- - -## Fee Structure - -`fees` is an object keyed by **chain ID** (string). Each value is an array of fee options: - -```json -"fees": { - "1": [ - { - "feeToken": "0x...", - "prices": [ - { "id": "cpu", "price": 1 }, - { "id": "ram", "price": 0.5 }, - { "id": "disk", "price": 0.2 }, - { "id": "gpu0", "price": 3 } - ] - } - ] -} -``` - -- **feeToken**: Token contract address used for payment on that chain. -- **prices**: List of `{ "id": "", "price": }`. - The `id` in `prices` must match a connection-level resource `id`. Only resources listed here are billable; omit a resource to offer it at no charge. - ---- - -## Cost Examples - -Assume: - -- **CPU** price = 1 (per CPU per minute) -- **RAM** price = 0.5 (per GB per minute) -- **Disk** price = 0.2 (per GB per minute) -- **GPU** price = 3 (per GPU per minute) - -Job: 2 CPUs, 4 GB RAM, 10 GB disk, 1 GPU, duration **125 seconds** (ceil = 3 minutes). - -- CPU: 1 × 2 × 3 = **6** -- RAM: 0.5 × 4 × 3 = **6** -- Disk: 0.2 × 10 × 3 = **6** -- GPU: 3 × 1 × 3 = **9** - **Total cost = 27** (in the smallest unit of the fee token). - ---- - -## Example 1: CPU, RAM, and Disk with Prices - -`cpu` and `ram` are auto-detected, so they don't need to appear in `resources`. Only `disk` is declared here to cap it at 10 GB. - -```json -[ - { - "socketPath": "/var/run/docker.sock", - "resources": [ - { "id": "disk", "total": 10 } - ], - "environments": [ - { - "storageExpiry": 604800, - "maxJobDuration": 3600, - "minJobDuration": 60, - "resources": [ - { "id": "cpu" }, - { "id": "ram" }, - { "id": "disk", "max": 10 } - ], - "fees": { - "1": [ - { - "feeToken": "0x967da4048cD07aB37855c090aAF366e4ce1b9F48", - "prices": [ - { "id": "cpu", "price": 1 }, - { "id": "ram", "price": 0.5 }, - { "id": "disk", "price": 0.2 } - ] - } - ] - }, - "free": { - "maxJobDuration": 60, - "minJobDuration": 10, - "maxJobs": 3, - "resources": [ - { "id": "cpu", "max": 1 }, - { "id": "ram", "max": 1 }, - { "id": "disk", "max": 1 } - ] - } - } - ] - } -] -``` - ---- - -## Example 2: CPU + NVIDIA GPU - -Get the GPU UUID: - -```bash -nvidia-smi --query-gpu=name,uuid --format=csv -``` - -Define the GPU at the **connection level** with `kind: "discrete"` and a single `DeviceID`. Each physical GPU is its own resource entry. The environment references it by `id`. - -```json -[ - { - "socketPath": "/var/run/docker.sock", - "resources": [ - { "id": "disk", "total": 10 }, - { - "id": "gpu0", - "kind": "discrete", - "type": "gpu", - "total": 1, - "description": "NVIDIA GeForce GTX 1060 3GB", - "platform": "nvidia", - "init": { - "deviceRequests": { - "Driver": "nvidia", - "DeviceIDs": ["GPU-294c6802-bb2f-fedb-f9e0-a26b9142dd81"], - "Capabilities": [["gpu"]] - } - } - } - ], - "environments": [ - { - "storageExpiry": 604800, - "maxJobDuration": 3600, - "minJobDuration": 60, - "resources": [ - { "id": "cpu" }, - { "id": "ram" }, - { "id": "disk", "max": 10 }, - { "id": "gpu0" } - ], - "fees": { - "1": [ - { - "feeToken": "0x123", - "prices": [ - { "id": "cpu", "price": 1 }, - { "id": "gpu0", "price": 3 } - ] - } - ] - }, - "free": { - "maxJobDuration": 60, - "minJobDuration": 10, - "maxJobs": 3, - "resources": [ - { "id": "cpu", "max": 1 }, - { "id": "ram", "max": 1 }, - { "id": "disk", "max": 1 } - ] - } - } - ] - } -] -``` - -The `id` in `fees.prices` (`"gpu0"`) must match the connection-level resource `id`. Price 3 = 3 units per GPU per minute. - ---- - -## Example 3: Multiple Chains and Tokens - -You can support several chains and multiple fee tokens per chain: - -```json -"fees": { - "1": [ - { - "feeToken": "0xTokenA", - "prices": [ - { "id": "cpu", "price": 1 }, - { "id": "ram", "price": 0.5 }, - { "id": "disk", "price": 0.2 } - ] - }, - { - "feeToken": "0xTokenB", - "prices": [ - { "id": "cpu", "price": 2 }, - { "id": "ram", "price": 1 }, - { "id": "disk", "price": 0.5 } - ] - } - ], - "137": [ - { - "feeToken": "0xPolygonToken", - "prices": [ - { "id": "cpu", "price": 1 }, - { "id": "ram", "price": 0.5 }, - { "id": "disk", "price": 0.2 } - ] - } - ] -} -``` - -Consumers choose chain and token when starting a job; the node uses the matching `prices` for that chain and token. - ---- - -## Example 4: AMD or Intel GPU - -For **AMD (e.g. ROCm)** or **Intel Arc**, define the GPU at the connection level with `init.advanced` instead of `deviceRequests`. The environment references it by `id` just like any other resource. - -- **AMD Radeon**: See [GPU.md – AMD Radeon](GPU.md#amd-radeon-example) for the `advanced` block (Devices, Binds, CapAdd, etc.). -- **Intel Arc**: See [GPU.md – Intel Arc GPU](GPU.md#intel-arc-gpu-example) for the `advanced` block (Devices, GroupAdd, CapAdd). - -In all cases, the pricing rule is: **price × amount × ceil(duration/60)** with amount = number of GPUs. - ---- - -## Dual-gate Availability (Fungible Resources) - -For `cpu`, `ram`, and `disk`, two independent checks must pass before a job is admitted: - -- **Gate 1 (per-environment ceiling)**: `env.total - env.inUse >= requested`. Controlled by `EnvironmentResourceRef.total` in the environment's `resources` array. Prevents one environment from starving others. -- **Gate 2 (engine-wide pool ceiling)**: The sum of in-use across all environments must not exceed the pool's `total`. Enforces the physical hardware limit. - -For discrete resources (GPU), only Gate 2 applies — and only for exclusive (`shareable: false`, the default) devices. - ---- - -## Checklist - -- [ ] `DOCKER_COMPUTE_ENVIRONMENTS` (or `dockerComputeEnvironments` in config) is a JSON array. -- [ ] GPUs and other custom hardware are defined in the **connection-level** `resources` array with `kind: "discrete"`. -- [ ] Each environment's `resources` array contains lightweight refs (`{ id, total?, min?, max? }`). -- [ ] `cpu`, `ram`, `disk` are auto-detected — only declare them in `resources` to cap/override the detected value. -- [ ] **Disk** and **RAM** amounts are in **GB**. -- [ ] `fees.prices[].id` matches a connection-level resource `id`. -- [ ] **CPU / GPU**: price = per resource per minute. -- [ ] **RAM / Disk**: price = per GB per minute. -- [ ] For free tier, list the same resource ids in `free.resources`; omit from `prices` if they should be free only. - -For GPU setup details (NVIDIA, AMD, Intel), see [GPU.md](GPU.md). For all env vars and config options, see [env.md](env.md). diff --git a/docs/compute.md b/docs/compute.md new file mode 100644 index 000000000..41616f52c --- /dev/null +++ b/docs/compute.md @@ -0,0 +1,932 @@ +# Compute (Compute-to-Data) Configuration + +The single reference for configuring your node's Docker compute environments — from +node-wide resource declaration, through GPU setup, down to per-environment constraints and +pricing. It covers the `DOCKER_COMPUTE_ENVIRONMENTS` variable (or `dockerComputeEnvironments` +config key), the two-level resource model, GPU vendors (NVIDIA/AMD/Intel), resource +constraints, availability gating, and the fee structure. + +> On-demand **services** run on the same compute environments and draw from the same +> resource pool described here — see [services.md](services.md). + +## Contents + +1. [Overview](#overview) +2. [Where to configure](#where-to-configure) +3. [Configuration layout](#configuration-layout) +4. [Node-wide (connection-level) resources](#node-wide-connection-level-resources) +5. [Configuring GPUs](#configuring-gpus) +6. [Environment-level resource refs](#environment-level-resource-refs) +7. [Resource constraints](#resource-constraints) +8. [Availability & tracking](#availability--tracking) +9. [Pricing](#pricing) +10. [Free tier](#free-tier) +11. [Verify](#verify) +12. [Checklist](#checklist) +13. [Full example](#full-example) + +--- + +## Overview + +- **Configuration**: Define compute environments via the `DOCKER_COMPUTE_ENVIRONMENTS` + environment variable (JSON) or via `config.json` under `dockerComputeEnvironments`. +- **Two-level layout**: Resources are defined once at the **Docker-connection level** + (`socketPath`) and **referenced** by each environment. This lets multiple environments + share the same hardware (e.g. both a paid and a free environment can use the same GPU). +- **Auto-detection**: `cpu` and `ram` are automatically detected from the host at startup. + `disk` is measured via `statfs`. You only need to declare them in `resources` if you want + to cap/override the detected value. +- **Resources**: The connection-level `resources` array holds full hardware definitions. + Each environment's `resources` array holds lightweight refs + (`{ id, total?, min?, max?, constraints? }`) pointing to those pool entries. +- **Dual-gate tracking** (fungible resources like CPU/RAM/disk): Gate 1 enforces the + per-environment ceiling; Gate 2 enforces the engine-wide physical pool ceiling. Both must + pass for a job to be admitted. +- **Pricing**: For each chain and fee token, you set a `price` per resource. Cost is + **price × amount × duration (in minutes, rounded up)**. +- **Free tier**: Environments can have a `free` block that permits jobs with no payment, but + with tighter resource limits. +- **Image building**: Free jobs cannot build images (Dockerfiles are not allowed). For paid + jobs, image build time counts toward billable duration. + +--- + +## Where to configure + +1. **Environment variable** + Set `DOCKER_COMPUTE_ENVIRONMENTS` to a JSON string (array of Docker-connection objects): + `export DOCKER_COMPUTE_ENVIRONMENTS='[{"socketPath":"/var/run/docker.sock",...}]'` + +2. **Config file** + Put the same array in your JSON config under the key `dockerComputeEnvironments`, and + point the node to that file (e.g. via `CONFIG_PATH`). + +If both are set, the environment variable overrides the config. See [env.md](env.md) for all +available fields. + +--- + +## Configuration layout + +``` +DOCKER_COMPUTE_ENVIRONMENTS +└── [ Docker connection ] ← socketPath, resources[], environments[] + ├── resources[] ← full hardware definitions (CPU, RAM, disk, GPU, …) + │ ├── { id: "cpu", total: 6 } (optional: caps auto-detected value) + │ ├── { id: "ram", total: 28 } (optional: caps auto-detected value) + │ ├── { id: "disk", total: 80 } (optional: caps auto-detected value) + │ └── { id: "gpu0", kind: "discrete", … } (required for custom hardware) + └── environments[] ← one or more compute environments + └── { id, fees, resources[], free? } + └── resources[] ← lightweight refs to the pool above + ├── { id: "cpu", total: 4, min: 1, max: 4, constraints: [...] } + ├── { id: "ram", total: 16, min: 1, max: 8 } + ├── { id: "disk", max: 20 } + └── { id: "gpu0" } +``` + +`cpu`, `ram`, and `disk` are **auto-detected** at startup — you do not need to declare them +in the connection-level `resources` array. Include them only to cap the detected value (e.g. +limit an 8-core host to 6 cores for compute). Custom hardware (GPUs, NICs) must always be +declared. + +--- + +## Node-wide (connection-level) resources + +These fields go in the `resources` array at the Docker-connection level: + +| Field | Description | +|---|---| +| `id` | Unique identifier used in env refs and `fees.prices[].id` (e.g. `"cpu"`, `"gpu0"`) | +| `total` | Maximum units available in the pool. For `cpu`/`ram`/`disk`, caps the auto-detected value. | +| `kind` | `"fungible"` (CPU, RAM, disk — interchangeable units) or `"discrete"` (GPU, FPGA — named device). Auto-inferred if omitted: `"discrete"` when `init` is present, `"fungible"` otherwise. | +| `shareable` | `discrete` only. `true` → multiple jobs may use the device simultaneously (e.g. NIC, TPM). `false` (default) → exclusive per job (GPU, FPGA). | +| `min` | Minimum units per job request | +| `max` | Maximum units per job request (defaults to `total`) | +| `type` | Hint string used for grouping and display: `"cpu"`, `"ram"`, `"disk"`, `"gpu"`, `"fpga"`, … Group constraints (see below) match on this. | +| `description` | Human-readable label shown in `getComputeEnvironments` | +| `driverVersion` | GPU driver version string | +| `memoryTotal` | GPU VRAM string (e.g. `"40960 MiB"`) | +| `platform` | GPU vendor: `"nvidia"`, `"amd"`, `"intel"` | +| `init` | Docker container configuration (`deviceRequests` for NVIDIA, `advanced` for AMD/Intel). Makes `kind` default to `"discrete"`. | +| `constraints` | Companion resource requirements — see [Resource constraints](#resource-constraints). | + +### CPU pinning with `cpuList` + +For the `cpu` resource only, you may pin jobs to specific host cores with `cpuList` instead +of `total` (the two are mutually exclusive). It is a cpuset string of ascending, +non-overlapping core IDs and/or ranges: `"3"`, `"0-1,3"`, `"0-15,32-47"` (no spaces, floats, +or negatives). The effective `total` becomes the number of expanded cores, and every id is +validated against the host's CPU count at startup. + +--- + +## Configuring GPUs + +Supporting GPUs comes down to: + +- define each GPU as a **named resource at the connection level** (same level as `socketPath`), +- pass Docker device args inside each GPU's `init` block, +- reference the GPU by `id` in the environment's `resources` list, +- set a price for each GPU in the environment's `fees` (see [Pricing](#pricing)). + +### Key rules + +- Each physical GPU is its own resource with a unique `id`. For NVIDIA (`init.deviceRequests`), + give each resource exactly **one** `DeviceID`; AMD/Intel instead map the device through + `init.advanced.Devices` (e.g. `/dev/dri/card0`, `/dev/dxg`, `/dev/dri/renderD128`). +- `kind: "discrete"` (non-fungible): only one job at a time can use the device. This is the + default when `init` is present. +- `cpu`, `ram`, and `disk` are **auto-detected** — you do not need to declare them unless you + want to cap their totals. +- Environment `resources` are **lightweight refs** (`id` + optional + `total`/`min`/`max`/`constraints`). Hardware details (`init`, `driverVersion`, `platform`, + etc.) live only at connection level. + +> **Security note**: `init.advanced` entries (`Binds`, `CapAdd`, `Devices`, `SecurityOpt`) +> apply to every job in every environment that references the resource. Review them carefully +> before adding to production configs. + +### NVIDIA GPU + +Install the [NVIDIA CUDA drivers](https://docs.nvidia.com/cuda/cuda-installation-guide-linux/) +and [NVIDIA container toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html). + +Get the GPU UUID: + +```bash +nvidia-smi --query-gpu=name,uuid,driver_version,memory.total --format=csv +# NVIDIA GeForce GTX 1060 3GB, GPU-294c6802-bb2f-fedb-f9e0-a26b9142dd81, 570.195.03, 3072 MiB +``` + +Define the GPU at the **connection level** with `kind: "discrete"` and a single `DeviceID`. +The environment references it by `id`. + +```json +[ + { + "socketPath": "/var/run/docker.sock", + "resources": [ + { + "id": "gpu0", + "kind": "discrete", + "type": "gpu", + "total": 1, + "description": "NVIDIA GeForce GTX 1060 3GB", + "platform": "nvidia", + "driverVersion": "570.195.03", + "memoryTotal": "3072 MiB", + "init": { + "deviceRequests": { + "Driver": "nvidia", + "DeviceIDs": ["GPU-294c6802-bb2f-fedb-f9e0-a26b9142dd81"], + "Capabilities": [["gpu"]] + } + }, + "constraints": [{ "id": "ram", "min": 2 }, { "id": "cpu", "min": 1 }] + } + ], + "environments": [ + { + "id": "gpu-env", + "description": "NVIDIA GPU environment", + "storageExpiry": 604800, + "maxJobDuration": 3600, + "minJobDuration": 60, + "resources": [ + { "id": "cpu", "min": 1, "max": 4 }, + { "id": "ram", "min": 1, "max": 8 }, + { "id": "disk", "min": 1, "max": 50 }, + { "id": "gpu0" } + ], + "fees": { + "1": [ + { + "feeToken": "0x967da4048cD07aB37855c090aAF366e4ce1b9F48", + "prices": [{ "id": "cpu", "price": 1 }, { "id": "gpu0", "price": 3 }] + } + ] + } + } + ] + } +] +``` + +### AMD Radeon (ROCm) + +Install [ROCm](https://rocm.docs.amd.com/projects/radeon/en/latest/docs/install/wsl/install-radeon.html), +then define the GPU with `init.advanced` instead of `deviceRequests`: + +```json +{ + "id": "gpu0", + "kind": "discrete", + "type": "gpu", + "total": 1, + "description": "AMD Radeon RX 9070 XT", + "driverVersion": "26.2.2", + "memoryTotal": "16384 MiB", + "init": { + "advanced": { + "IpcMode": "host", + "ShmSize": 8589934592, + "CapAdd": ["SYS_PTRACE"], + "Devices": ["/dev/dxg", "/dev/dri/card0"], + "Binds": [ + "/usr/lib/wsl/lib/libdxcore.so:/usr/lib/libdxcore.so", + "/opt/rocm/lib/libhsa-runtime64.so.1:/opt/rocm/lib/libhsa-runtime64.so.1" + ], + "SecurityOpt": { "seccomp": "unconfined" } + } + }, + "constraints": [{ "id": "ram", "min": 4 }, { "id": "cpu", "min": 2 }] +} +``` + +### Intel Arc + +Install the [Intel GPU drivers](https://dgpu-docs.intel.com/driver/installation.html), then: + +```json +{ + "id": "gpu0", + "kind": "discrete", + "type": "gpu", + "total": 1, + "description": "Intel Arc A770M Graphics", + "driverVersion": "32.0.101.8531", + "memoryTotal": "16384 MiB", + "init": { + "advanced": { + "Devices": ["/dev/dri/renderD128", "/dev/dri/card0"], + "GroupAdd": ["video", "render"], + "CapAdd": ["SYS_ADMIN"] + } + }, + "constraints": [{ "id": "ram", "min": 2 }, { "id": "cpu", "min": 1 }] +} +``` + +### Multiple GPUs shared between environments + +Each physical GPU is its own resource. Multiple environments can reference the same GPUs; the +engine tracks discrete usage **globally**, so no GPU is ever double-allocated across +environments. + +```json +"resources": [ + { "id": "gpu0", "kind": "discrete", "type": "gpu", "total": 1, + "init": { "deviceRequests": { "Driver": "nvidia", "DeviceIDs": ["GPU-uuid-a"], "Capabilities": [["gpu"]] } } }, + { "id": "gpu1", "kind": "discrete", "type": "gpu", "total": 1, + "init": { "deviceRequests": { "Driver": "nvidia", "DeviceIDs": ["GPU-uuid-b"], "Capabilities": [["gpu"]] } } } +], +"environments": [ + { "id": "premium", "resources": [ { "id": "cpu" }, { "id": "ram" }, { "id": "gpu0" }, { "id": "gpu1" } ], "fees": { ... } }, + { "id": "standard", "resources": [ { "id": "cpu" }, { "id": "ram" } ], "fees": { ... } } +] +``` + +### Shareable devices (NIC, TPM, HSM) + +Use `kind: "discrete"` and `shareable: true` for devices that multiple jobs may use +simultaneously. The engine tracks `inUse` for visibility but never blocks allocation on them. + +```json +{ + "id": "nic0", + "kind": "discrete", + "shareable": true, + "type": "network", + "total": 1, + "description": "SR-IOV NIC", + "init": { "advanced": { "Devices": [{ "PathOnHost": "/dev/net/tun", "PathInContainer": "/dev/net/tun" }] } } +} +``` + +> `shareable: true` is **not** allowed on `type: "gpu"` or `type: "fpga"` — the node refuses +> to start. GPUs and FPGAs require exclusive per-job access. + +### Migration from the old format + +The old format placed hardware resources (`init`, `driverVersion`, etc.) inside +environments. This is now a startup error. + +```json +// Old (rejected): +"environments": [{ "resources": [{ "id": "myGPU", "total": 1, "init": {...} }] }] + +// New: +"resources": [{ "id": "myGPU", "kind": "discrete", "total": 1, "init": {...} }], +"environments": [{ "resources": [{ "id": "myGPU" }] }] +``` + +Move all `init`, `description`, `driverVersion`, `platform`, `memoryTotal`, `type`, `kind`, +and `constraints` fields to the connection-level `resources` array. Each environment's +`resources` keeps only `id` and optionally `total`/`min`/`max`/`constraints`. + +--- + +## Environment-level resource refs + +These fields go in each environment's `resources` array (lightweight refs to the pool): + +| Field | Description | +|---|---| +| `id` | Must match a connection-level resource `id`. `cpu`, `ram`, `disk` are always valid (auto-detected) and are injected as baseline refs even if omitted. | +| `total` | Environment aggregate ceiling: maximum units all jobs in this environment can use simultaneously. Omit to default to the pool total. (Fungible resources only.) | +| `min` | Per-job minimum override for this environment | +| `max` | Per-job maximum override (capped to `total` if both are set) | +| `constraints` | Per-env override — see below. Replaces the pool resource's constraints entirely for this environment. Omit to inherit pool constraints. Set `[]` to remove all constraints for this env. | + +--- + +## Resource constraints + +A **constraint** ties one resource's requested amount to another's. Constraints are declared +on a "parent" resource (via its `constraints` array) and are evaluated only when that parent +is actually requested (`amount > 0`). There are two forms. + +### Constraint fields + +| Field | Description | +|---|---| +| `id` | Target a **single** resource by exact id (e.g. `"ram"`, `"gpu0"`). | +| `type` | Target a **group** of resources by `type` (e.g. `"gpu"`), aggregated across all resources of that type the environment exposes. Mutually exclusive with `id` — set exactly one. | +| `min` | Minimum units of the target. | +| `max` | Maximum units of the target. | +| `perUnit` | `true`/omitted → **ratio**: the bound is `parentAmount × value`. `false` → **floor/ceiling**: the bound is the absolute `value`, regardless of how much of the parent was requested. | +| `aggregate` | Single-`id` targets only. When `true`, this constraint's per-parent contribution is **summed** with matching aggregate constraints on other requested resources into one shared target — so per-device GPUs can jointly scale a companion resource (see below). Rejected on a `type` group. | + +Behavior: +- **min** — if the target is below the required minimum it is **auto-bumped** up to it (for a + group, the deficit is distributed across the group's members, preferring the ones with the + most availability). If the required minimum exceeds the target's aggregate max, the request + is rejected with `Cannot satisfy constraint …`. +- **max** — if the target exceeds the allowed maximum the request is rejected with + `Too much …` (never auto-reduced). + +### Companion constraints (single `id`, ratio) + +The classic use: a GPU that needs a minimum amount of companion RAM/CPU per unit. This also +prevents the GPU from being scheduled when fungible resources are exhausted. + +```json +{ + "id": "gpu0", + "kind": "discrete", + "constraints": [ + { "id": "ram", "min": 8 }, + { "id": "cpu", "min": 2 }, + { "id": "disk", "min": 10 } + ] +} +``` + +With `perUnit` defaulting to `true`, requesting 1 `gpu0` requires ≥ 8 GB RAM; the amounts +scale with the parent (`requiredMin = gpuAmount × min`). + +### `min` + `max` ratios: counted vs per-device resources + +Both `min` and `max` are supported, and with the default (ratio) semantics **both** scale by +the parent amount: `requiredMin = parentAmount × min`, `requiredMax = parentAmount × max`. +`min` **auto-bumps** the target up; `max` is **never auto-reduced** — exceeding it throws. + +Whether the ratio *aggregates across multiple GPUs* depends on how the GPUs are modeled, +because a constraint's **parent is always a single resource entry** — the engine evaluates +each resource's constraints on its own. (The `type` grouping described below applies only to +the constraint's *target*, not its parent, so there is no "sum across all GPUs" as the +parent.) + +Take a GPU constraint of `{ "id": "cpu", "min": 1, "max": 4 }` — "1–4 CPUs per GPU": + +**Counted model** — one `gpu` resource with `total: 2`. The parent amount *is* the number of +GPUs requested, so the ratio aggregates: + +| Request | Result | +|---|---| +| `gpu: 2` | `cpu` auto-bumped to **2** (= 2 × 1) | +| `gpu: 2, cpu: 8` | allowed (max = 2 × 4 = **8**) | +| `gpu: 2, cpu: 10` | rejected — `Too much cpu for 2 gpu. Max allowed: 8, requested: 10` | + +**Per-device model** — separate `GPU1` / `GPU2` resources (each `max: 1`), each carrying the +same constraint. Each is evaluated independently; the parent amount is only ever 0 or 1, so +the bounds **do not sum**: + +| Request | Result | +|---|---| +| `GPU1: 1, GPU2: 1` | `cpu` = **1**, *not* 2 — each GPU only requires ≥ 1, they don't add up | +| `GPU1: 1, GPU2: 1, cpu: 5` | rejected — `Too much cpu for 1 GPU1. Max allowed: 4` (max is **4** per GPU, not 8) | + +So the "N GPUs ⇒ N×min / N×max" behavior only holds for a single **counted** GPU resource. +The trade-off: a counted resource with explicit `DeviceIDs` attaches *all* of them to the +container regardless of the requested amount, so it can't cleanly pin "1 of 2 physical GPUs" +— use it when you don't need per-device selection. The recommended one-resource-per-GPU model +gives clean per-device allocation and global tracking, but its per-unit constraints are +per-GPU, not aggregated. + +#### `aggregate`: summing across per-device GPUs + +To get per-device allocation **and** the summed behavior, mark the constraint `aggregate: true` +on each GPU. Matching aggregate constraints (same target `id`) accumulate their per-parent +contribution across every requested GPU into one shared requirement: + +```json +{ "id": "GPU1", "constraints": [{ "id": "cpu", "min": 1, "max": 4, "aggregate": true }] }, +{ "id": "GPU2", "constraints": [{ "id": "cpu", "min": 1, "max": 4, "aggregate": true }] } +``` + +| Request | Result | +|---|---| +| `GPU1: 1` | `cpu` in **[1, 4]** (only one GPU contributes) | +| `GPU1: 1, GPU2: 1` | `cpu` auto-bumped to **2** (= 1+1); max is **8** (= 4+4) | +| `GPU1: 1, GPU2: 1, cpu: 10` | rejected — `Too much cpu for the requested resources. Max allowed: 8, requested: 10` | + +Notes: +- `aggregate` targets a **single `id`** (it sums *into* one resource); it cannot target a + `type` group — the config is rejected at startup if you try. +- The summed **min** auto-bumps the target up (capped by the target's own `max`); the summed + **max** is enforced but never auto-reduces. +- Non-aggregate constraints on the same target keep their independent per-parent behavior; the + aggregate pass runs after them and only raises the floor further. Avoid mixing aggregate and + non-aggregate constraints on the same target to keep behavior obvious. + +### Group constraints (`type`) + floor (`perUnit: false`) + +To express **"if any CPU is selected, the job needs at least 1 GPU — no matter which id"**, +target the `gpu` **group** with an absolute **floor**: + +```json +"environments": [ + { + "resources": [ + { "id": "cpu", "min": 1, "max": 16, + "constraints": [ { "type": "gpu", "min": 1, "perUnit": false } ] }, + { "id": "ram" }, + { "id": "disk" }, + { "id": "GPU1" }, + { "id": "GPU2" } + ] + } +] +``` + +- `type: "gpu"` aggregates across every `type:"gpu"` resource **the environment exposes** + (here `GPU1` + `GPU2`). +- `perUnit: false` makes `min: 1` an **absolute floor** — one GPU total is enough whether the + job asks for 1 CPU or 16 (a ratio would instead demand one GPU *per* CPU). +- If the consumer selects a CPU but requests no GPU, the engine auto-assigns one — picking the + GPU with the most availability. Because each physical GPU is its own single-`DeviceID` + resource, exactly that GPU is attached to the container. + +> **Place group constraints on the environment's ref, not on the connection-level pool.** +> A pool-level constraint is inherited by *every* environment; in a GPU-less environment the +> `type:"gpu"` group would be empty (aggregate max 0), so the floor of 1 could never be met +> and every CPU job there would be rejected with `Cannot satisfy`. + +### Per-env override + +Environments can override pool-level constraints via the `EnvironmentResourceRef`: + +```json +{ "id": "gpu0", "constraints": [{ "id": "ram", "min": 16 }, { "id": "cpu", "min": 4 }] } +``` + +Omit `constraints` to inherit the pool's; set `"constraints": []` to remove all constraints +for that environment. + +### More constraint recipes + +Common patterns, with the behavior verified against the constraint engine. Each snippet shows +just the relevant resource entry; place group/floor constraints on the environment's ref (see +the note above). + +**1. RAM range per GPU** — companion `min` + `max` (ratio): + +```json +{ "id": "gpu0", "constraints": [{ "id": "ram", "min": 8, "max": 16 }] } +``` + +Renting `gpu0` forces RAM into **[8, 16]** GB per GPU. RAM below 8 → bumped to 8; `ram: 20` → +rejected (`Too much ram for 1 gpu0. Max allowed: 16, requested: 20`). + +**2. One GPU per CPU (strict 1:1)** — group + ratio (`perUnit` omitted), on the env `cpu` ref: + +```json +{ "id": "cpu", "constraints": [{ "type": "gpu", "min": 1 }] } +``` + +`cpu: 2` → the engine auto-assigns **2 GPUs**; `cpu: 3` when only 2 GPUs are exposed → rejected +(`Cannot satisfy constraint: 3 cpu requires at least 3 gpu resources, but max is 2`). Contrast +with `perUnit: false`, which needs just **one** GPU regardless of CPU count. + +**3. Cap accelerators per job** — group ceiling (`max` + `perUnit: false`), on the `cpu` ref: + +```json +{ "id": "cpu", "constraints": [{ "type": "gpu", "max": 2, "perUnit": false }] } +``` + +At most **2 GPUs** total per job regardless of CPUs; requesting 3 → rejected (`Too much gpu +resources for 1 cpu. Max allowed: 2, requested: 3`). + +**4. Total RAM scales with GPU count** — `aggregate` across per-device GPUs: + +```json +{ "id": "GPU1", "constraints": [{ "id": "ram", "min": 8, "aggregate": true }] }, +{ "id": "GPU2", "constraints": [{ "id": "ram", "min": 8, "aggregate": true }] } +``` + +2 GPUs → RAM auto-bumped to **16**; 1 GPU → RAM **8**. + +**5. Several companions at once** — multiple constraints on one GPU: + +```json +{ "id": "gpu0", "constraints": [ + { "id": "ram", "min": 8 }, + { "id": "cpu", "min": 2 }, + { "id": "disk", "min": 20 } +] } +``` + +Renting `gpu0` auto-bumps RAM → 8, CPU → 2, disk → 20 in a single pass. + +**6. Absolute RAM floor for a GPU job** — `perUnit: false` on a single `id`: + +```json +{ "id": "gpu0", "constraints": [{ "id": "ram", "min": 32, "perUnit": false }] } +``` + +Any job renting `gpu0` gets RAM ≥ **32** total, regardless of GPU count (a ratio would instead +scale RAM with the number of GPUs). + +--- + +## Availability & tracking + +Before a job is admitted, the engine checks availability per requested resource: + +- **Gate 1 (per-environment ceiling)** — *fungible only*: `env.total - env.inUse >= requested`. + Controlled by the ref's `total`. Prevents one environment from starving others. +- **Gate 2 (engine-wide pool ceiling)** — *fungible + exclusive discrete*: usage summed + across all environments must not exceed the pool's physical `total`. + +Tracking rules: +- **Fungible** resources (cpu/ram/disk) are tracked **per-environment**. +- **Discrete** resources (GPU/FPGA) are tracked **globally** — a GPU used in one environment + is unavailable in all others. +- **Shareable discrete** (`shareable:true`, e.g. NIC) are tracked for visibility but never + block allocation. + +Running on-demand **services** occupy the same pool and are counted in the same accounting. + +--- + +## Pricing + +`fees` is an object keyed by **chain ID** (string). Each value is an array of fee options: + +```json +"fees": { + "1": [ + { + "feeToken": "0x...", + "prices": [ + { "id": "cpu", "price": 1 }, + { "id": "ram", "price": 0.5 }, + { "id": "disk", "price": 0.2 }, + { "id": "gpu0", "price": 3 } + ] + } + ] +} +``` + +- **feeToken**: token contract address used for payment on that chain. +- **prices**: list of `{ "id": "", "price": }`. The `id` must match a + connection-level resource `id`. Only resources listed here are billable; omit a resource to + offer it at no charge. + +### Pricing units & cost formula + +| Resource | Unit of `amount` | Price meaning | Cost formula | +| -------- | ---------------- | ------------------ | ------------------------------------ | +| **CPU** | Number of CPUs | Per CPU per minute | `price × cpus × ceil(duration/60)` | +| **RAM** | Gigabytes (GB) | Per GB per minute | `price × ramGB × ceil(duration/60)` | +| **Disk** | Gigabytes (GB) | Per GB per minute | `price × diskGB × ceil(duration/60)` | +| **GPU** | Number of GPUs | Per GPU per minute | `price × gpus × ceil(duration/60)` | + +Duration is always in seconds, converted to minutes with **ceil(duration / 60)** (e.g. 61 +seconds → 2 minutes). + +**Example** — CPU=1, RAM=0.5, Disk=0.2, GPU=3; job of 2 CPUs, 4 GB RAM, 10 GB disk, 1 GPU for +125 s (ceil = 3 min): + +- CPU: 1 × 2 × 3 = **6** +- RAM: 0.5 × 4 × 3 = **6** +- Disk: 0.2 × 10 × 3 = **6** +- GPU: 3 × 1 × 3 = **9** → **Total = 27** (smallest unit of the fee token). + +### Multiple chains and tokens + +You can support several chains and multiple fee tokens per chain; consumers choose chain and +token when starting a job, and the node uses the matching `prices`. + +```json +"fees": { + "1": [ { "feeToken": "0xTokenA", "prices": [ { "id": "cpu", "price": 1 } ] }, + { "feeToken": "0xTokenB", "prices": [ { "id": "cpu", "price": 2 } ] } ], + "137": [ { "feeToken": "0xPolygonToken", "prices": [ { "id": "cpu", "price": 1 } ] } ] +} +``` + +--- + +## Free tier + +An environment may expose a `free` block permitting jobs with no payment but tighter limits. +List the same resource ids under `free.resources`; a resource not listed there is unavailable +to free jobs even if the paid environment offers it. Free jobs cannot build images. + +```json +"free": { + "maxJobDuration": 60, + "minJobDuration": 10, + "maxJobs": 3, + "resources": [ + { "id": "cpu", "max": 1 }, + { "id": "ram", "max": 2 }, + { "id": "disk", "max": 5 }, + { "id": "gpu0" } + ] +} +``` + +--- + +## Verify + +```bash +curl http://localhost:8000/api/services/computeEnvironments +``` + +The response includes each environment's `resources` fully resolved (including `init`, +`driverVersion`, etc.) and `inUse` counters. Start a free GPU job: + +```json +{ + "command": "freeStartCompute", + "algorithm": { + "meta": { + "container": { + "image": "tensorflow/tensorflow", + "tag": "2.17.0-gpu", + "entrypoint": "python $ALGO" + }, + "rawcode": "import tensorflow as tf\nprint('Num GPUs Available:', len(tf.config.list_physical_devices('GPU')))" + } + }, + "consumerAddress": "0x00", + "signature": "123", + "nonce": 1, + "environment": "", + "resources": [ + { "id": "cpu", "amount": 1 }, + { "id": "gpu0", "amount": 1 } + ] +} +``` + +--- + +## Checklist + +- [ ] `DOCKER_COMPUTE_ENVIRONMENTS` (or `dockerComputeEnvironments` in config) is a JSON array. +- [ ] GPUs and other custom hardware are defined in the **connection-level** `resources` + array with `kind: "discrete"` and a unique `id` each (NVIDIA: exactly one `DeviceID`; + AMD/Intel: the device mapped via `init.advanced.Devices`). +- [ ] Each environment's `resources` array contains lightweight refs + (`{ id, total?, min?, max?, constraints? }`). +- [ ] `cpu`, `ram`, `disk` are auto-detected — declare them only to cap/override the value. +- [ ] **Disk** and **RAM** amounts are in **GB**. +- [ ] Group constraints (`type`) live on the **environment ref**, not the connection-level pool. +- [ ] `fees.prices[].id` matches a connection-level resource `id`. +- [ ] **CPU / GPU** price = per resource per minute; **RAM / Disk** price = per GB per minute. +- [ ] For the free tier, list resource ids in `free.resources`; omit from `prices` to make + them free only. + +--- + +## Full example + +A single, valid `DOCKER_COMPUTE_ENVIRONMENTS` (or `dockerComputeEnvironments`) config that +exercises **everything** in this guide at once: + +- a **connection-level pool** that caps CPU/RAM/disk and declares **two NVIDIA GPUs**, each + with companion (`id`, ratio) constraints; +- a paid **`gpu-premium`** environment that guarantees a GPU for any CPU job (group + floor + constraint), prices resources on **two chains**, and offers a **free tier**; +- a CPU-only **`cpu-standard`** environment — cheaper, no GPU, with its own free tier. + +> This exact JSON is validated against the config schema in CI. Replace the GPU `DeviceIDs`, +> `feeToken` addresses, and totals with your own. + +```json +[ + { + "socketPath": "/var/run/docker.sock", + "resources": [ + { "id": "cpu", "type": "cpu", "total": 16 }, + { "id": "ram", "type": "ram", "total": 64 }, + { "id": "disk", "type": "disk", "total": 2000 }, + { + "id": "GPU1", + "kind": "discrete", + "type": "gpu", + "total": 1, + "description": "NVIDIA A100 40GB (slot 0)", + "platform": "nvidia", + "driverVersion": "570.195.03", + "memoryTotal": "40960 MiB", + "init": { + "deviceRequests": { + "Driver": "nvidia", + "DeviceIDs": ["GPU-uuid-1"], + "Capabilities": [["gpu"]] + } + }, + "constraints": [{ "id": "ram", "min": 8 }, { "id": "cpu", "min": 2 }] + }, + { + "id": "GPU2", + "kind": "discrete", + "type": "gpu", + "total": 1, + "description": "NVIDIA A100 40GB (slot 1)", + "platform": "nvidia", + "driverVersion": "570.195.03", + "memoryTotal": "40960 MiB", + "init": { + "deviceRequests": { + "Driver": "nvidia", + "DeviceIDs": ["GPU-uuid-2"], + "Capabilities": [["gpu"]] + } + }, + "constraints": [{ "id": "ram", "min": 8 }, { "id": "cpu", "min": 2 }] + } + ], + "environments": [ + { + "id": "gpu-premium", + "description": "Paid GPU environment — any CPU job is guaranteed a GPU", + "storageExpiry": 604800, + "maxJobDuration": 3600, + "minJobDuration": 60, + "maxJobs": 2, + "enableNetwork": true, + "resources": [ + { "id": "cpu", "min": 1, "max": 8, + "constraints": [{ "type": "gpu", "min": 1, "perUnit": false }] }, + { "id": "ram", "min": 1, "max": 32 }, + { "id": "disk", "min": 1, "max": 500 }, + { "id": "GPU1" }, + { "id": "GPU2" } + ], + "access": { "addresses": [], "accessLists": null }, + "fees": { + "1": [ + { + "feeToken": "0x967da4048cD07aB37855c090aAF366e4ce1b9F48", + "prices": [ + { "id": "cpu", "price": 1 }, + { "id": "ram", "price": 0.5 }, + { "id": "disk", "price": 0.1 }, + { "id": "GPU1", "price": 10 }, + { "id": "GPU2", "price": 10 } + ] + } + ], + "137": [ + { + "feeToken": "0x2791Bca1f2de4661ED88A30C99A7a9449Aa84174", + "prices": [ + { "id": "cpu", "price": 0.5 }, + { "id": "ram", "price": 0.2 }, + { "id": "GPU1", "price": 5 }, + { "id": "GPU2", "price": 5 } + ] + } + ] + }, + "free": { + "maxJobDuration": 300, + "minJobDuration": 10, + "maxJobs": 1, + "access": { "addresses": [], "accessLists": null }, + "resources": [ + { "id": "cpu", "max": 2, + "constraints": [{ "type": "gpu", "min": 1, "perUnit": false }] }, + { "id": "ram", "max": 8 }, + { "id": "disk", "max": 10 }, + { "id": "GPU1", "constraints": [{ "id": "ram", "min": 4 }, { "id": "cpu", "min": 1 }] } + ] + } + }, + { + "id": "cpu-standard", + "description": "CPU-only environment — cheaper, no GPU", + "storageExpiry": 604800, + "maxJobDuration": 1800, + "minJobDuration": 60, + "maxJobs": 10, + "enableNetwork": false, + "resources": [ + { "id": "cpu", "total": 8, "min": 1, "max": 4 }, + { "id": "ram", "total": 16, "min": 1, "max": 8 }, + { "id": "disk", "max": 100 } + ], + "access": { "addresses": [], "accessLists": null }, + "fees": { + "1": [ + { + "feeToken": "0x967da4048cD07aB37855c090aAF366e4ce1b9F48", + "prices": [ + { "id": "cpu", "price": 0.3 }, + { "id": "ram", "price": 0.1 }, + { "id": "disk", "price": 0.05 } + ] + } + ] + }, + "free": { + "maxJobDuration": 120, + "minJobDuration": 10, + "maxJobs": 3, + "access": { "addresses": [], "accessLists": null }, + "resources": [ + { "id": "cpu", "max": 1 }, + { "id": "ram", "max": 2 }, + { "id": "disk", "max": 5 } + ] + } + } + ] + } +] +``` + +### What this config does + +**Connection-level pool** — `cpu`/`ram`/`disk` are auto-detected; the declared `total`s only +*cap* them (16 cores, 64 GB RAM, 2 TB disk max for compute). `GPU1` and `GPU2` are two physical +NVIDIA cards, each its own `discrete` resource with a single `DeviceID` and a **companion +constraint**: renting a GPU requires ≥ 8 GB RAM and ≥ 2 CPUs (ratio — per GPU). + +**`gpu-premium` (paid + free)** +- **Guaranteed GPU**: the `cpu` ref carries `{ "type": "gpu", "min": 1, "perUnit": false }` — a + group + **floor** constraint. Any job that requests a CPU must end up with **at least one + GPU total** (`GPU1` *or* `GPU2`), regardless of how many CPUs. If the consumer didn't ask for + a GPU, the engine auto-assigns the freer one. +- **Constraint chaining & ordering**: because `cpu` is listed **before** the GPU refs, a single + resolution pass first satisfies the CPU→GPU floor, then applies the auto-assigned GPU's own + companion constraints (RAM/CPU). List the floor's parent (`cpu`) before the GPUs so this + chains in one pass. +- **Pricing on two chains**: Ethereum (`"1"`) and Polygon (`"137"`), each with its own + `feeToken` and per-resource prices. Consumers pick the chain/token at job start. +- **Free tier**: no payment, tighter caps (≤ 2 CPU, 8 GB RAM, 10 GB disk, 300 s), a single + concurrent free job, and access to `GPU1` only. Free-tier resource refs are **configured + independently** from the paid ones — note the floor constraint and `GPU1`'s companion + constraint are repeated here, because constraints for free jobs are read from + `free.resources`, not inherited from the paid refs. + +**`cpu-standard` (paid + free)** — CPU-only, cheaper prices, higher job concurrency +(`maxJobs: 10`), a modest free tier. It deliberately carries **no** GPU and **no** group +constraint — that's why the `type:"gpu"` floor lives on the environment ref and not the pool: a +pool-level floor would be inherited here and reject every CPU job (empty GPU group). + +### Worked scenarios (on `gpu-premium`, paid, chain `"1"`) + +The engine resolves the requested resources through the constraints before pricing. Resolved +allocations (verified against the constraint engine): + +| Consumer requests | Resolved allocation | Why | +|---|---|---| +| `cpu: 2` | `cpu=2, ram=8, disk=1, GPU1=1` | Floor auto-assigns `GPU1`; its companion bumps RAM to 8 (CPU already ≥ 2). | +| `cpu: 1, GPU2: 1` | `cpu=2, ram=8, disk=1, GPU2=1` | `GPU2` already satisfies the floor; its companion bumps RAM to 8 **and CPU to 2**. | +| `cpu: 1` while `GPU1` is busy | `cpu=2, ram=8, disk=1, GPU2=1` | Floor auto-assigns the freer GPU (`GPU2`); companion bumps RAM/CPU. | + +**Cost** for the first row run for 600 s (`ceil(600/60) = 10` min) at chain `"1"` prices +(cpu 1, ram 0.5, disk 0.1, GPU1 10): + +``` +cpu: 1 × 2 × 10 = 20 +ram: 0.5 × 8 × 10 = 40 +disk: 0.1 × 1 × 10 = 1 +GPU1: 10 × 1 × 10 = 100 + ---- + total = 161 (smallest unit of the fee token) +``` + +For all env vars and config options, see [env.md](env.md). diff --git a/docs/env.md b/docs/env.md index af04fa99f..ab1995d92 100644 --- a/docs/env.md +++ b/docs/env.md @@ -131,7 +131,7 @@ Environmental variables are also tracked in `ENVIRONMENT_VARIABLES` within `src/ - `SERVICE_TEMPLATES_PATH`: Path to a folder of operator-published Service-on-Demand template files (`*.json`, validated against the template schema). The folder is re-read on every `serviceTemplates` request, so templates can be added, edited, or removed without restarting the node. Maps to the `serviceTemplatesPath` config field. Defaults to `databases/serviceTemplates/`. See the [Services guide](services.md). Example: `docs/serviceTemplates/` -The `DOCKER_COMPUTE_ENVIRONMENTS` environment variable is used to configure Docker-based compute environments in Ocean Node. For GPU setup and examples see [GPU Guide](GPU.md). For pricing configuration see [Compute pricing](compute-pricing.md). +The `DOCKER_COMPUTE_ENVIRONMENTS` environment variable is used to configure Docker-based compute environments in Ocean Node. For the full guide — resources, GPU setup, constraints and pricing — see [Compute Configuration](compute.md). `cpu`, `ram`, and `disk` resources are **auto-detected** from the host at startup. All resource values are expressed in natural units: CPU in cores, RAM and disk in GB. @@ -211,7 +211,7 @@ The config has a two-level structure: - **cpuList** *(cpu resource only)*: Restricts which host core IDs compute containers may be pinned to. Comma-separated core IDs and/or integer ranges, e.g. `"3"`, `"0-1,3"` or `"32-63"`. Ranges `a-b` must have `b` strictly greater than `a`, all parts must be ascending and non-overlapping, core IDs may not exceed `8192`, and every core ID must exist on the host — otherwise the node fails to start with an error. Mutually exclusive with **total**: the cpu resource must specify exactly one of the two, and with `cpuList` the effective total is the number of listed cores. - **min** / **max**: Per-job minimum/maximum. - **description**, **platform**, **driverVersion**, **memoryTotal**: Informational metadata. - - **init**: Docker device configuration (`deviceRequests` for NVIDIA, `advanced` for AMD/Intel). See [GPU Guide](GPU.md). + - **init**: Docker device configuration (`deviceRequests` for NVIDIA, `advanced` for AMD/Intel). See [Compute Configuration](compute.md#configuring-gpus). - **constraints**: Cross-resource requirements. `{ "id": "ram", "min": 4 }` means renting this resource also requires 4 GB RAM. #### Environment-level fields diff --git a/docs/services.md b/docs/services.md index 6c8619b9c..06951f9bc 100644 --- a/docs/services.md +++ b/docs/services.md @@ -14,8 +14,8 @@ consumer can connect to while it runs. The consumer supplies the container spec directly in the request: an `image` (referenced by `tag` or `checksum`, or an inline `dockerfile` when the operator allows building), optional `dockerCmd` / `dockerEntrypoint`, the container ports to expose, the -requested resources (cpu/ram/disk/gpu), the duration, and encrypted `userData` that is -injected as container environment variables. +requested resources ([cpu/ram/disk/gpu](compute.md)), the duration, and encrypted `userData` +that is injected as container environment variables. ## Lifecycle @@ -102,6 +102,12 @@ and access can be restricted with the environment's `access` allow-list `serviceTemplatesPath` (default `databases/serviceTemplates/`); template secret values are never returned by the API (only the env-var keys are exposed). +The compute environments a service can run on — and the resources (cpu/ram/disk/gpu) it may +request — are the same ones configured at the node's Docker-connection level for compute jobs, +and services draw from and are counted against the same shared resource pool. For how to +declare resources, configure GPUs, set per-environment constraints, and price them, see the +[Compute Configuration guide](compute.md). + ## Security model & important notices - **Container hardening.** Service containers are created with diff --git a/src/@types/C2D/C2D.ts b/src/@types/C2D/C2D.ts index 7551a38e4..74bd3e267 100644 --- a/src/@types/C2D/C2D.ts +++ b/src/@types/C2D/C2D.ts @@ -33,9 +33,15 @@ export type ComputeResourceType = 'cpu' | 'ram' | 'disk' | any export type ComputeResourceKind = 'discrete' | 'fungible' export interface ResourceConstraint { - id: ComputeResourceType // the resource being constrained - min?: number // min units of this resource per unit of parent resource - max?: number // max units of this resource per unit of parent resource + // Exactly one of `id` | `type` must be set. + id?: ComputeResourceType // exact single-resource target (e.g. 'ram', 'gpu0') + type?: string // group target: aggregate across ALL env resources whose `type` matches (e.g. 'gpu') + min?: number // min units of the constrained resource; per unit of parent when `perUnit` (default), absolute aggregate when `perUnit:false` + max?: number // max units of the constrained resource; per unit of parent when `perUnit` (default), absolute aggregate when `perUnit:false` + perUnit?: boolean // undefined/true = RATIO (parentAmount * min); false = FLOOR (absolute min/max, only enforced when parent > 0) + aggregate?: boolean // when true, this constraint's contribution SUMS with matching aggregate + // constraints on other requested parents into one shared target (single `id` only). e.g. two + // per-device GPUs each with {id:'cpu',min:1,max:4,aggregate:true} → cpu in [2,8] when both rented. } export interface ComputeResourcesPricingInfo { diff --git a/src/components/c2d/compute_engine_base.ts b/src/components/c2d/compute_engine_base.ts index a3a8c4d7a..e03a96ed5 100644 --- a/src/components/c2d/compute_engine_base.ts +++ b/src/components/c2d/compute_engine_base.ts @@ -327,6 +327,24 @@ export abstract class C2DEngine { resources: ComputeResourceRequest[], env: ComputeEnvironment, isFree: boolean + ): void { + // Two-phase on purpose: a later parent's min bump can raise a target past an earlier + // parent's already-checked max (e.g. gpu1 {cpu min:8} bumping past gpu0 {cpu max:6}), + // so all floors settle first (direct, then aggregate) and every ceiling is validated + // afterwards against the final amounts — the outcome no longer depends on the order + // resources appear in the env definition. + this.enforceDirectConstraints(resources, env, isFree, 'min') + this.applyAggregateConstraints(resources, env, isFree) + this.enforceDirectConstraints(resources, env, isFree, 'max') + } + + // Non-aggregate constraints, one phase at a time: 'min' raises targets to their floors + // (throwing when a floor exceeds the target's own max), 'max' rejects ceilings violations. + protected enforceDirectConstraints( + resources: ComputeResourceRequest[], + env: ComputeEnvironment, + isFree: boolean, + phase: 'min' | 'max' ): void { const envResources = isFree ? (env.free?.resources ?? []) : (env.resources ?? []) for (const envResource of envResources) { @@ -335,29 +353,53 @@ export abstract class C2DEngine { if (!parentAmount || parentAmount <= 0) continue for (const constraint of envResource.constraints) { - let constrainedAmount = this.getResourceRequest(resources, constraint.id) ?? 0 - - if (constraint.min !== undefined) { - const requiredMin = parentAmount * constraint.min + // Aggregate constraints sum across parents — handled in applyAggregateConstraints. + if (constraint.aggregate) continue + // perUnit (default true) => RATIO (parentAmount * value); false => absolute FLOOR/ceiling. + const perUnit = constraint.perUnit !== false + // A constraint targets either a single resource (`id`) or a group (`type`, aggregated). + const isGroup = constraint.type !== undefined + const targetIds = isGroup + ? this.getGroupResourceIds(env, constraint.type!, isFree) + : [constraint.id as string] + const targetLabel = isGroup + ? `${constraint.type} resources` + : String(constraint.id) + const constrainedAmount = isGroup + ? this.getGroupRequestedAmount(resources, targetIds) + : (this.getResourceRequest(resources, constraint.id) ?? 0) + + if (phase === 'min' && constraint.min !== undefined) { + const requiredMin = perUnit ? parentAmount * constraint.min : constraint.min if (constrainedAmount < requiredMin) { - const constrainedMaxMin = this.getMaxMinResource(constraint.id, env, isFree) - if (requiredMin > constrainedMaxMin.max) { + // Aggregate per-job ceiling of the target (single max, or sum of group maxes). + const targetMax = isGroup + ? this.getGroupMax(env, targetIds, isFree) + : this.getMaxMinResource(constraint.id, env, isFree).max + if (requiredMin > targetMax) { throw new Error( - `Cannot satisfy constraint: ${parentAmount} ${envResource.id} requires at least ${requiredMin} ${constraint.id}, but max is ${constrainedMaxMin.max}` + `Cannot satisfy constraint: ${parentAmount} ${envResource.id} requires at least ${requiredMin} ${targetLabel}, but max is ${targetMax}` ) } - this.setResourceAmount(resources, constraint.id, requiredMin) - constrainedAmount = requiredMin + if (isGroup) { + this.bumpGroupToFloor( + resources, + env, + targetIds, + requiredMin - constrainedAmount, + isFree + ) + } else { + this.setResourceAmount(resources, constraint.id, requiredMin) + } } } - if (constraint.max !== undefined) { - const requiredMax = parentAmount * constraint.max - // re-read in case it was bumped above - constrainedAmount = this.getResourceRequest(resources, constraint.id) ?? 0 + if (phase === 'max' && constraint.max !== undefined) { + const requiredMax = perUnit ? parentAmount * constraint.max : constraint.max if (constrainedAmount > requiredMax) { throw new Error( - `Too much ${constraint.id} for ${parentAmount} ${envResource.id}. Max allowed: ${requiredMax}, requested: ${constrainedAmount}` + `Too much ${targetLabel} for ${parentAmount} ${envResource.id}. Max allowed: ${requiredMax}, requested: ${constrainedAmount}` ) } } @@ -365,6 +407,69 @@ export abstract class C2DEngine { } } + // Aggregate constraints (`aggregate: true`) accumulate their per-parent contribution + // ADDITIVELY into a shared single-`id` target, summed across every requested parent that + // carries a matching aggregate constraint. This is how per-device GPUs (GPU1, GPU2, each + // `max:1`) can jointly require e.g. cpu min = Σ(gpuAmount × min) and cpu max = Σ(gpuAmount × + // max) — 2 GPUs with a {min:1,max:4} per-GPU rule → cpu in [2, 8]. Non-aggregate constraints + // keep their independent per-parent behavior (handled in enforceDirectConstraints); this runs + // between its min and max phases and only ever raises the target's floor. + protected applyAggregateConstraints( + resources: ComputeResourceRequest[], + env: ComputeEnvironment, + isFree: boolean + ): void { + const envResources = isFree ? (env.free?.resources ?? []) : (env.resources ?? []) + const summedMin = new Map() + const summedMax = new Map() + const hasMax = new Set() + + for (const parent of envResources) { + if (!parent.constraints || parent.constraints.length === 0) continue + const parentAmount = this.getResourceRequest(resources, parent.id) ?? 0 + if (parentAmount <= 0) continue + for (const c of parent.constraints) { + // aggregate targets a single resource id (validated by schema) + if (!c.aggregate || c.id === undefined) continue + const perUnit = c.perUnit !== false + if (c.min !== undefined) { + const contrib = perUnit ? parentAmount * c.min : c.min + summedMin.set(c.id, (summedMin.get(c.id) ?? 0) + contrib) + } + if (c.max !== undefined) { + const contrib = perUnit ? parentAmount * c.max : c.max + summedMax.set(c.id, (summedMax.get(c.id) ?? 0) + contrib) + hasMax.add(c.id) + } + } + } + + // min: raise each target up to its summed floor (respecting the target's own max) + for (const [targetId, requiredMin] of summedMin) { + const current = this.getResourceRequest(resources, targetId) ?? 0 + if (current < requiredMin) { + const targetMax = this.getMaxMinResource(targetId, env, isFree).max + if (requiredMin > targetMax) { + throw new Error( + `Cannot satisfy aggregate constraint: requires at least ${requiredMin} ${targetId}, but max is ${targetMax}` + ) + } + this.setResourceAmount(resources, targetId, requiredMin) + } + } + + // max: reject if a target exceeds its summed ceiling (never auto-reduced) + for (const targetId of hasMax) { + const current = this.getResourceRequest(resources, targetId) ?? 0 + const requiredMax = summedMax.get(targetId) + if (current > requiredMax) { + throw new Error( + `Too much ${targetId} for the requested resources. Max allowed: ${requiredMax}, requested: ${current}` + ) + } + } + } + protected setResourceAmount( resources: ComputeResourceRequest[], id: ComputeResourceType, @@ -378,6 +483,70 @@ export abstract class C2DEngine { } } + // Returns the ids of every resource active in this env (paid or free list) whose `type` + // matches — the concrete members a `type` group constraint aggregates over. + protected getGroupResourceIds( + env: ComputeEnvironment, + type: string, + isFree: boolean + ): string[] { + const envResources = isFree ? (env.free?.resources ?? []) : (env.resources ?? []) + return envResources.filter((r) => r.type === type).map((r) => r.id) + } + + // Sum of requested amounts across a set of resource ids (missing => 0). + protected getGroupRequestedAmount( + resources: ComputeResourceRequest[], + ids: string[] + ): number { + let total = 0 + for (const id of ids) total += this.getResourceRequest(resources, id) ?? 0 + return total + } + + // Aggregate per-job ceiling of a group: sum of each member's max. + protected getGroupMax(env: ComputeEnvironment, ids: string[], isFree: boolean): number { + let total = 0 + for (const id of ids) total += this.getMaxMinResource(id, env, isFree).max + return total + } + + // Raise members of a `type` group until their combined requested amount grows by `deficit` + // units. Prefers members with the most availability (lowest inUse), then config order, and + // never exceeds any member's own max. Relies on checkAndFillMissingResources' pre-fill loop + // having already inserted an entry for every declared resource, so setResourceAmount always + // finds its target. inUse here is a point-in-time hint to reduce false deferrals; the + // authoritative availability gate is checkIfResourcesAreAvailable. + // NOTE: inUse is read from env.resources (not env.free.resources) on purpose — group + // constraints target discrete resources (GPUs), whose inUse there is the GLOBAL count + // (paid + free, across all envs), i.e. the binding availability signal. The free list holds + // free-only usage and would under-count paid usage of the same physical device. + protected bumpGroupToFloor( + resources: ComputeResourceRequest[], + env: ComputeEnvironment, + ids: string[], + deficit: number, + isFree: boolean + ): void { + let remaining = deficit + const candidates = ids + .map((id) => { + const current = this.getResourceRequest(resources, id) ?? 0 + const { max } = this.getMaxMinResource(id, env, isFree) + const inUse = this.getResource(env.resources, id)?.inUse ?? 0 + return { id, current, headroom: max - current, inUse } + }) + .filter((c) => c.headroom > 0) + .sort((a, b) => a.inUse - b.inUse) + + for (const c of candidates) { + if (remaining <= 0) break + const bump = Math.min(c.headroom, remaining) + this.setResourceAmount(resources, c.id, c.current + bump) + remaining -= bump + } + } + public async getUsedResources(env: ComputeEnvironment): Promise { const usedResources: { [x: string]: any } = {} const usedFreeResources: { [x: string]: any } = {} diff --git a/src/test/unit/compute.test.ts b/src/test/unit/compute.test.ts index 5588c8eab..a36a02498 100644 --- a/src/test/unit/compute.test.ts +++ b/src/test/unit/compute.test.ts @@ -568,6 +568,531 @@ describe('Compute Jobs Database', () => { expect(err.message).to.include('ram') } }) + + // ── Group (type) + FLOOR (perUnit:false) constraints ────────────────── + // "if any cpu is selected, the job needs at least 1 GPU — no matter which id". + // The constraint lives on the env's cpu ref and targets the `type:'gpu'` group. + const cpuFloorConstraint: any[] = [{ type: 'gpu', min: 1, perUnit: false }] + const twoGpuResources = (inUse1 = 0, inUse2 = 0): any[] => [ + { + id: 'cpu', + type: 'cpu', + kind: 'fungible', + total: 8, + min: 1, + max: 8, + inUse: 0, + constraints: cpuFloorConstraint + }, + { id: 'ram', type: 'ram', kind: 'fungible', total: 32, min: 1, max: 32, inUse: 0 }, + { + id: 'disk', + type: 'disk', + kind: 'fungible', + total: 500, + min: 0, + max: 500, + inUse: 0 + }, + { + id: 'GPU1', + type: 'gpu', + kind: 'discrete', + total: 1, + min: 0, + max: 1, + inUse: inUse1 + }, + { + id: 'GPU2', + type: 'gpu', + kind: 'discrete', + total: 1, + min: 0, + max: 1, + inUse: inUse2 + } + ] + + it('group floor (type:gpu): satisfied by any one GPU → no change', async function () { + const env = makeEnv(twoGpuResources()) + const req: ComputeResourceRequest[] = [ + { id: 'cpu', amount: 2 }, + { id: 'GPU2', amount: 1 } + ] + const result = await engine.checkAndFillMissingResources(req, env, false) + // aggregate gpu (0 + 1) already meets floor 1 → nothing bumped + expect(result.find((r) => r.id === 'GPU1').amount).to.equal(0) + expect(result.find((r) => r.id === 'GPU2').amount).to.equal(1) + }) + + it('group floor: cpu selected, no GPU requested → exactly one GPU auto-bumped to 1', async function () { + const env = makeEnv(twoGpuResources()) + // 8 cpu but floor is absolute (perUnit:false) → still only 1 GPU total needed + const req: ComputeResourceRequest[] = [{ id: 'cpu', amount: 8 }] + const result = await engine.checkAndFillMissingResources(req, env, false) + const g1 = result.find((r) => r.id === 'GPU1').amount + const g2 = result.find((r) => r.id === 'GPU2').amount + expect(g1 + g2).to.equal(1) + }) + + it('group floor: prefers the GPU with lower inUse when auto-bumping', async function () { + // GPU1 already globally in use, GPU2 free → GPU2 should be the one bumped + const env = makeEnv(twoGpuResources(1, 0)) + const req: ComputeResourceRequest[] = [{ id: 'cpu', amount: 1 }] + const result = await engine.checkAndFillMissingResources(req, env, false) + expect(result.find((r) => r.id === 'GPU1').amount).to.equal(0) + expect(result.find((r) => r.id === 'GPU2').amount).to.equal(1) + }) + + it('group floor: required floor exceeds aggregate group max → throws Cannot satisfy', async function () { + const resources = twoGpuResources() + resources[0].constraints = [{ type: 'gpu', min: 3, perUnit: false }] + const env = makeEnv(resources) + const req: ComputeResourceRequest[] = [{ id: 'cpu', amount: 1 }] + try { + await engine.checkAndFillMissingResources(req, env, false) + assert.fail('Expected error was not thrown') + } catch (err: any) { + expect(err.message).to.include('Cannot satisfy') + expect(err.message).to.include('gpu resources') + } + }) + + it('group floor: constraint skipped when parent (cpu) not requested', async function () { + const resources = twoGpuResources() + resources[0].min = 0 // allow cpu to stay 0 so the parent is truly absent + const env = makeEnv(resources) + const req: ComputeResourceRequest[] = [{ id: 'ram', amount: 2 }] + const result = await engine.checkAndFillMissingResources(req, env, false) + expect(result.find((r) => r.id === 'GPU1').amount).to.equal(0) + expect(result.find((r) => r.id === 'GPU2').amount).to.equal(0) + }) + + it('group ceiling (type:gpu, max, perUnit:false): total group above max → throws Too much', async function () { + const resources = twoGpuResources() + resources[0].constraints = [{ type: 'gpu', max: 1, perUnit: false }] + const env = makeEnv(resources) + const req: ComputeResourceRequest[] = [ + { id: 'cpu', amount: 1 }, + { id: 'GPU1', amount: 1 }, + { id: 'GPU2', amount: 1 } + ] + try { + await engine.checkAndFillMissingResources(req, env, false) + assert.fail('Expected error was not thrown') + } catch (err: any) { + expect(err.message).to.include('Too much gpu resources') + expect(err.message).to.include('Max allowed: 1') + } + }) + + it('back-compat: exact-id constraint with no perUnit still uses ratio semantics', async function () { + const resources = [ + { ...baseResources[0], constraints: [{ id: 'ram', min: 2, max: 8 }] }, + ...baseResources.slice(1) + ] + const env = makeEnv(resources) + const req: ComputeResourceRequest[] = [ + { id: 'cpu', amount: 4 }, + { id: 'ram', amount: 4 }, + { id: 'disk', amount: 50 } + ] + const result = await engine.checkAndFillMissingResources(req, env, false) + // 4 cpu * 2 (ratio) = 8 + expect(result.find((r) => r.id === 'ram').amount).to.equal(8) + }) + + it('exact-id constraint with perUnit:false → absolute floor, not multiplied by parent', async function () { + const resources = [ + { ...baseResources[0], constraints: [{ id: 'ram', min: 8, perUnit: false }] }, + ...baseResources.slice(1) + ] + const env = makeEnv(resources) + const req: ComputeResourceRequest[] = [ + { id: 'cpu', amount: 4 }, + { id: 'ram', amount: 4 }, + { id: 'disk', amount: 50 } + ] + const result = await engine.checkAndFillMissingResources(req, env, false) + // absolute floor 8, NOT 4*8=32 + expect(result.find((r) => r.id === 'ram').amount).to.equal(8) + }) + + // ── Aggregate constraints (sum across per-device siblings) ──────────── + // Two per-device GPUs, each { id:'cpu', min:1, max:4, aggregate:true }. Renting both + // sums to cpu in [2, 8]; renting one gives [1, 4]. + const aggCpuConstraint = [{ id: 'cpu', min: 1, max: 4, aggregate: true }] + const twoAggGpuResources = (): any[] => [ + { id: 'cpu', type: 'cpu', kind: 'fungible', total: 16, min: 1, max: 16, inUse: 0 }, + { id: 'ram', type: 'ram', kind: 'fungible', total: 64, min: 1, max: 64, inUse: 0 }, + { + id: 'GPU1', + type: 'gpu', + kind: 'discrete', + total: 1, + min: 0, + max: 1, + inUse: 0, + constraints: aggCpuConstraint + }, + { + id: 'GPU2', + type: 'gpu', + kind: 'discrete', + total: 1, + min: 0, + max: 1, + inUse: 0, + constraints: aggCpuConstraint + } + ] + + it('aggregate min sums across two GPUs → cpu bumped to 2 (1+1)', async function () { + const env = makeEnv(twoAggGpuResources()) + const req: ComputeResourceRequest[] = [ + { id: 'GPU1', amount: 1 }, + { id: 'GPU2', amount: 1 } + ] + const result = await engine.checkAndFillMissingResources(req, env, false) + expect(result.find((r) => r.id === 'cpu').amount).to.equal(2) + }) + + it('aggregate max sums across two GPUs → cpu 8 allowed', async function () { + const env = makeEnv(twoAggGpuResources()) + const req: ComputeResourceRequest[] = [ + { id: 'GPU1', amount: 1 }, + { id: 'GPU2', amount: 1 }, + { id: 'cpu', amount: 8 } + ] + const result = await engine.checkAndFillMissingResources(req, env, false) + expect(result.find((r) => r.id === 'cpu').amount).to.equal(8) + }) + + it('aggregate max sums across two GPUs → cpu above 8 throws', async function () { + const env = makeEnv(twoAggGpuResources()) + const req: ComputeResourceRequest[] = [ + { id: 'GPU1', amount: 1 }, + { id: 'GPU2', amount: 1 }, + { id: 'cpu', amount: 10 } + ] + try { + await engine.checkAndFillMissingResources(req, env, false) + assert.fail('Expected error was not thrown') + } catch (err: any) { + expect(err.message).to.include('Too much cpu') + expect(err.message).to.include('Max allowed: 8') + } + }) + + it('aggregate with a single GPU → cpu bumped to 1 (only one contributes)', async function () { + const env = makeEnv(twoAggGpuResources()) + const req: ComputeResourceRequest[] = [{ id: 'GPU1', amount: 1 }] + const result = await engine.checkAndFillMissingResources(req, env, false) + expect(result.find((r) => r.id === 'cpu').amount).to.equal(1) + expect(result.find((r) => r.id === 'GPU2').amount).to.equal(0) + }) + + it('without aggregate the same per-GPU constraint does NOT sum (contrast)', async function () { + const resources = twoAggGpuResources() + // drop the aggregate flag → independent per-GPU constraints + resources[2].constraints = [{ id: 'cpu', min: 1, max: 4 }] + resources[3].constraints = [{ id: 'cpu', min: 1, max: 4 }] + const env = makeEnv(resources) + const result = await engine.checkAndFillMissingResources( + [ + { id: 'GPU1', amount: 1 }, + { id: 'GPU2', amount: 1 } + ], + env, + false + ) + // per-GPU: cpu floor is 1 (not 2) + expect(result.find((r) => r.id === 'cpu').amount).to.equal(1) + // per-GPU max is 4 (not 8): cpu:5 already throws + try { + await engine.checkAndFillMissingResources( + [ + { id: 'GPU1', amount: 1 }, + { id: 'GPU2', amount: 1 }, + { id: 'cpu', amount: 5 } + ], + makeEnv(resources), + false + ) + assert.fail('Expected error was not thrown') + } catch (err: any) { + expect(err.message).to.include('Max allowed: 4') + } + }) + + // ── More constraint recipes (mirrors docs/compute.md "More constraint recipes") ── + const mkCpu = (constraints?: any[]) => ({ + id: 'cpu', + type: 'cpu', + kind: 'fungible', + total: 32, + min: 1, + max: 32, + inUse: 0, + ...(constraints ? { constraints } : {}) + }) + const mkRam = { + id: 'ram', + type: 'ram', + kind: 'fungible', + total: 128, + min: 1, + max: 128, + inUse: 0 + } + const mkDisk = { + id: 'disk', + type: 'disk', + kind: 'fungible', + total: 2000, + min: 0, + max: 2000, + inUse: 0 + } + const mkGpu = (id: string, constraints?: any[]) => ({ + id, + type: 'gpu', + kind: 'discrete', + total: 1, + min: 0, + max: 1, + inUse: 0, + ...(constraints ? { constraints } : {}) + }) + + it('recipe 1: companion range {id:ram,min:8,max:16} bounds RAM per GPU', async function () { + const resources = [ + mkCpu(), + mkRam, + mkDisk, + mkGpu('GPU1', [{ id: 'ram', min: 8, max: 16 }]) + ] + const bumped = await engine.checkAndFillMissingResources( + [{ id: 'GPU1', amount: 1 }], + makeEnv(resources), + false + ) + expect(bumped.find((r) => r.id === 'ram').amount).to.equal(8) + try { + await engine.checkAndFillMissingResources( + [ + { id: 'GPU1', amount: 1 }, + { id: 'ram', amount: 20 } + ], + makeEnv(resources), + false + ) + assert.fail('Expected error was not thrown') + } catch (err: any) { + expect(err.message).to.include('Too much ram') + expect(err.message).to.include('Max allowed: 16') + } + }) + + it('recipe 2: group ratio {type:gpu,min:1} → one GPU per CPU', async function () { + const resources = [ + mkCpu([{ type: 'gpu', min: 1 }]), + mkRam, + mkDisk, + mkGpu('GPU1'), + mkGpu('GPU2') + ] + const r = await engine.checkAndFillMissingResources( + [{ id: 'cpu', amount: 2 }], + makeEnv(resources), + false + ) + // 2 CPUs (ratio, perUnit default) → 2 GPUs auto-assigned + const total = + r.find((x) => x.id === 'GPU1').amount + r.find((x) => x.id === 'GPU2').amount + expect(total).to.equal(2) + }) + + it('recipe 2b: group ratio requiring more GPUs than exposed → Cannot satisfy', async function () { + const resources = [ + mkCpu([{ type: 'gpu', min: 1 }]), + mkRam, + mkDisk, + mkGpu('GPU1'), + mkGpu('GPU2') + ] + try { + await engine.checkAndFillMissingResources( + [{ id: 'cpu', amount: 3 }], + makeEnv(resources), + false + ) + assert.fail('Expected error was not thrown') + } catch (err: any) { + expect(err.message).to.include('Cannot satisfy') + expect(err.message).to.include('at least 3 gpu resources') + expect(err.message).to.include('max is 2') + } + }) + + it('recipe 3: group ceiling {type:gpu,max:2,perUnit:false} caps total GPUs', async function () { + const resources = [ + mkCpu([{ type: 'gpu', max: 2, perUnit: false }]), + mkRam, + mkDisk, + mkGpu('GPU1'), + mkGpu('GPU2'), + mkGpu('GPU3') + ] + // 2 GPUs is fine + const ok = await engine.checkAndFillMissingResources( + [ + { id: 'cpu', amount: 1 }, + { id: 'GPU1', amount: 1 }, + { id: 'GPU2', amount: 1 } + ], + makeEnv(resources), + false + ) + expect(ok.find((r) => r.id === 'GPU3').amount).to.equal(0) + // 3 GPUs exceeds the cap + try { + await engine.checkAndFillMissingResources( + [ + { id: 'cpu', amount: 1 }, + { id: 'GPU1', amount: 1 }, + { id: 'GPU2', amount: 1 }, + { id: 'GPU3', amount: 1 } + ], + makeEnv(resources), + false + ) + assert.fail('Expected error was not thrown') + } catch (err: any) { + expect(err.message).to.include('Too much gpu resources') + expect(err.message).to.include('Max allowed: 2') + } + }) + + it('recipe 4: aggregate {id:ram,min:8,aggregate:true} sums RAM across GPUs', async function () { + const agg = [{ id: 'ram', min: 8, aggregate: true }] + const resources = [mkCpu(), mkRam, mkDisk, mkGpu('GPU1', agg), mkGpu('GPU2', agg)] + const two = await engine.checkAndFillMissingResources( + [ + { id: 'GPU1', amount: 1 }, + { id: 'GPU2', amount: 1 } + ], + makeEnv(resources), + false + ) + expect(two.find((r) => r.id === 'ram').amount).to.equal(16) + const one = await engine.checkAndFillMissingResources( + [{ id: 'GPU1', amount: 1 }], + makeEnv(resources), + false + ) + expect(one.find((r) => r.id === 'ram').amount).to.equal(8) + }) + + it('recipe 5: multiple companion constraints on one GPU bump each target', async function () { + const resources = [ + mkCpu(), + mkRam, + mkDisk, + mkGpu('GPU1', [ + { id: 'ram', min: 8 }, + { id: 'cpu', min: 2 }, + { id: 'disk', min: 20 } + ]) + ] + const r = await engine.checkAndFillMissingResources( + [{ id: 'GPU1', amount: 1 }], + makeEnv(resources), + false + ) + expect(r.find((x) => x.id === 'ram').amount).to.equal(8) + expect(r.find((x) => x.id === 'cpu').amount).to.equal(2) + expect(r.find((x) => x.id === 'disk').amount).to.equal(20) + }) + + it('recipe 6: absolute RAM floor {id:ram,min:32,perUnit:false} on a GPU', async function () { + const resources = [ + mkCpu(), + mkRam, + mkDisk, + mkGpu('GPU1', [{ id: 'ram', min: 32, perUnit: false }]) + ] + const r = await engine.checkAndFillMissingResources( + [{ id: 'GPU1', amount: 1 }], + makeEnv(resources), + false + ) + expect(r.find((x) => x.id === 'ram').amount).to.equal(32) + }) + + // ── Contradictory per-GPU constraints must be rejected regardless of order ── + // gpu0 caps cpu at 6 while gpu1 requires at least 8: renting both is unsatisfiable. + // A single-pass check let this through when gpu0 appeared first (its max was validated + // before gpu1's min bumped cpu to 8); the max phase now re-validates ceilings against + // the settled amounts, so both orderings reject. + it('contradictory constraints (gpu0 cpu max:6 vs gpu1 cpu min:8) → rejected in both env orders', async function () { + const gpu0 = () => mkGpu('gpu0', [{ id: 'cpu', max: 6 }]) + const gpu1 = () => mkGpu('gpu1', [{ id: 'cpu', min: 8 }]) + const bothGpus: ComputeResourceRequest[] = [ + { id: 'cpu', amount: 4 }, + { id: 'gpu0', amount: 1 }, + { id: 'gpu1', amount: 1 } + ] + for (const resources of [ + [mkCpu(), mkRam, mkDisk, gpu0(), gpu1()], + [mkCpu(), mkRam, mkDisk, gpu1(), gpu0()] + ]) { + try { + await engine.checkAndFillMissingResources( + bothGpus.map((r) => ({ ...r })), + makeEnv(resources), + false + ) + assert.fail('Expected error was not thrown') + } catch (err: any) { + expect(err.message).to.include('Too much cpu') + expect(err.message).to.include('1 gpu0') + expect(err.message).to.include('Max allowed: 6') + expect(err.message).to.include('requested: 8') + } + } + }) + + it('contradictory constraints: renting a single GPU still works', async function () { + const resources = [ + mkCpu(), + mkRam, + mkDisk, + mkGpu('gpu0', [{ id: 'cpu', max: 6 }]), + mkGpu('gpu1', [{ id: 'cpu', min: 8 }]) + ] + // gpu0 alone: cpu 4 ≤ 6 → untouched + const r0 = await engine.checkAndFillMissingResources( + [ + { id: 'cpu', amount: 4 }, + { id: 'gpu0', amount: 1 } + ], + makeEnv(resources), + false + ) + expect(r0.find((x) => x.id === 'cpu').amount).to.equal(4) + // gpu1 alone: cpu bumped to its floor of 8 + const r1 = await engine.checkAndFillMissingResources( + [ + { id: 'cpu', amount: 4 }, + { id: 'gpu1', amount: 1 } + ], + makeEnv(resources), + false + ) + expect(r1.find((x) => x.id === 'cpu').amount).to.equal(8) + }) }) describe('testing checkIfResourcesAreAvailable', function () { @@ -984,6 +1509,309 @@ describe('Schema validation (C2DDockerConfigSchema)', () => { expect(result.success).to.equal(true) }) + const gpuPool = [ + { + id: 'gpu0', + kind: 'discrete', + type: 'gpu', + total: 1, + init: { + deviceRequests: { + Driver: 'nvidia', + DeviceIDs: ['uuid-a'], + Capabilities: [['gpu']] + } + } + } + ] + + it('resource constraint with both id and type is rejected', function () { + const config = [ + { + ...validBase, + resources: gpuPool, + environments: [ + { + ...validBase.environments[0], + resources: [ + { id: 'cpu', constraints: [{ id: 'ram', type: 'gpu', min: 1 }] }, + { id: 'ram' }, + { id: 'disk' }, + { id: 'gpu0' } + ] + } + ] + } + ] + const result = C2DDockerConfigSchema.safeParse(config) + expect(result.success).to.equal(false) + const msgs = result.error?.issues.map((i) => i.message).join(' ') + expect(msgs).to.include('mutually exclusive') + }) + + it('resource constraint with neither id nor type is rejected', function () { + const config = [ + { + ...validBase, + resources: gpuPool, + environments: [ + { + ...validBase.environments[0], + resources: [ + { id: 'cpu', constraints: [{ min: 1 }] }, + { id: 'ram' }, + { id: 'disk' }, + { id: 'gpu0' } + ] + } + ] + } + ] + const result = C2DDockerConfigSchema.safeParse(config) + expect(result.success).to.equal(false) + const msgs = result.error?.issues.map((i) => i.message).join(' ') + expect(msgs).to.include('either "id" or "type"') + }) + + it('valid group+floor constraint (type:gpu, perUnit:false) parses successfully', function () { + const config = [ + { + ...validBase, + resources: gpuPool, + environments: [ + { + ...validBase.environments[0], + resources: [ + { id: 'cpu', constraints: [{ type: 'gpu', min: 1, perUnit: false }] }, + { id: 'ram' }, + { id: 'disk' }, + { id: 'gpu0' } + ] + } + ] + } + ] + const result = C2DDockerConfigSchema.safeParse(config) + expect(result.success).to.equal(true) + }) + + it('aggregate constraint targeting a type group is rejected', function () { + const config = [ + { + ...validBase, + resources: gpuPool, + environments: [ + { + ...validBase.environments[0], + resources: [ + { id: 'cpu' }, + { id: 'ram' }, + { id: 'disk' }, + { id: 'gpu0', constraints: [{ type: 'gpu', min: 1, aggregate: true }] } + ] + } + ] + } + ] + const result = C2DDockerConfigSchema.safeParse(config) + expect(result.success).to.equal(false) + const msgs = result.error?.issues.map((i) => i.message).join(' ') + expect(msgs).to.include('aggregate constraints must target a single "id"') + }) + + it('valid aggregate constraint (single id) parses successfully', function () { + const config = [ + { + ...validBase, + resources: gpuPool, + environments: [ + { + ...validBase.environments[0], + resources: [ + { id: 'cpu' }, + { id: 'ram' }, + { id: 'disk' }, + { + id: 'gpu0', + constraints: [{ id: 'cpu', min: 1, max: 4, aggregate: true }] + } + ] + } + ] + } + ] + const result = C2DDockerConfigSchema.safeParse(config) + expect(result.success).to.equal(true) + }) + + // Keeps the "Full example" config in docs/compute.md honest — if the schema changes in a way + // that would break the documented example, this test fails. + it('the docs/compute.md full example config parses successfully', function () { + const config: any = [ + { + socketPath: '/var/run/docker.sock', + resources: [ + { id: 'cpu', type: 'cpu', total: 16 }, + { id: 'ram', type: 'ram', total: 64 }, + { id: 'disk', type: 'disk', total: 2000 }, + { + id: 'GPU1', + kind: 'discrete', + type: 'gpu', + total: 1, + description: 'NVIDIA A100 40GB (slot 0)', + platform: 'nvidia', + driverVersion: '570.195.03', + memoryTotal: '40960 MiB', + init: { + deviceRequests: { + Driver: 'nvidia', + DeviceIDs: ['GPU-uuid-1'], + Capabilities: [['gpu']] + } + }, + constraints: [ + { id: 'ram', min: 8 }, + { id: 'cpu', min: 2 } + ] + }, + { + id: 'GPU2', + kind: 'discrete', + type: 'gpu', + total: 1, + description: 'NVIDIA A100 40GB (slot 1)', + platform: 'nvidia', + driverVersion: '570.195.03', + memoryTotal: '40960 MiB', + init: { + deviceRequests: { + Driver: 'nvidia', + DeviceIDs: ['GPU-uuid-2'], + Capabilities: [['gpu']] + } + }, + constraints: [ + { id: 'ram', min: 8 }, + { id: 'cpu', min: 2 } + ] + } + ], + environments: [ + { + id: 'gpu-premium', + description: 'Paid GPU environment — any CPU job is guaranteed a GPU', + storageExpiry: 604800, + maxJobDuration: 3600, + minJobDuration: 60, + maxJobs: 2, + enableNetwork: true, + resources: [ + { + id: 'cpu', + min: 1, + max: 8, + constraints: [{ type: 'gpu', min: 1, perUnit: false }] + }, + { id: 'ram', min: 1, max: 32 }, + { id: 'disk', min: 1, max: 500 }, + { id: 'GPU1' }, + { id: 'GPU2' } + ], + access: { addresses: [], accessLists: null }, + fees: { + '1': [ + { + feeToken: '0x967da4048cD07aB37855c090aAF366e4ce1b9F48', + prices: [ + { id: 'cpu', price: 1 }, + { id: 'ram', price: 0.5 }, + { id: 'disk', price: 0.1 }, + { id: 'GPU1', price: 10 }, + { id: 'GPU2', price: 10 } + ] + } + ], + '137': [ + { + feeToken: '0x2791Bca1f2de4661ED88A30C99A7a9449Aa84174', + prices: [ + { id: 'cpu', price: 0.5 }, + { id: 'ram', price: 0.2 }, + { id: 'GPU1', price: 5 }, + { id: 'GPU2', price: 5 } + ] + } + ] + }, + free: { + maxJobDuration: 300, + minJobDuration: 10, + maxJobs: 1, + access: { addresses: [], accessLists: null }, + resources: [ + { + id: 'cpu', + max: 2, + constraints: [{ type: 'gpu', min: 1, perUnit: false }] + }, + { id: 'ram', max: 8 }, + { id: 'disk', max: 10 }, + { + id: 'GPU1', + constraints: [ + { id: 'ram', min: 4 }, + { id: 'cpu', min: 1 } + ] + } + ] + } + }, + { + id: 'cpu-standard', + description: 'CPU-only environment — cheaper, no GPU', + storageExpiry: 604800, + maxJobDuration: 1800, + minJobDuration: 60, + maxJobs: 10, + enableNetwork: false, + resources: [ + { id: 'cpu', total: 8, min: 1, max: 4 }, + { id: 'ram', total: 16, min: 1, max: 8 }, + { id: 'disk', max: 100 } + ], + access: { addresses: [], accessLists: null }, + fees: { + '1': [ + { + feeToken: '0x967da4048cD07aB37855c090aAF366e4ce1b9F48', + prices: [ + { id: 'cpu', price: 0.3 }, + { id: 'ram', price: 0.1 }, + { id: 'disk', price: 0.05 } + ] + } + ] + }, + free: { + maxJobDuration: 120, + minJobDuration: 10, + maxJobs: 3, + access: { addresses: [], accessLists: null }, + resources: [ + { id: 'cpu', max: 1 }, + { id: 'ram', max: 2 }, + { id: 'disk', max: 5 } + ] + } + } + ] + } + ] + const result = C2DDockerConfigSchema.safeParse(config) + expect(result.success).to.equal(true) + }) + describe('cpuList validation', function () { const withCpuResource = (cpuEntry: any) => [{ ...validBase, resources: [cpuEntry] }] @@ -1326,6 +2154,21 @@ describe('resolveResourceKind / resolveConnectionResourcePool / resolveEnvironme expect(gpuRes.constraints[0]).to.deep.equal({ id: 'ram', min: 8 }) }) + it('per-env group+floor constraint (type/perUnit) is carried through and deep-cloned', function () { + const envDef: any = { + resources: [ + { id: 'cpu', constraints: [{ type: 'gpu', min: 1, perUnit: false }] }, + { id: 'gpu0' } + ] + } + const result = engine.resolveEnvironmentResources(envDef, pool) + const cpuRes = result.find((r: ComputeResource) => r.id === 'cpu') + expect(cpuRes.constraints).to.deep.equal([{ type: 'gpu', min: 1, perUnit: false }]) + // deep-cloned: mutating the resolved constraint must not affect the envDef source + cpuRes.constraints[0].min = 99 + expect(envDef.resources[0].constraints[0].min).to.equal(1) + }) + it('ref.constraints: [] removes all constraints for this env', function () { const envDef = { resources: [{ id: 'gpu0', constraints: [] as any[] }] } const result = engine.resolveEnvironmentResources(envDef, pool) diff --git a/src/utils/config/schemas.ts b/src/utils/config/schemas.ts index 8c8040a00..5b4820384 100644 --- a/src/utils/config/schemas.ts +++ b/src/utils/config/schemas.ts @@ -167,11 +167,30 @@ export const DockerRegistryAuthSchema = z export const DockerRegistrysSchema = z.record(z.string(), DockerRegistryAuthSchema) -const ResourceConstraintSchema = z.object({ - id: z.string(), - min: z.number().optional(), - max: z.number().optional() -}) +// A constraint targets EITHER a single resource by `id` OR a group of resources by `type` +// (aggregated). `perUnit` (default true) keeps the historical ratio semantics +// (requiredMin = parentAmount * min); `perUnit:false` makes min/max an absolute floor/ceiling +// enforced only when the parent resource is requested. +// NOTE: keep `perUnit` optional() with no .default() — a default would inject `perUnit:true` +// into every parsed constraint and break existing deep-equal expectations on constraints. +const ResourceConstraintSchema = z + .object({ + id: z.string().optional(), + type: z.string().optional(), + min: z.number().optional(), + max: z.number().optional(), + perUnit: z.boolean().optional(), + aggregate: z.boolean().optional() + }) + .refine((c) => c.id !== undefined || c.type !== undefined, { + message: 'Each resource constraint must specify either "id" or "type"' + }) + .refine((c) => !(c.id !== undefined && c.type !== undefined), { + message: '"id" and "type" are mutually exclusive in a resource constraint' + }) + .refine((c) => !(c.aggregate === true && c.type !== undefined), { + message: 'aggregate constraints must target a single "id", not a "type" group' + }) // cpuList shape: comma-separated core IDs and/or integer ranges ("3", "0-1,3") — no // spaces, signs or floats. Every dash-separated piece of a part must be a plain