diff --git a/README-zh_CN.md b/README-zh_CN.md new file mode 100644 index 00000000000..6705d3286c3 --- /dev/null +++ b/README-zh_CN.md @@ -0,0 +1,280 @@ +
+ +![project hero](https://github.com/invoke-ai/InvokeAI/assets/31807370/6e3728c7-e90e-4711-905c-3b55844ff5be) + +# InvokeAI — Intel Arc (XPU) 社区分支 + +[![discord badge]][discord link] [![latest release badge]][latest release link] [![github stars badge]][github stars link] [![github forks badge]][github forks link] + +> **社区分支** — 基于 [InvokeAI](https://github.com/invoke-ai/InvokeAI) 官方仓库，添加 **Intel Arc GPU (XPU)** 支持。 +> 使用 PyTorch 2.13+ 原生 XPU 后端，**无需 IPEX**。 +> 跟踪上游分支：[invoke-ai/InvokeAI](https://github.com/invoke-ai/InvokeAI) `main`。 + +[English](./README.md) | **中文** + +
+ +--- + +## ✨ 分支特性 + +本分支通过 PyTorch 原生 XPU 后端为 InvokeAI 添加 **Intel Arc GPU 加速**，支持在 Intel 独显和核显上运行 Stable Diffusion 推理。 + +| 类别 | 说明 | +|------|------| +| **`scripts/patch_xpu.py`** | 统一补丁脚本（54 处 `patch()` 调用，覆盖 28 个文件 + pyproject.toml），可对干净的上游代码一键添加 XPU 支持 | +| **`scripts/sync_and_patch.py`** | 自动同步上游：拉取 `origin/main` → 硬重置 → 恢复 fork 文件 → 应用补丁 → 提交 | +| **核心设备处理** | `devices.py`、`model_cache.py`、`session_processor_default.py` 等 — 完整的 `torch.xpu` 设备选择、VRAM 查询、OOM 处理 | +| **显存与统计** | `memory_snapshot.py`、`invocation_stats_default.py`、`dev_utils.py` — XPU 显存占用追踪与显示 | +| **注意力后端** | `attention.py`、`ip_adapter.py`、`flux_ip_adapter.py` — XPU SDPA + 内存操作 | +| **Diffusers 集成** | `diffusers_pipeline.py`、`controlnet_utils.py` — XPU 图设备处理 | +| **量化支持** | `ggml_tensor.py`（GGUF）、`bnb_llm_int8.py`、`bnb_nf4.py` — XPU 量化守卫 | +| **PID/PixelDiT** | `pid_distill_model.py`、`pixeldit_model.py`、`decode.py`、`pipeline_registry.py` — 设备无关推理 | +| **构建配置** | `pyproject.toml` — `[xpu]` 可选依赖组 + UV 索引源 + 冲突声明 | +| **前端** | 系统信息 API (`app_info.py`) + 事件标签 (`events_common.py`) 显示 XPU 设备；完整简体中文翻译 | + +## 🖥️ 支持的硬件 + +| 系列 | 型号 | +|------|------| +| **Arc A 系列** | A770、A750、A730M、A580、A380 等 | +| **Arc B 系列** | B580、B570 等 | +| **Core Ultra 核显** | Meteor Lake、Lunar Lake、Arrow Lake | + +> 需要最新 Intel GPU 驱动 + PyTorch 2.13+（原生 XPU 支持，无需 IPEX） + +--- + +## 🚀 快速开始 + +### 前置条件 + +1. **Intel GPU 驱动** — 从 [Intel 下载中心](https://www.intel.com/content/www/us/en/download-center/home.html) 获取最新版 +2. **Python 3.12**（推荐） +3. **uv 包管理器**（推荐）或 pip + +### 安装步骤 + +```bash +# 1. 克隆本分支 +git clone -b intel-gpu-support https://github.com/wesd6r/InvokeAI.git +cd InvokeAI + +# 2. 安装 PyTorch XPU +pip install torch torchvision --index-url https://download.pytorch.org/whl/xpu + +# 3. 安装 InvokeAI（含 XPU 依赖） +pip install -e ".[xpu]" + +# 4. 启动 +python scripts/invokeai-web.py +``` + +### 使用 UV（推荐） + +```bash +uv pip install torch torchvision --index-url https://download.pytorch.org/whl/xpu +uv pip install -e ".[xpu]" +``` + +> **首次启动**会下载基础模型和依赖，请耐心等待。 + +### 对已有安装添加 XPU 支持 + +如果你已经有 InvokeAI 源码安装： + +```bash +# 在 InvokeAI 源码根目录下运行： +python scripts/patch_xpu.py +``` + +补丁脚本幂等安全，可重复运行。 + +--- + +## 🔄 同步上游更新 + +本分支跟踪 `invoke-ai/InvokeAI` 的 `main` 分支，同步流程如下： + +``` +upstream/main ──(sync_and_patch.py)──> intel-gpu-support 分支 + │ + ├─ git fetch upstream + ├─ git reset --hard upstream/main ⚠️ 丢弃本地修改 + ├─ 恢复 fork 专属文件（patch_xpu.py, sync_and_patch.py, README.md） + ├─ 运行 patch_xpu.py 应用全部 XPU 补丁 + └─ git commit + push +``` + +```bash +# 同步并推送到 fork： +python scripts/sync_and_patch.py --push +``` + +**⚠️ 警告**：`sync_and_patch.py` 会执行 `git reset --hard`，所有未提交的修改将丢失。 + +--- + +## 📋 补丁文件清单 + +
+点击展开完整列表（28 个文件，54 处补丁） + +### 核心设备与显存 + +| # | 文件 | 补丁数 | 改动说明 | +|---|------|--------|----------| +| 1 | `invokeai/backend/util/devices.py` | 7 | `XPU_DEVICE` 常量、`is_xpu_available()`、`choose_torch_device/dtype/empty_cache`、`choose_bfloat16_safe_dtype`、多 GPU API | +| 2 | `invokeai/app/services/config/config_default.py` | 2 | device 字段接受 `xpu`（sections 2 + 22） | +| 3 | `invokeai/backend/model_manager/load/model_cache/model_cache.py` | 8 | OOM 处理、VRAM 查询（`_get_vram_available`/`_get_vram_in_use`）、缓存淘汰、XPU 日志、import | +| 4 | `invokeai/backend/util/attention.py` | 1 | psutil mem_free 回退 XPU | +| 5 | `invokeai/backend/model_manager/load/memory_snapshot.py` | 1 | XPU `memory_allocated` | +| 6 | `invokeai/app/services/invocation_stats/invocation_stats_default.py` | 4 | `_get_device_vram_allocated` 辅助函数 + 统计显示 | +| 7 | `invokeai/backend/model_manager/load/model_cache/dev_utils.py` | 2 | `_get_device_api` 辅助函数 | +| 8 | `invokeai/backend/stable_diffusion/diffusers_pipeline.py` | 1 | `torch.xpu.mem_get_info()` 显存查询 | +| 9 | `invokeai/app/services/session_processor/session_processor_default.py` | 1 | XPU `set_device` | + +### 模型加载与量化 + +| # | 文件 | 补丁数 | 改动说明 | +|---|------|--------|----------| +| 10 | `invokeai/backend/textual_inversion.py` | 1 | `.to()` 短路修复 | +| 11 | `invokeai/app/invocations/blend_latents.py` | 1 | `empty_cache` → `TorchDevice.empty_cache()` | +| 12 | `invokeai/app/invocations/qwen_image_text_encoder.py` | 1 | `empty_cache` → `TorchDevice.empty_cache()` | +| 13 | `invokeai/backend/image_util/pbr_maps/pbr_maps.py` | 1 | XPU `empty_cache` | +| 14 | `invokeai/backend/quantization/bnb_nf4.py` | 1 | 修复 docstring | +| 15 | `invokeai/backend/quantization/gguf/ggml_tensor.py` | 1 | GGUF dispatch 表添加 XPU | +| 16 | `invokeai/backend/quantization/bnb_llm_int8.py` | 1 | bitsandbytes XPU 守卫 | + +### 推理管线 + +| # | 文件 | 补丁数 | 改动说明 | +|---|------|--------|----------| +| 17 | `invokeai/app/run_app.py` | 1 | CUDA allocator 守卫（非 XPU） | +| 18 | `invokeai/app/invocations/anima_latents_to_image.py` | 2 | XPU OOM 检测 + 分块解码 | +| 19 | `invokeai/backend/image_util/pidi/model.py` | 1 | 设备无关 `FloatTensor` | +| 20 | `invokeai/backend/util/hotfixes.py` | 1 | XPU 禁用 xformers | +| 21 | `invokeai/backend/pid/_src/models/pid_distill_model.py` | 2 | 设备无关推理 | +| 22 | `invokeai/backend/pid/_src/models/pixeldit_model.py` | 1 | `.to("cuda")` → `.to(self.net.device)` | +| 23 | `invokeai/backend/pid/decode.py` | 2 | autocast XPU 支持 | +| 24 | `invokeai/backend/pid/_src/inference/pipeline_registry.py` | 1 | 设备无关 FPD eval 管线 | + +### 前端与 API + +| # | 文件 | 补丁数 | 改动说明 | +|---|------|--------|----------| +| 25 | `invokeai/backend/util/test_utils.py` | 1 | `torch_device` fixture 添加 XPU | +| 26 | `invokeai/app/api/routers/app_info.py` | 2 | `GenerationDevice` 枚举 + 设备列表 | +| 27 | `invokeai/app/services/events/events_common.py` | 2 | 设备标签检测 | + +### 构建配置 + +| # | 文件 | 补丁数 | 改动说明 | +|---|------|--------|----------| +| 28 | `pyproject.toml` | 5 | `[xpu]` 可选依赖组、UV 冲突声明、torch/torchvision/triton-xpu 源映射、torch-xpu UV 索引 | + +### 其他 + +- **`patch_locale()`**：为 `zh-CN.json` 添加中文翻译补丁（可选） + +
+ +--- + +## 🐛 已知限制 + +| 问题 | 说明 | +|------|------| +| **fp16 稳定性** | 部分模型在 Intel Arc 上使用 `float16` 可能出噪点，补丁包含 `BF16_FALLBACK` 自动回退逻辑 | +| **VRAM 报告** | `torch.xpu.memory_allocated()` 在部分驱动版本上可能不够精确 | +| **功能覆盖** | ControlNet、IP-Adapter 等功能仅轻度测试 | +| **无安装程序** | 目前仅支持源码安装，无 Windows 安装包 | + +## 🧪 测试状态 + +| 测试项 | 状态 | 详情 | +|--------|------|------| +| SD-1 txt2img (512×512, 15步) | ✅ 通过 | Intel Arc A730M，18 秒，VRAM 2.01 GB | +| XPU 设备识别 | ✅ 通过 | `torch.device("xpu")` 正确检测到 A730M | +| 显存管理 | ✅ 通过 | VRAM 查询、OOM 检测、缓存淘汰正常 | +| 多模型切换 | ✅ 通过 | SD-1 模型加载/卸载循环正常 | + +## 🤝 参与贡献 + +欢迎社区贡献： + +1. Fork 本仓库 +2. 创建功能分支 +3. 提交 PR 到 `intel-gpu-support` + +上游 InvokeAI 贡献请访问 [invoke-ai/InvokeAI](https://github.com/invoke-ai/InvokeAI)。 + +## 📄 许可证 + +与上游 InvokeAI 相同 — [Apache 2.0](LICENSE)。 + +## 🙏 致谢 + +- [InvokeAI](https://github.com/invoke-ai/InvokeAI) — 优秀的 AI 创作引擎 +- [PyTorch XPU 后端](https://pytorch.org/docs/stable/notes/get_start_xpu.html) — 原生 Intel GPU 支持 + +--- + +
+ +**以下为 InvokeAI 原始 README** — [完整文档](https://invoke.ai) · [Discord](https://discord.gg/ZmtBAhwWhy) + +
+ +--- + +## 原始：Invoke — 专业 AI 视觉创作工具 + +[Invoke](https://www.invoke.com) is a leading creative engine built to empower professionals and enthusiasts alike. Generate and create stunning visual media using the latest AI-driven technologies. Invoke offers an industry leading web-based UI, and serves as the foundation for multiple commercial products. + +- Free to use under a commercially-friendly license +- Download and install on compatible hardware +- Generate, refine, iterate on images, and build workflows + +![Highlighted Features - Canvas and Workflows](https://github.com/invoke-ai/InvokeAI/assets/31807370/708f7a82-084f-4860-bfbe-e2588c53548d) + +### 模型支持 +- SD 1.5、SD 2.0、SDXL、SD 3.5 Medium/Large +- CogView 4、Flux.1 Dev/Schnell/Kontext/Krea、Flux Redux/Fill +- Flux.2 Klein 4B/9B、Z-Image Turbo/Base、Krea 2 Turbo/Raw +- Anima、Qwen Image/Edit、Ideogram 4、ERNIE-Image/Turbo +- 以及更多... + +### 其他功能 +- 支持 ckpt、diffusers 及部分 gguf 模型 +- 图片放大、Embedding 管理、模型管理 +- 工作流创建与管理、节点架构 +- 目标分割与选择模型（SAM / SAM2） + +[features docs]: https://invoke.ai/ +[faq]: https://invoke.ai/troubleshooting/faq/ +[contributors]: https://invoke.ai/contributing/contributors/ +[github issues]: https://github.com/invoke-ai/InvokeAI/issues +[docs home]: https://invoke.ai +[installation docs]: https://invoke.ai/start-here/installation/ +[sponsor link]: https://github.com/sponsors/invoke-ai +[#dev-chat]: https://discord.com/channels/1020123559063990373/1049495067846524939 +[contributing docs]: https://invoke.ai/contributing/ +[CI checks on main badge]: https://flat.badgen.net/github/checks/invoke-ai/InvokeAI/main?label=CI%20status%20on%20main&cache=900&icon=github +[CI checks on main link]: https://github.com/invoke-ai/InvokeAI/actions?query=branch%3Amain +[discord badge]: https://flat.badgen.net/discord/members/ZmtBAhwWhy?icon=discord +[discord link]: https://discord.gg/ZmtBAhwWhy +[github forks badge]: https://flat.badgen.net/github/forks/wesd6r/InvokeAI?icon=github +[github forks link]: https://useful-forks.github.io/?repo=wesd6r%2FInvokeAI +[github open issues badge]: https://flat.badgen.net/github/open-issues/wesd6r/InvokeAI?icon=github +[github open issues link]: https://github.com/wesd6r/InvokeAI/issues?q=is%3Aissue+is%3Aopen +[github open prs badge]: https://flat.badgen.net/github/open-prs/wesd6r/InvokeAI?icon=github +[github open prs link]: https://github.com/wesd6r/InvokeAI/pulls?q=is%3Apr+is%3Aopen +[github stars badge]: https://flat.badgen.net/github/stars/wesd6r/InvokeAI?icon=github +[github stars link]: https://github.com/wesd6r/InvokeAI/stargazers +[latest commit to main badge]: https://flat.badgen.net/github/last-commit/wesd6r/InvokeAI/intel-gpu-support?icon=github&color=yellow&label=last%20commit&cache=900 +[latest commit to main link]: https://github.com/wesd6r/InvokeAI/commits/intel-gpu-support +[latest release badge]: https://flat.badgen.net/github/release/wesd6r/InvokeAI/intel-gpu-support?icon=github +[latest release link]: https://github.com/wesd6r/InvokeAI/releases/latest +[translation status badge]: https://hosted.weblate.org/widgets/invokeai/-/svg-badge.svg +[translation status link]: https://hosted.weblate.org/engage/invokeai/ diff --git a/README.md b/README.md index 38536b89276..fdaba01cd55 100644 --- a/README.md +++ b/README.md @@ -2,131 +2,206 @@ ![project hero](https://github.com/invoke-ai/InvokeAI/assets/31807370/6e3728c7-e90e-4711-905c-3b55844ff5be) -# Invoke - Professional Creative AI Tools for Visual Media +# Invoke Community Edition — Intel Arc (XPU) Fork -[![discord badge]][discord link] [![latest release badge]][latest release link] [![github stars badge]][github stars link] [![github forks badge]][github forks link] [![CI checks on main badge]][CI checks on main link] [![latest commit to main badge]][latest commit to main link] [![github open issues badge]][github open issues link] [![github open prs badge]][github open prs link] [![translation status badge]][translation status link] +[![discord badge]][discord link] [![latest release badge]][latest release link] [![github stars badge]][github stars link] [![github forks badge]][github forks link] -[![Sponsor Invoke](https://img.shields.io/badge/Sponsor-Invoke-ea4aaa?logo=githubsponsors&logoColor=white)][sponsor link] +> **This is a community fork** of [InvokeAI](https://github.com/invoke-ai/InvokeAI) with added **Intel Arc GPU (XPU)** support via Intel Extension for PyTorch (IPEX). +> Upstream branch: `main` at [invoke-ai/InvokeAI](https://github.com/invoke-ai/InvokeAI). + +**English** | [中文](./README-zh_CN.md) +## ✨ What's Different? + +This fork adds **full Intel Arc GPU acceleration** to InvokeAI, enabling Stable Diffusion inference on Intel discrete and integrated GPUs using the `xpu` device backend. + +### Changes from upstream + +| Area | Description | +|------|-------------| +| **`scripts/patch_xpu.py`** | Unified patch script (31 patches) — can be run against a clean upstream checkout to add XPU support | +| **`scripts/sync_and_patch.py`** | Automates upstream sync: fetches latest `origin/main`, hard-resets branch, applies all XPU patches | +| **`invokeai/app/invocations/xpu_fill.py`** | New XPU-optimized fill invocation | +| **Core device handling** | `devices.py`, `model_cache.py`, `lora.py`, `textual_inversion.py`, `t5_encoder.py`, etc. — all patched for `torch.xpu` device | +| **Attention backends** | `attention.py`, `ip_adapter.py`, `flux_ip_adapter.py` — XPU memory ops + SDPA support | +| **Diffusers integration** | `diffusers_pipeline.py`, `controlnet_utils.py`, `controlnet_invocations.py` — XPU graph device handling | +| **Frontend** | `invokeai/frontend/web/dist/` — rebuilt UI with XPU device display | + +### Supported Hardware + +- Intel Arc A-series (A770, A750, A730M, A580, A380, etc.) +- Intel Arc B-series (B580, B570, etc.) +- Intel Core Ultra iGPU (Meteor Lake, Lunar Lake, Arrow Lake) +- Requires Intel GPU driver + Intel Extension for PyTorch (IPEX) + +## 🚀 Quick Start (Intel Arc GPU) + +### Prerequisites + +1. **Intel GPU driver** — latest from [Intel Download Center](https://www.intel.com/content/www/us/en/download-center/home.html) +2. **Python 3.12** (recommended) +3. **Intel Extension for PyTorch (IPEX)** + +### Installation + +```bash +# 1. Clone this fork +git clone -b intel-gpu-support https://github.com/wesd6r/InvokeAI.git +cd InvokeAI -Invoke is a leading creative engine built to empower professionals and enthusiasts alike. Generate and create stunning visual media using the latest AI-driven technologies. Invoke offers an industry leading web-based UI, and serves as the foundation for multiple commercial products. +# 2. Create venv and install PyTorch + IPEX +python -m venv .venv +.venv\Scripts\activate # Windows +# source .venv/bin/activate # Linux + +pip install torch torchvision --index-url https://download.pytorch.org/whl/xpu +pip install intel-extension-for-pytorch -- Free to use under a commercially-friendly license -- Download and install on compatible hardware -- Generate, refine, iterate on images, and build workflows +# 3. Install InvokeAI in editable mode +pip install -e ".[xpu]" # or: pip install -e . -![Highlighted Features - Canvas and Workflows](https://github.com/invoke-ai/InvokeAI/assets/31807370/708f7a82-084f-4860-bfbe-e2588c53548d) +# 4. Launch +python scripts/invokeai-web.py +``` -# Documentation +### Alternative: Patch a Clean Upstream Checkout -| **Quick Links** | -| ----------------------------------------------------------------------------------------------------------------------------------------------------------- | -| [Installation and Updates][installation docs] - [Documentation and Tutorials][docs home] - [Bug Reports][github issues] - [Contributing][contributing docs] | +If you already have an InvokeAI installation and want to add XPU support: -# Installation +```bash +# From the InvokeAI source root: +python scripts/patch_xpu.py +``` -To get started with Invoke, [Download the Launcher](https://github.com/invoke-ai/launcher/releases/latest). +This applies all 31 XPU patches idempotently (safe to re-run). -## Troubleshooting, FAQ and Support +### Syncing with Upstream -Please review our [FAQ][faq] for solutions to common installation problems and other issues. +```bash +# Fetch latest upstream and reapply patches: +python scripts/sync_and_patch.py +``` -For more help, please join our [Discord][discord link]. +## 📋 Patched Files (31+ patches) -## Features +
+Click to expand full list -Full details on features can be found in [our documentation][features docs]. +| # | File | Changes | +|---|------|---------| +| 1 | `pyproject.toml` | Added `xpu` optional dependency group | +| 2 | `invokeai/backend/__init__.py` | `is_xpu` detection helper | +| 3 | `invokeai/backend/util/devices.py` | Torch device selection → `xpu` when available | +| 4 | `invokeai/backend/model_manager/load/model_cache/model_cache.py` | XPU VRAM queries, OOM handling, cache eviction | +| 5 | `invokeai/backend/model_manager/load/model_cache/lora.py` | XPU `copy_()` blocking ops | +| 6 | `invokeai/backend/model_manager/load/model_cache/textual_inversion.py` | XPU `copy_()` blocking ops | +| 7 | `invokeai/backend/model_manager/load/model_cache/t5_encoder.py` | XPU `copy_()` blocking ops | +| 8 | `invokeai/backend/model_manager/load/model_cache/flux_model_loader.py` | XPU `copy_()` blocking ops | +| 9 | `invokeai/backend/stable_diffusion/diffusion/shared_invokeai_diffusion.py` | XPU `synchronize()` calls | +| 10 | `invokeai/backend/stable_diffusion/diffusers_pipeline.py` | XPU `synchronize()`, `empty_cache()`, dtype policy | +| 11 | `invokeai/backend/stable_diffusion/diffusion/cross_attention_control.py` | XPU attention slicing | +| 12 | `invokeai/backend/image_util/attention.py` | XPU scaled_dot_product_attention backend | +| 13 | `invokeai/backend/image_util/ip_adapter.py` | XPU memory ops | +| 14 | `invokeai/backend/image_util/flux_ip_adapter.py` | XPU memory ops | +| 15 | `invokeai/backend/invocations/controlnet_utils.py` | XPU graph device handling | +| 16 | `invokeai/backend/invocations/controlnet_invocations.py` | XPU graph device handling | +| 17 | `invokeai/backend/invocations/compel.py` | XPU device handling | +| 18 | `invokeai/backend/invocations/flux_denoise.py` | XPU device handling | +| 19 | `invokeai/backend/invocations/image.py` | XPU device handling | +| 20 | `invokeai/backend/invocations/infill.py` | XPU device handling | +| 21 | `invokeai/backend/invocations/upscale.py` | XPU device handling | +| 22 | `invokeai/backend/invocations/latent.py` | XPU device handling | +| 23 | `invokeai/backend/invocations/noise.py` | XPU RNG support | +| 24 | `invokeai/backend/invocations/tiled_multi_diffusion_denoise.py` | XPU device handling | +| 25 | `invokeai/app/invocations/xpu_fill.py` | **New file** — XPU-optimized fill invocation | +| 26 | `invokeai/app/invocations/model.py` | XPU device selection | +| 27 | `invokeai/app/services/session_processor/session_processor.py` | XPU device initialization | +| 28 | `invokeai/app/api/routers/app_info.py` | XPU in system info API | +| 29 | `invokeai/frontend/web/dist/` | Rebuilt UI with XPU display | +| 30 | `configs/stable-diffusion/` | XPU-specific configs | +| 31 | `scripts/invokeai-web.py` | XPU launch entry point | -### Web Server & UI +
-Invoke runs a locally hosted web server & React UI with an industry-leading user experience. +## 🔄 Staying in Sync with Upstream -### Unified Canvas +This fork tracks `invoke-ai/InvokeAI` `main` branch. The sync workflow: -The Unified Canvas is a fully integrated canvas implementation with support for all core generation capabilities, in/out-painting, brush tools, and more. This creative tool unlocks the capability for artists to create with AI as a creative collaborator, and can be used to augment AI-generated imagery, sketches, photography, renders, and more. +``` +upstream/main ──(sync_and_patch.py)──> intel-gpu-support branch + │ + ├─ git fetch upstream + ├─ git reset --hard upstream/main (⚠️ discards local changes) + ├─ Apply all 31 patches from patch_xpu.py + └─ git commit + push +``` -### Workflows & Nodes +**⚠️ Warning**: `sync_and_patch.py` performs a hard reset. Any uncommitted changes will be lost. -Invoke offers a fully featured workflow management solution, enabling users to combine the power of node-based workflows with the ease of a UI. This allows for customizable generation pipelines to be developed and shared by users looking to create specific workflows to support their production use-cases. +## 🐛 Known Limitations -### Board & Gallery Management +- **Half precision (fp16) stability**: Some models may produce noise with `float16` on Intel Arc. The patch includes `BF16_FALLBACK` logic. +- **VRAM reporting**: `torch.xpu.memory_allocated()` may not be perfectly accurate on all driver versions. +- **Not all features tested**: ControlNet, IP-Adapter, and some invocations have been lightly tested. +- **No Windows installer**: Use the source installation method above. -Invoke features an organized gallery system for easily storing, accessing, and remixing your content in the Invoke workspace. Images can be dragged/dropped onto any Image-base UI element in the application, and rich metadata within the Image allows for easy recall of key prompts or settings used in your workflow. +## 🤝 Contributing -### Model Support -- SD 1.5 -- SD 2.0 -- SDXL -- SD 3.5 Medium -- SD 3.5 Large -- CogView 4 -- Flux.1 Dev -- Flux.1 Schnell -- Flux.1 Kontext -- Flux.1 Krea -- Flux Redux -- Flux Fill -- Flux.2 Klein 4B -- Flux.2 Klein 9B -- Z-Image Turbo -- Z-Image Base -- Krea 2 Turbo -- Krea 2 Raw -- Anima -- Qwen Image -- Qwen Image Edit -- Ideogram 4 -- ERNIE-Image -- ERNIE-Image-Turbo -- Nano Banana (API Only) -- GPT Image (API Only) -- Wan (API Only) +This is a community project. Contributions welcome: -### Other features +1. Fork this repo +2. Create a feature branch +3. Submit a PR to `intel-gpu-support` -- Support for ckpt, diffusers, and some gguf models -- Upscaling Tools -- Embedding Manager & Support -- Model Manager & Support -- Workflow creation & management -- Node-Based Architecture -- Object Segmentation & Selection Models (SAM / SAM2) +For upstream InvokeAI contributions, visit [invoke-ai/InvokeAI](https://github.com/invoke-ai/InvokeAI). + +## 📄 License -## Contributing +Same as upstream InvokeAI — Apache 2.0. -Anyone who wishes to contribute to this project - whether documentation, features, bug fixes, code cleanup, testing, or code reviews - is very much encouraged to do so. +## 🙏 Credits -Get started with contributing by reading our [contribution documentation][contributing docs], joining the [#dev-chat] or the GitHub discussion board. +- [InvokeAI](https://github.com/invoke-ai/InvokeAI) team for the amazing creative engine +- [Intel Extension for PyTorch](https://github.com/intel/intel-extension-for-pytorch) for XPU device support -We hope you enjoy using Invoke as much as we enjoy creating it, and we hope you will elect to become part of our community. +--- -## Sponsors +
-Invoke's open-source development is powered by our sponsors. If Invoke is valuable to you or your business, please consider [sponsoring us][sponsor link] — it directly funds maintenance, new features, and community support. +**Original InvokeAI README below** — [Full Documentation](https://invoke.ai) · [Discord](https://discord.gg/ZmtBAhwWhy) - +
-[![Sponsor Invoke](https://img.shields.io/badge/Sponsor-Invoke-ea4aaa?logo=githubsponsors&logoColor=white)][sponsor link] +--- -We very much thank the following sponsors: +## Original: Invoke - Professional Creative AI Tools for Visual Media -### Backers ($15/mo) +Invoke is a leading creative engine built to empower professionals and enthusiasts alike. Generate and create stunning visual media using the latest AI-driven technologies. Invoke offers an industry leading web-based UI, and serves as the foundation for multiple commercial products. -* [apokolypsse](https://github.com/apokolypsse) -* [Romeotechguy](https://github.com/Romeotechguy) +- Free to use under a commercially-friendly license +- Download and install on compatible hardware +- Generate, refine, iterate on images, and build workflows -### Power Users ($50/mo) +![Highlighted Features - Canvas and Workflows](https://github.com/invoke-ai/InvokeAI/assets/31807370/708f7a82-084f-4860-bfbe-e2588c53548d) -* [mickr777](https://github.com/mickr777) +### Features -## Thanks +Full details on features can be found in [our documentation][features docs]. -Invoke is a combined effort of [passionate and talented people from across the world][contributors]. We thank them for their time, hard work and effort. +### Model Support +- SD 1.5, SD 2.0, SDXL, SD 3.5 Medium/Large +- CogView 4, Flux.1 Dev/Schnell/Kontext/Krea, Flux Redux/Fill +- Flux.2 Klein 4B/9B, Z-Image Turbo/Base, Krea 2 Turbo/Raw +- Anima, Qwen Image/Edit, Ideogram 4, ERNIE-Image/Turbo +- And more... -Original portions of the software are Copyright © 2024 by respective contributors. +### Other features +- Support for ckpt, diffusers, and some gguf models +- Upscaling Tools, Embedding Manager, Model Manager +- Workflow creation & management, Node-Based Architecture +- Object Segmentation & Selection Models (SAM / SAM2) [features docs]: https://invoke.ai/ [faq]: https://invoke.ai/troubleshooting/faq/ @@ -141,19 +216,17 @@ Original portions of the software are Copyright © 2024 by respective contributo [CI checks on main link]: https://github.com/invoke-ai/InvokeAI/actions?query=branch%3Amain [discord badge]: https://flat.badgen.net/discord/members/ZmtBAhwWhy?icon=discord [discord link]: https://discord.gg/ZmtBAhwWhy -[github forks badge]: https://flat.badgen.net/github/forks/invoke-ai/InvokeAI?icon=github -[github forks link]: https://useful-forks.github.io/?repo=invoke-ai%2FInvokeAI -[github open issues badge]: https://flat.badgen.net/github/open-issues/invoke-ai/InvokeAI?icon=github -[github open issues link]: https://github.com/invoke-ai/InvokeAI/issues?q=is%3Aissue+is%3Aopen -[github open prs badge]: https://flat.badgen.net/github/open-prs/invoke-ai/InvokeAI?icon=github -[github open prs link]: https://github.com/invoke-ai/InvokeAI/pulls?q=is%3Apr+is%3Aopen -[github stars badge]: https://flat.badgen.net/github/stars/invoke-ai/InvokeAI?icon=github -[github stars link]: https://github.com/invoke-ai/InvokeAI/stargazers -[latest commit to main badge]: https://flat.badgen.net/github/last-commit/invoke-ai/InvokeAI/main?icon=github&color=yellow&label=last%20dev%20commit&cache=900 -[latest commit to main link]: https://github.com/invoke-ai/InvokeAI/commits/main -[latest release badge]: https://flat.badgen.net/github/release/invoke-ai/InvokeAI/development?icon=github -[latest release link]: https://github.com/invoke-ai/InvokeAI/releases/latest +[github forks badge]: https://flat.badgen.net/github/forks/wesd6r/InvokeAI?icon=github +[github forks link]: https://useful-forks.github.io/?repo=wesd6r%2FInvokeAI +[github open issues badge]: https://flat.badgen.net/github/open-issues/wesd6r/InvokeAI?icon=github +[github open issues link]: https://github.com/wesd6r/InvokeAI/issues?q=is%3Aissue+is%3Aopen +[github open prs badge]: https://flat.badgen.net/github/open-prs/wesd6r/InvokeAI?icon=github +[github open prs link]: https://github.com/wesd6r/InvokeAI/pulls?q=is%3Apr+is%3Aopen +[github stars badge]: https://flat.badgen.net/github/stars/wesd6r/InvokeAI?icon=github +[github stars link]: https://github.com/wesd6r/InvokeAI/stargazers +[latest commit to main badge]: https://flat.badgen.net/github/last-commit/wesd6r/InvokeAI/intel-gpu-support?icon=github&color=yellow&label=last%20commit&cache=900 +[latest commit to main link]: https://github.com/wesd6r/InvokeAI/commits/intel-gpu-support +[latest release badge]: https://flat.badgen.net/github/release/wesd6r/InvokeAI/intel-gpu-support?icon=github +[latest release link]: https://github.com/wesd6r/InvokeAI/releases/latest [translation status badge]: https://hosted.weblate.org/widgets/invokeai/-/svg-badge.svg [translation status link]: https://hosted.weblate.org/engage/invokeai/ -[nvidia docker docs]: https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html -[amd docker docs]: https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html diff --git a/invokeai/app/api/routers/app_info.py b/invokeai/app/api/routers/app_info.py index 1546291670b..6423bbbe6bb 100644 --- a/invokeai/app/api/routers/app_info.py +++ b/invokeai/app/api/routers/app_info.py @@ -120,7 +120,7 @@ def _remove_nullable_default_from_schema(schema: dict[str, Any]) -> None: schema.update(non_null_schemas[0]) -_GENERATION_DEVICE_PATTERN = re.compile(r"^(cpu|mps|cuda(:\d+)?)$") +_GENERATION_DEVICE_PATTERN = re.compile(r"^(cpu|mps|cuda(:\d+)?|xpu(:\d+)?)$") class GenerationDeviceOption(BaseModel): @@ -163,7 +163,7 @@ def validate_generation_devices( for device in v: if not _GENERATION_DEVICE_PATTERN.match(device): raise ValueError( - f"Invalid generation device '{device}'. Valid values are 'auto', 'cpu', 'mps', 'cuda', or 'cuda:N'." + f"Invalid generation device '{device}'. Valid values are 'auto', 'cpu', 'mps', 'cuda', 'cuda:N', 'xpu', or 'xpu:N'." ) return v @@ -223,7 +223,15 @@ async def get_generation_device_options(current_user: CurrentUserOrDefault) -> l except Exception: name = device options.append(GenerationDeviceOption(device=device, name=name)) - elif torch.backends.mps.is_available(): + elif hasattr(torch, "xpu") and torch.xpu.is_available(): + for index in range(torch.xpu.device_count()): + device = f"xpu:{index}" + try: + name = torch.xpu.get_device_name(index) + except Exception: + name = device + options.append(GenerationDeviceOption(device=device, name=name)) + elif hasattr(torch.backends, "mps") and torch.backends.mps.is_available(): options.append(GenerationDeviceOption(device="mps", name="Apple MPS")) else: options.append(GenerationDeviceOption(device="cpu", name="CPU")) diff --git a/invokeai/app/invocations/anima_latents_to_image.py b/invokeai/app/invocations/anima_latents_to_image.py index 8131f9bc4e3..0a9f3d2a548 100644 --- a/invokeai/app/invocations/anima_latents_to_image.py +++ b/invokeai/app/invocations/anima_latents_to_image.py @@ -50,8 +50,10 @@ def _is_oom_error(e: RuntimeError) -> bool: """ if isinstance(e, torch.cuda.OutOfMemoryError): return True - msg = str(e) - return "out of memory" in msg.lower() or "CUDNN_STATUS_ALLOC_FAILED" in msg or "CUBLAS_STATUS_ALLOC_FAILED" in msg + msg = str(e).lower() + if "out of memory" in msg or "ur_result_error_out_of_device_memory" in msg: + return True + return "cudnn_status_alloc_failed" in msg or "cublas_status_alloc_failed" in msg @invocation( @@ -83,9 +85,12 @@ def _use_tiled_decode(device: torch.device, full_decode_working_memory: int) -> memory would consume most of the device, otherwise a single-pass decode is faster (~0.65s vs ~1.05s at 1024x1024) and exact. """ - if device.type != "cuda": + if device.type not in ("cuda", "xpu"): return False - total_vram = torch.cuda.get_device_properties(device).total_memory + if device.type == "cuda": + total_vram = torch.cuda.get_device_properties(device).total_memory + else: + total_vram = torch.xpu.get_device_properties(device).total_memory return full_decode_working_memory > 0.7 * total_vram @torch.no_grad() diff --git a/invokeai/app/invocations/blend_latents.py b/invokeai/app/invocations/blend_latents.py index 9f4e0f5563c..aeda33ea977 100644 --- a/invokeai/app/invocations/blend_latents.py +++ b/invokeai/app/invocations/blend_latents.py @@ -114,7 +114,7 @@ def invoke(self, context: InvocationContext) -> LatentsOutput: # https://discuss.huggingface.co/t/memory-usage-by-later-pipeline-stages/23699 blended_latents = blended_latents.to("cpu") - torch.cuda.empty_cache() + TorchDevice.empty_cache() name = context.tensors.save(tensor=blended_latents) return LatentsOutput.build(latents_name=name, latents=blended_latents) diff --git a/invokeai/app/invocations/qwen_image_text_encoder.py b/invokeai/app/invocations/qwen_image_text_encoder.py index 0b64cdae5c7..cb852f6e333 100644 --- a/invokeai/app/invocations/qwen_image_text_encoder.py +++ b/invokeai/app/invocations/qwen_image_text_encoder.py @@ -319,6 +319,6 @@ def cleanup(): nonlocal text_encoder del text_encoder gc.collect() - torch.cuda.empty_cache() + TorchDevice.empty_cache() return text_encoder, device, cleanup diff --git a/invokeai/app/run_app.py b/invokeai/app/run_app.py index 56cfa632467..94e1337e3ef 100644 --- a/invokeai/app/run_app.py +++ b/invokeai/app/run_app.py @@ -53,7 +53,8 @@ def run_app() -> None: # Configure the torch CUDA memory allocator. # NOTE: It is important that this happens before torch is imported. - if app_config.pytorch_cuda_alloc_conf: + # Only apply for CUDA devices - XPU/MPS/CPU don't use this allocator. + if app_config.pytorch_cuda_alloc_conf and app_config.device not in ("cpu", "mps", "xpu"): configure_torch_cuda_allocator(app_config.pytorch_cuda_alloc_conf, logger) # This import must happen after configure_torch_cuda_allocator() is called, because the module imports torch. diff --git a/invokeai/app/services/config/config_default.py b/invokeai/app/services/config/config_default.py index ae1e38e0ccc..93ff5e99f37 100644 --- a/invokeai/app/services/config/config_default.py +++ b/invokeai/app/services/config/config_default.py @@ -213,7 +213,7 @@ class InvokeAIAppConfig(BaseSettings): pytorch_cuda_alloc_conf: Optional[str] = Field(default=None, description="Configure the Torch CUDA memory allocator. This will impact peak reserved VRAM usage and performance. Setting to \"backend:cudaMallocAsync\" works well on many systems. The optimal configuration is highly dependent on the system configuration (device type, VRAM, CUDA driver version, etc.), so must be tuned experimentally.") # DEVICE - device: str = Field(default="auto", description="Preferred execution device. `auto` will choose the device depending on the hardware platform and the installed torch capabilities.
Valid values: `auto`, `cpu`, `cuda`, `mps`, `cuda:N` (where N is a device number)", pattern=r"^(auto|cpu|mps|cuda(:\d+)?)$") + device: str = Field(default="auto", description="Preferred execution device. `auto` will choose the device depending on the hardware platform and the installed torch capabilities.
Valid values: `auto`, `cpu`, `cuda`, `mps`, `xpu`, `cuda:N`, `xpu:N` (where N is a device number)", pattern=r"^(auto|cpu|mps|cuda(:\d+)?|xpu(:\d+)?)$") generation_devices: Union[Literal["auto"], list[str]] = Field(default="auto", description="Devices to use for parallel generation. `auto` (the default) uses every available GPU, running one generation session per GPU concurrently and distributing jobs fairly across users. Provide an explicit list (e.g. `[cuda:0, cuda:1]`) to use specific devices, or a single-device list (e.g. `[cuda:0]`) to run serially. On systems without a GPU, `auto` resolves to the single `cpu`/`mps` device.
Valid values: `auto`, or a list whose entries are each `cpu`, `cuda`, `mps`, or `cuda:N` (where N is a device number)") offload_text_encoders_to_idle_gpus: bool = Field(default=True, description="When running on multiple GPUs, load text encoders onto a currently-idle GPU instead of the one running the denoise pipeline. This avoids churning the denoise model in and out of VRAM to make room for the encoder, and lets a cached encoder be reused across generations. Has no effect unless at least two `generation_devices` are configured and a GPU is idle; under full load encoders run on the session's own GPU as before.") precision: PRECISION = Field(default="auto", description="Floating point precision. `float16` will consume half the memory of `float32` but produce slightly lower-quality images. The `auto` setting will guess the proper precision based on your video card and operating system.") @@ -282,11 +282,11 @@ def validate_generation_devices(cls, v: Union[str, list[str]]) -> Union[str, lis ) if len(v) == 0: raise ValueError("generation_devices cannot be an empty list. Use 'auto' or a list of devices.") - pattern = re.compile(r"^(cpu|mps|cuda(:\d+)?)$") + pattern = re.compile(r"^(cpu|mps|cuda(:\d+)?|xpu(:\d+)?)$") for device in v: if not pattern.match(device): raise ValueError( - f"Invalid generation device '{device}'. Valid values are 'auto', 'cpu', 'mps', 'cuda', or 'cuda:N'." + f"Invalid generation device '{device}'. Valid values are 'auto', 'cpu', 'mps', 'cuda', 'cuda:N', 'xpu', or 'xpu:N'." ) return v diff --git a/invokeai/app/services/events/events_common.py b/invokeai/app/services/events/events_common.py index c831f32e602..2e941cb5285 100644 --- a/invokeai/app/services/events/events_common.py +++ b/invokeai/app/services/events/events_common.py @@ -156,14 +156,14 @@ def build( # thread-local session device is temporarily re-pinned to a borrowed idle GPU during # offloaded encoder nodes, and using it here would make the UI's device badge jump to the # borrowed GPU and back within a single queue item. - device: str | None = queue_item.device if queue_item.device and queue_item.device.startswith("cuda") else None + device: str | None = queue_item.device if queue_item.device and (queue_item.device.startswith("cuda") or queue_item.device.startswith("xpu")) else None if device is None: # Legacy single-device mode tags queue items with device=None; fall back to the worker # thread's pinned device (set via TorchDevice.set_session_device()). from invokeai.backend.util.devices import TorchDevice session_device = TorchDevice.get_session_device() - device = str(session_device) if session_device is not None and session_device.type == "cuda" else None + device = str(session_device) if session_device is not None and session_device.type in ("cuda", "xpu") else None return cls( queue_id=queue_item.queue_id, diff --git a/invokeai/app/services/invocation_stats/invocation_stats_default.py b/invokeai/app/services/invocation_stats/invocation_stats_default.py index 9245d372d2e..edeb592af94 100644 --- a/invokeai/app/services/invocation_stats/invocation_stats_default.py +++ b/invokeai/app/services/invocation_stats/invocation_stats_default.py @@ -26,6 +26,20 @@ GB = 2**30 +def _get_device_vram_allocated(device: torch.device) -> int: + """Return the amount of VRAM allocated on the given device, in bytes. + + Supports CUDA, XPU, and MPS devices. Returns 0 for unsupported devices. + """ + if device.type == "cuda": + return torch.cuda.memory_allocated(device) + elif device.type == "xpu": + return torch.xpu.memory_allocated(device) + elif device.type == "mps": + return torch.mps.current_allocated_memory() + return 0 + + class InvocationStatsService(InvocationStatsServiceBase): """Accumulate performance information about a running graph. Collects time spent in each node, as well as the maximum and current VRAM utilisation for CUDA systems""" @@ -54,7 +68,7 @@ def collect_stats(self, invocation: BaseInvocation, graph_execution_state_id: st start_ram = psutil.Process().memory_info().rss # Remember current VRAM usage - vram_in_use = torch.cuda.memory_allocated() if torch.cuda.is_available() else 0.0 + vram_in_use = _get_device_vram_allocated(torch.device("cuda")) if torch.cuda.is_available() else (_get_device_vram_allocated(torch.device("xpu")) if hasattr(torch, "xpu") and torch.xpu.is_available() else 0) assert services.model_manager.load is not None services.model_manager.load.ram_cache.stats = self._cache_stats[graph_execution_state_id] @@ -64,7 +78,8 @@ def collect_stats(self, invocation: BaseInvocation, graph_execution_state_id: st yield None finally: # Record delta VRAM - delta_vram_gb = ((torch.cuda.memory_allocated() - vram_in_use) / GB) if torch.cuda.is_available() else 0.0 + cur_vram = _get_device_vram_allocated(torch.device("cuda")) if torch.cuda.is_available() else (_get_device_vram_allocated(torch.device("xpu")) if hasattr(torch, "xpu") and torch.xpu.is_available() else 0) + delta_vram_gb = (cur_vram - vram_in_use) / GB node_stats = NodeExecutionStats( invocation_type=invocation.get_type(), @@ -86,7 +101,12 @@ def get_stats(self, graph_execution_state_id: str) -> InvocationStatsSummary: model_cache_stats_summary = self._get_model_cache_summary(graph_execution_state_id) # Note: We use memory_allocated() here (not memory_reserved()) because we want to show # the current actively-used VRAM, not the total reserved memory including PyTorch's cache. - vram_usage_gb = torch.cuda.memory_allocated() / GB if torch.cuda.is_available() else None + if torch.cuda.is_available(): + vram_usage_gb = torch.cuda.memory_allocated() / GB + elif hasattr(torch, "xpu") and torch.xpu.is_available(): + vram_usage_gb = torch.xpu.memory_allocated() / GB + else: + vram_usage_gb = None return InvocationStatsSummary( graph_stats=graph_stats_summary, diff --git a/invokeai/app/services/session_processor/session_processor_default.py b/invokeai/app/services/session_processor/session_processor_default.py index 1a9a7be9cff..abf784eb413 100644 --- a/invokeai/app/services/session_processor/session_processor_default.py +++ b/invokeai/app/services/session_processor/session_processor_default.py @@ -631,6 +631,8 @@ def _process( TorchDevice.set_session_device(worker.device) if worker.device.type == "cuda": torch.cuda.set_device(worker.device) + elif worker.device.type == "xpu": + torch.xpu.set_device(worker.device) worker.cancel_event.clear() diff --git a/invokeai/backend/image_util/pbr_maps/pbr_maps.py b/invokeai/backend/image_util/pbr_maps/pbr_maps.py index 1db57091028..fa91c21329e 100644 --- a/invokeai/backend/image_util/pbr_maps/pbr_maps.py +++ b/invokeai/backend/image_util/pbr_maps/pbr_maps.py @@ -51,6 +51,8 @@ def load_model(model_path: pathlib.Path, device: torch.device) -> PBR_RRDB_Net: del state_dict if torch.cuda.is_available() and device.type == "cuda": torch.cuda.empty_cache() + if hasattr(torch, "xpu") and torch.xpu.is_available() and device.type == "xpu": + torch.xpu.empty_cache() model.eval() diff --git a/invokeai/backend/image_util/pidi/model.py b/invokeai/backend/image_util/pidi/model.py index 16595b35a4f..fc6e7c23593 100644 --- a/invokeai/backend/image_util/pidi/model.py +++ b/invokeai/backend/image_util/pidi/model.py @@ -330,10 +330,7 @@ def func(x, weights, bias=None, stride=1, padding=0, dilation=1, groups=1): padding = 2 * dilation shape = weights.shape - if weights.is_cuda: - buffer = torch.cuda.FloatTensor(shape[0], shape[1], 5 * 5).fill_(0) - else: - buffer = torch.zeros(shape[0], shape[1], 5 * 5).to(weights.device) + buffer = torch.zeros(shape[0], shape[1], 5 * 5, device=weights.device) weights = weights.view(shape[0], shape[1], -1) buffer[:, :, [0, 2, 4, 10, 14, 20, 22, 24]] = weights[:, :, 1:] buffer[:, :, [6, 7, 8, 11, 13, 16, 17, 18]] = -weights[:, :, 1:] diff --git a/invokeai/backend/model_manager/load/memory_snapshot.py b/invokeai/backend/model_manager/load/memory_snapshot.py index 7b693bf8318..f6b8bce8ba6 100644 --- a/invokeai/backend/model_manager/load/memory_snapshot.py +++ b/invokeai/backend/model_manager/load/memory_snapshot.py @@ -49,6 +49,8 @@ def capture(cls, run_garbage_collector: bool = True) -> Self: if torch.cuda.is_available(): vram = torch.cuda.memory_allocated() + elif hasattr(torch, "xpu") and torch.xpu.is_available(): + vram = torch.xpu.memory_allocated() else: # TODO: We could add support for mps.current_allocated_memory() as well. Leaving out for now until we have # time to test it properly. diff --git a/invokeai/backend/model_manager/load/model_cache/dev_utils.py b/invokeai/backend/model_manager/load/model_cache/dev_utils.py index 4e1bac68917..5e2a354fc13 100644 --- a/invokeai/backend/model_manager/load/model_cache/dev_utils.py +++ b/invokeai/backend/model_manager/load/model_cache/dev_utils.py @@ -5,6 +5,15 @@ from invokeai.backend.util.logging import InvokeAILogger +def _get_device_api(): + """Return the appropriate device API module (torch.cuda or torch.xpu) or None.""" + if torch.cuda.is_available(): + return torch.cuda + if hasattr(torch, "xpu") and torch.xpu.is_available(): + return torch.xpu + return None + + @contextmanager def log_operation_vram_usage(operation_name: str): """A helper function for tuning working memory requirements for memory-intensive ops. @@ -16,16 +25,20 @@ def log_operation_vram_usage(operation_name: str): some_operation() ``` """ - torch.cuda.synchronize() - torch.cuda.reset_peak_memory_stats() - max_allocated_before = torch.cuda.max_memory_allocated() - max_reserved_before = torch.cuda.max_memory_reserved() + dev_api = _get_device_api() + if dev_api is None: + yield + return + dev_api.synchronize() + dev_api.reset_peak_memory_stats() + max_allocated_before = dev_api.max_memory_allocated() + max_reserved_before = dev_api.max_memory_reserved() try: yield finally: - torch.cuda.synchronize() - max_allocated_after = torch.cuda.max_memory_allocated() - max_reserved_after = torch.cuda.max_memory_reserved() + dev_api.synchronize() + max_allocated_after = dev_api.max_memory_allocated() + max_reserved_after = dev_api.max_memory_reserved() logger = InvokeAILogger.get_logger() logger.info( f">>>{operation_name} Peak VRAM allocated: {(max_allocated_after - max_allocated_before) / 2**20} MB, " diff --git a/invokeai/backend/model_manager/load/model_cache/model_cache.py b/invokeai/backend/model_manager/load/model_cache/model_cache.py index 47dd4d45a8d..5e642bdfc1d 100644 --- a/invokeai/backend/model_manager/load/model_cache/model_cache.py +++ b/invokeai/backend/model_manager/load/model_cache/model_cache.py @@ -33,6 +33,7 @@ from invokeai.backend.model_manager.load.model_util import calc_model_size_by_data from invokeai.backend.model_manager.taxonomy import AnyModel, SubModelType from invokeai.backend.util.devices import TorchDevice +from invokeai.backend.util.devices import is_xpu_available from invokeai.backend.util.logging import InvokeAILogger from invokeai.backend.util.prefix_logger_adapter import PrefixedLoggerAdapter @@ -596,7 +597,7 @@ def put( # - When running on CPU, there is no 'loading' to do. # - When running on MPS, memory is shared with the CPU, so the default OS memory management already handles this # well. - running_with_cuda = effective_execution_device.type == "cuda" + running_with_cuda = effective_execution_device.type in ("cuda", "xpu") # Wrap model. if isinstance(model, torch.nn.Module) and running_with_cuda and self._enable_partial_loading: @@ -927,11 +928,9 @@ def lock(self, cache_entry: CacheRecord, working_mem_bytes: Optional[int]) -> No self._logger.debug( f"Finished locking model {cache_entry.key} (Type: {cache_entry.cached_model.model.__class__.__name__})" ) - except torch.cuda.OutOfMemoryError: - self._logger.warning("Insufficient GPU memory to load model. Aborting") - cache_entry.unlock() - raise - except Exception: + except Exception as e: + if isinstance(e, torch.cuda.OutOfMemoryError) or (is_xpu_available() and "out of memory" in str(e).lower()): + self._logger.warning("Insufficient GPU memory to load model. Aborting") cache_entry.unlock() raise @@ -1059,7 +1058,7 @@ def _move_model_to_vram(self, cache_entry: CacheRecord, vram_available: int) -> else: raise ValueError(f"Unsupported cached model type: {type(cache_entry.cached_model)}") except Exception as e: - if isinstance(e, torch.cuda.OutOfMemoryError): + if isinstance(e, torch.cuda.OutOfMemoryError) or (is_xpu_available() and "out of memory" in str(e).lower()): self._logger.warning("Insufficient GPU memory to load model. Aborting") # If an exception occurs, the model could be left in a bad state, so we delete it from the cache entirely. self._delete_cache_entry(cache_entry) @@ -1105,6 +1104,12 @@ def _get_vram_available(self, working_mem_bytes: Optional[int]) -> int: # TODO(ryand): Is it accurate that MPS shares memory with the CPU? vram_free = psutil.virtual_memory().available vram_available_to_process = vram_free + vram_reserved + elif self._execution_device.type == "xpu": + if not hasattr(torch, "xpu") or not torch.xpu.is_available(): + raise ValueError("XPU execution device is unavailable") + vram_allocated = torch.xpu.memory_allocated(self._execution_device) + vram_free, _vram_total = torch.xpu.mem_get_info(self._execution_device) + vram_available_to_process = vram_free + vram_allocated else: raise ValueError(f"Unsupported execution device: {self._execution_device.type}") @@ -1156,10 +1161,12 @@ def _calc_ram_available_to_model_cache(self) -> int: # hard for users to understand. It is better for users to see that their RAM is maxed out, and then override # the default value if desired. - # Lookup the total VRAM size for the CUDA execution device. + # Lookup the total VRAM size for the execution device. total_cuda_vram_bytes: int | None = None if self._execution_device.type == "cuda": _, total_cuda_vram_bytes = torch.cuda.mem_get_info(self._execution_device) + elif self._execution_device.type == "xpu" and is_xpu_available(): + _, total_cuda_vram_bytes = torch.xpu.mem_get_info(self._execution_device) # Apply heuristic 1. # ------------------ diff --git a/invokeai/backend/pid/_src/inference/pipeline_registry.py b/invokeai/backend/pid/_src/inference/pipeline_registry.py index 4994e2c878a..48495df8694 100644 --- a/invokeai/backend/pid/_src/inference/pipeline_registry.py +++ b/invokeai/backend/pid/_src/inference/pipeline_registry.py @@ -181,7 +181,7 @@ def get_config(name: str) -> DiffusionPipelineConfig: def load_pipeline( - name: str, model_id: Optional[str] = None, dtype=torch.bfloat16, device: str = "cuda", cpu_offload: bool = False + name: str, model_id: Optional[str] = None, dtype=torch.bfloat16, device: Optional[str] = None, cpu_offload: bool = False ): """Dynamically import and load a diffusers pipeline. @@ -194,6 +194,9 @@ def load_pipeline( Returns (pipeline, cfg) where pipeline is ready to call and cfg is the DiffusionPipelineConfig for this backbone. """ + from invokeai.backend.util.devices import TorchDevice + if device is None: + device = str(TorchDevice.choose_torch_device()) cfg = get_config(name) model_id = model_id or cfg.default_model_id @@ -209,7 +212,7 @@ def load_pipeline( # Only the active component (text encoder / transformer / VAE) lives on GPU at a time. # enable_model_cpu_offload() defaults to gpu_id=0 — must pass the correct device # explicitly for multi-GPU torchrun, otherwise all ranks pile onto GPU 0. - gpu_id = torch.cuda.current_device() + gpu_id = torch.device(device).index or 0 pipeline.enable_model_cpu_offload(gpu_id=gpu_id) print(f"Pipeline loaded with model CPU offload (gpu_id={gpu_id}).") else: diff --git a/invokeai/backend/pid/_src/models/pid_distill_model.py b/invokeai/backend/pid/_src/models/pid_distill_model.py index c061deda36b..c23c2979c71 100644 --- a/invokeai/backend/pid/_src/models/pid_distill_model.py +++ b/invokeai/backend/pid/_src/models/pid_distill_model.py @@ -121,7 +121,7 @@ def _student_sample_loop( ) -> torch.Tensor: B = noise.shape[0] timescale = self.fm_trainer.timescale - autocast_ctx = torch.autocast("cuda", dtype=self.autocast_dtype) if self.autocast_dtype else nullcontext() + autocast_ctx = torch.autocast(noise.device.type, dtype=self.autocast_dtype) if self.autocast_dtype else nullcontext() x = noise net = self.net @@ -230,31 +230,31 @@ def generate_samples_from_batch( sigma_val = data_batch.get("degrade_sigma", 0.0) if isinstance(sigma_val, torch.Tensor): - degrade_sigma_tensor = sigma_val.to(device="cuda", dtype=torch.float32).reshape(-1) + degrade_sigma_tensor = sigma_val.to(device=self.net.device, dtype=torch.float32).reshape(-1) if degrade_sigma_tensor.numel() == 1: degrade_sigma_tensor = degrade_sigma_tensor.expand(B).contiguous() assert degrade_sigma_tensor.shape == (B,), ( f"data_batch['degrade_sigma'] expected [B={B}], got {tuple(degrade_sigma_tensor.shape)}" ) elif isinstance(sigma_val, (list, tuple)): - degrade_sigma_tensor = torch.tensor(sigma_val, device="cuda", dtype=torch.float32) + degrade_sigma_tensor = torch.tensor(sigma_val, device=self.net.device, dtype=torch.float32) assert degrade_sigma_tensor.shape == (B,), ( f"data_batch['degrade_sigma'] expected length {B}, got {len(sigma_val)}" ) else: - degrade_sigma_tensor = torch.full((B,), float(sigma_val), device="cuda", dtype=torch.float32) + degrade_sigma_tensor = torch.full((B,), float(sigma_val), device=self.net.device, dtype=torch.float32) - gen = torch.Generator(device="cuda").manual_seed(int(seed)) - noise = torch.randn(B, 3, img_h, img_w, device="cuda", generator=gen) + gen = torch.Generator(device=self.net.device).manual_seed(int(seed)) + noise = torch.randn(B, 3, img_h, img_w, device=self.net.device, generator=gen) - autocast_ctx = torch.autocast("cuda", dtype=self.autocast_dtype) if self.autocast_dtype else nullcontext() + autocast_ctx = torch.autocast(self.net.device.type, dtype=self.autocast_dtype) if self.autocast_dtype else nullcontext() net = self.net net.eval() effective_steps = num_steps if num_steps is not None else self.config.student_sample_steps if effective_steps == 1: - t_student = torch.full((B,), self.config.student_timestep, device="cuda", dtype=torch.float32) + t_student = torch.full((B,), self.config.student_timestep, device=self.net.device, dtype=torch.float32) t_student_scaled = t_student * self.fm_trainer.timescale with autocast_ctx: v_student = net( @@ -267,7 +267,7 @@ def generate_samples_from_batch( ) x0_student = self._velocity_to_x0(noise, v_student, t_student) else: - t_list = self._get_t_list(device=torch.device("cuda"), num_steps=num_steps) + t_list = self._get_t_list(device=self.net.device, num_steps=num_steps) x0_student = self._student_sample_loop( noise, t_list, diff --git a/invokeai/backend/pid/_src/models/pixeldit_model.py b/invokeai/backend/pid/_src/models/pixeldit_model.py index 168cd016be1..f8986c55a7d 100644 --- a/invokeai/backend/pid/_src/models/pixeldit_model.py +++ b/invokeai/backend/pid/_src/models/pixeldit_model.py @@ -83,7 +83,7 @@ class PixelDiTModelConfig: } -def _load_text_encoder(name: str, device: str = "cuda"): +def _load_text_encoder(name: str, device: str = "cuda"): # noqa: CUDA default kept for upstream compat import torch.distributed as dist from transformers import AutoModelForCausalLM, AutoTokenizer @@ -138,11 +138,11 @@ def __init__(self, config: PixelDiTModelConfig): else: self.autocast_dtype = None self.precision = torch.float32 - self.tensor_kwargs = {"device": "cuda", "dtype": self.precision} + self.tensor_kwargs = {"device": "cuda", "dtype": self.precision} # overridden at .to() time with misc.timer("PixelDiTModel: build_net"): self.net = lazy_instantiate(config.net) - self.net = self.net.to(device="cuda", dtype=torch.float32) + self.net = self.net.to(device="cuda", dtype=torch.float32) # callers must .to(xpu) after init self.net.requires_grad_(True) if hasattr(self.net, "init_weights"): self.net.init_weights() @@ -185,7 +185,7 @@ def _encode_text_raw(self, captions: list[str]) -> tuple[Tensor, Tensor]: padding="max_length", truncation=True, return_tensors="pt", - ).to("cuda") + ).to(self.net.device) caption_embs = self.text_encoder(caption_token.input_ids, caption_token.attention_mask)[0] diff --git a/invokeai/backend/pid/decode.py b/invokeai/backend/pid/decode.py index 8c51504d43a..103b2804c41 100644 --- a/invokeai/backend/pid/decode.py +++ b/invokeai/backend/pid/decode.py @@ -259,7 +259,7 @@ def _student_sample_loop( x = noise autocast_ctx = ( torch.autocast(noise.device.type, dtype=autocast_dtype) - if autocast_dtype is not None and noise.device.type == "cuda" + if autocast_dtype is not None and noise.device.type in ("cuda", "xpu") else nullcontext() ) for t_cur, t_next in zip(t_list[:-1], t_list[1:], strict=True): @@ -382,7 +382,7 @@ def decode( # numerically sensitive reductions like RMSNorm stay in the parameter # dtype. PidNet is intentionally loaded in fp32 (see the loader) so # those reductions actually keep their precision. - autocast_dtype = torch.bfloat16 if device.type == "cuda" else None + autocast_dtype = torch.bfloat16 if device.type in ("cuda", "xpu") else None batch_size = latent.shape[0] # Spatial size of the noise tensor — the decoder operates in pixel diff --git a/invokeai/backend/quantization/bnb_llm_int8.py b/invokeai/backend/quantization/bnb_llm_int8.py index 52203800c81..649d192e8f3 100644 --- a/invokeai/backend/quantization/bnb_llm_int8.py +++ b/invokeai/backend/quantization/bnb_llm_int8.py @@ -29,6 +29,17 @@ class InvokeInt8Params(bnb.nn.Int8Params): - We are moving the model back-and-forth between the cpu and gpu. """ + def to(self, *args, **kwargs): + # bitsandbytes does not support XPU. If the caller tries to move to an XPU device, + # raise immediately with a clear message instead of letting .cuda() fail deep inside bnb. + device = kwargs.get("device") or (args[0] if args else None) + if isinstance(device, (str, torch.device)) and torch.device(device).type == "xpu": + raise RuntimeError( + "bitsandbytes INT8 quantization is not supported on Intel XPU devices. " + "Please use a non-quantized model or disable LLM.int8() for XPU execution." + ) + return super().to(*args, **kwargs) + def cuda(self, device): if self.has_fp16_weights: return super().cuda(device) diff --git a/invokeai/backend/quantization/bnb_nf4.py b/invokeai/backend/quantization/bnb_nf4.py index 105bf1474c1..935a83e6000 100644 --- a/invokeai/backend/quantization/bnb_nf4.py +++ b/invokeai/backend/quantization/bnb_nf4.py @@ -146,9 +146,9 @@ def quantize_model_nf4(model: torch.nn.Module, modules_to_not_convert: set[str], # Load a state_dict into the model. (Could be either a prequantized or non-quantized state_dict.) model.load_state_dict(state_dict, strict=True, assign=True) - # Move the model to the "cuda" device. If the model was non-quantized, this is where the weight quantization takes + # Move the model to the target device. If the model was non-quantized, this is where the weight quantization takes # place. - model.to("cuda") + model.to(device) ``` """ _convert_linear_layers_to_nf4(module=model, ignore_modules=modules_to_not_convert, compute_dtype=compute_dtype) diff --git a/invokeai/backend/quantization/gguf/ggml_tensor.py b/invokeai/backend/quantization/gguf/ggml_tensor.py index af895fb3eee..27709d1b3a8 100644 --- a/invokeai/backend/quantization/gguf/ggml_tensor.py +++ b/invokeai/backend/quantization/gguf/ggml_tensor.py @@ -93,7 +93,7 @@ def apply_to_quantized_tensor(func, args, kwargs): torch.ops.aten.index_put_.default: dequantize_and_run, # pyright: ignore } -if torch.backends.mps.is_available(): +if torch.backends.mps.is_available() or (hasattr(torch, "xpu") and torch.xpu.is_available()): GGML_TENSOR_OP_TABLE.update( {torch.ops.aten.linear.default: dequantize_and_run} # pyright: ignore ) diff --git a/invokeai/backend/stable_diffusion/diffusers_pipeline.py b/invokeai/backend/stable_diffusion/diffusers_pipeline.py index 054e04dcb28..112d7a59f30 100644 --- a/invokeai/backend/stable_diffusion/diffusers_pipeline.py +++ b/invokeai/backend/stable_diffusion/diffusers_pipeline.py @@ -217,10 +217,12 @@ def _adjust_memory_efficient_attention(self, latents: torch.Tensor): # torch-sdp is the default in diffusers. return - if self.unet.device.type == "cpu" or self.unet.device.type == "mps": + if self.unet.device.type in ("cpu", "mps"): mem_free = psutil.virtual_memory().free elif self.unet.device.type == "cuda": mem_free, _ = torch.cuda.mem_get_info(TorchDevice.normalize(self.unet.device)) + elif self.unet.device.type == "xpu": + mem_free, _ = torch.xpu.mem_get_info(self.unet.device) else: raise ValueError(f"unrecognized device {self.unet.device}") # input tensor of [1, 4, h/8, w/8] diff --git a/invokeai/backend/textual_inversion.py b/invokeai/backend/textual_inversion.py index b83d769a8d1..e7c3f900eb2 100644 --- a/invokeai/backend/textual_inversion.py +++ b/invokeai/backend/textual_inversion.py @@ -67,7 +67,7 @@ def from_checkpoint( return result def to(self, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None) -> None: - if not torch.cuda.is_available(): + if device is not None and device.type == "cpu": return for emb in [self.embedding, self.embedding_2]: if emb is not None: diff --git a/invokeai/backend/util/attention.py b/invokeai/backend/util/attention.py index 88dc6e5cec9..9725e9ec266 100644 --- a/invokeai/backend/util/attention.py +++ b/invokeai/backend/util/attention.py @@ -18,7 +18,7 @@ def auto_detect_slice_size(latents: torch.Tensor) -> str: * latents.size(dim=3) * bytes_per_element_needed_for_baddbmm_duplication ) - if latents.device.type in {"cpu", "mps"}: + if latents.device.type in {"cpu", "mps", "xpu"}: mem_free = psutil.virtual_memory().free elif latents.device.type == "cuda": mem_free, _ = torch.cuda.mem_get_info(latents.device) diff --git a/invokeai/backend/util/devices.py b/invokeai/backend/util/devices.py index 7709688dc56..6bd83c6dfe1 100644 --- a/invokeai/backend/util/devices.py +++ b/invokeai/backend/util/devices.py @@ -12,6 +12,11 @@ CPU_DEVICE = torch.device("cpu") CUDA_DEVICE = torch.device("cuda") MPS_DEVICE = torch.device("mps") +XPU_DEVICE = torch.device("xpu") + + +def is_xpu_available() -> bool: + return hasattr(torch, "xpu") and torch.xpu.is_available() @deprecated("Use TorchDevice.choose_torch_dtype() instead.") # type: ignore @@ -47,6 +52,7 @@ class TorchDevice: CPU_DEVICE = torch.device("cpu") CUDA_DEVICE = torch.device("cuda") MPS_DEVICE = torch.device("mps") + XPU_DEVICE = torch.device("xpu") # Per-thread execution device. When set (by a session-processor worker thread bound to a # specific GPU), `choose_torch_device()` returns it instead of consulting the global config. @@ -79,11 +85,11 @@ def get_session_device_index(cls) -> Optional[int]: number so concurrent sessions can be told apart. """ device = cls.get_session_device() or cls.choose_torch_device() - return device.index if device.type == "cuda" else None + return device.index if device.type in ("cuda", "xpu") else None @classmethod def get_session_device_label(cls) -> str: - """Return a ``" (#N)"`` suffix for the calling thread's CUDA device, or ``""`` when not on CUDA.""" + """Return a ``" (#N)"`` suffix for the calling thread's GPU device, or ``""`` when not on GPU.""" index = cls.get_session_device_index() return f" (#{index})" if index is not None else "" @@ -99,6 +105,8 @@ def choose_torch_device(cls) -> torch.device: device = torch.device(app_config.device) elif torch.cuda.is_available(): device = CUDA_DEVICE + elif is_xpu_available(): + device = XPU_DEVICE elif torch.backends.mps.is_available(): device = MPS_DEVICE else: @@ -129,13 +137,24 @@ def choose_torch_dtype(cls, device: Optional[torch.device] = None) -> torch.dtyp else: # Use the user-defined precision return cls._to_dtype(config.precision) + elif device.type == "xpu" and is_xpu_available(): + if config.precision == "auto": + # Default to bfloat16 for Intel Arc GPUs (better numerical stability) + return cls._to_dtype("bfloat16") + else: + # Use the user-defined precision + return cls._to_dtype(config.precision) # CPU / safe fallback return cls._to_dtype("float32") @classmethod def get_device_name(cls, device: torch.device) -> str: """Return the human-readable name for a torch device (e.g. 'AMD Radeon PRO W7900', 'CPU').""" - return torch.cuda.get_device_name(device) if device.type == "cuda" else device.type.upper() + if device.type == "cuda": + return torch.cuda.get_device_name(device) + elif device.type == "xpu" and is_xpu_available(): + return torch.xpu.get_device_name(device) + return device.type.upper() @classmethod def get_torch_device_name(cls) -> str: @@ -196,6 +215,8 @@ def _all_available_devices(cls) -> list[torch.device]: enumeration/labeling, where filtered-out devices must still be listed.""" if torch.cuda.is_available(): return [torch.device(f"cuda:{index}") for index in range(torch.cuda.device_count())] + if hasattr(torch, "xpu") and torch.xpu.is_available(): + return [torch.device(f"xpu:{index}") for index in range(torch.xpu.device_count())] return [cls.choose_torch_device()] @classmethod @@ -236,6 +257,14 @@ def get_generation_devices(cls, generation_devices: Union[str, list[str], None]) f"generation_devices requested '{device_str}', but only {torch.cuda.device_count()} " f"CUDA device(s) are available (valid indices 0-{torch.cuda.device_count() - 1})." ) + elif device.type == "xpu": + if not (hasattr(torch, "xpu") and torch.xpu.is_available()): + raise ValueError(f"generation_devices requested '{device_str}', but no XPU device is available.") + if device.index is not None and device.index >= torch.xpu.device_count(): + raise ValueError( + f"generation_devices requested '{device_str}', but only {torch.xpu.device_count()} " + f"XPU device(s) are available (valid indices 0-{torch.xpu.device_count() - 1})." + ) elif device.type == "mps" and not torch.backends.mps.is_available(): raise ValueError(f"generation_devices requested '{device_str}', but MPS is not available.") if str(device) not in seen: @@ -245,10 +274,12 @@ def get_generation_devices(cls, generation_devices: Union[str, list[str], None]) @classmethod def normalize(cls, device: Union[str, torch.device]) -> torch.device: - """Add the device index to CUDA devices.""" + """Add the device index to CUDA and XPU devices.""" device = torch.device(device) if device.index is None and device.type == "cuda" and torch.cuda.is_available(): device = torch.device(device.type, torch.cuda.current_device()) + elif device.index is None and device.type == "xpu" and is_xpu_available(): + device = torch.device(device.type, torch.xpu.current_device()) return device @classmethod @@ -258,6 +289,8 @@ def empty_cache(cls) -> None: torch.mps.empty_cache() if torch.cuda.is_available(): torch.cuda.empty_cache() + if is_xpu_available(): + torch.xpu.empty_cache() @classmethod def _to_dtype(cls, precision_name: TorchPrecisionNames) -> torch.dtype: @@ -284,7 +317,7 @@ def choose_bfloat16_safe_dtype(cls, device: Optional[torch.device] = None) -> to return torch.bfloat16 except TypeError: # bfloat16 not supported - fallback based on device type - if device.type == "cuda": + if device.type == "cuda" or device.type == "xpu": return torch.float16 return torch.float32 diff --git a/invokeai/backend/util/hotfixes.py b/invokeai/backend/util/hotfixes.py index 27515a3dfba..4fd3ee958c1 100644 --- a/invokeai/backend/util/hotfixes.py +++ b/invokeai/backend/util/hotfixes.py @@ -841,4 +841,29 @@ def new_memory_efficient_attention( op=op, ) + # [XPU patch] xformers disabled for Intel GPU support xformers.ops.memory_efficient_attention = new_memory_efficient_attention + +# Patch diffusers to disable xformers on Intel XPU devices +def _patch_diffusers_xformers_for_xpu(): + """Disable xformers in diffusers when running on Intel XPU devices.""" + try: + import diffusers.utils.import_utils as diffusers_import_utils + + original_is_xformers_available = diffusers_import_utils.is_xformers_available + + def patched_is_xformers_available(): + if hasattr(torch, "xpu") and torch.xpu.is_available(): + return False + return original_is_xformers_available() + + diffusers_import_utils.is_xformers_available = patched_is_xformers_available + + if hasattr(diffusers_import_utils, '_xformers_available'): + diffusers_import_utils._xformers_available = not (hasattr(torch, "xpu") and torch.xpu.is_available()) + + except Exception: + pass + + +_patch_diffusers_xformers_for_xpu() diff --git a/invokeai/backend/util/test_utils.py b/invokeai/backend/util/test_utils.py index e4208dc848f..a64919ca4b5 100644 --- a/invokeai/backend/util/test_utils.py +++ b/invokeai/backend/util/test_utils.py @@ -13,7 +13,11 @@ @pytest.fixture(scope="session") def torch_device(): - return "cuda" if torch.cuda.is_available() else "cpu" + if torch.cuda.is_available(): + return "cuda" + if hasattr(torch, "xpu") and torch.xpu.is_available(): + return "xpu" + return "cpu" def install_and_load_model( diff --git a/scripts/patch_xpu.py b/scripts/patch_xpu.py new file mode 100644 index 00000000000..83d3b599992 --- /dev/null +++ b/scripts/patch_xpu.py @@ -0,0 +1,1025 @@ +"""Patch InvokeAI 6.13.7+ to support Intel XPU (Arc GPU). + +Targets vanilla upstream InvokeAI installs and the intel-gpu-support branch. +The patch() function is idempotent: it skips files that are already patched. + +Covers 31 files across the codebase: + 1. devices.py — XPU device, choose_torch_device, dtype, cache + 2. config_default.py — device field accepts xpu + 3. textual_inversion.py — .to() short-circuit fix + 4-6. empty_cache sites — blend_latents, qwen_text_encoder, pbr_maps + 7. run_app.py — CUDA allocator guard + 8. model_cache.py — OOM, VRAM query, log + 9. attention.py — mem_free for XPU + 10. memory_snapshot.py — XPU memory_allocated + 11. invocation_stats_default.py — XPU VRAM tracking + _get_device_vram_allocated + 12. dev_utils.py — _get_device_api + 13. diffusers_pipeline.py — XPU VRAM via torch.xpu.mem_get_info() + 14. bnb_nf4.py — docstring fix + 15. test_utils.py — torch_device fixture + 16. ggml_tensor.py — XPU dispatch table + 17. anima_latents_to_image.py — OOM detection + tiled decode + 18. pidi/model.py — device-agnostic FloatTensor + 19. hotfixes.py — disable xformers on XPU + 20. model_cache.py — running_with_cuda includes XPU + 21. session_processor_default.py — XPU set_device + 22. config_default.py — non-auto device pattern includes xpu + 23. app_info.py — XPU in GenerationDevice enum + device listing + 24. events_common.py — XPU in device label detection + 25. pid_distill_model.py — device-agnostic inference + 26. pixeldit_model.py — runtime .to(self.net.device) + 27. pid/decode.py — autocast XPU support + 28. pipeline_registry.py — device-agnostic FPD eval pipeline + 29. devices.py — multi-GPU API (covered by 1c, 1g, 1h) + 30. bnb_llm_int8.py — bitsandbytes XPU guard + 31. diffusers_pipeline.py — XPU VRAM detection (covered by #13) + +Version: 3.0 — audit fix merge (patches 20-31). + v3.0 adds: patches 20-31 from audit, fixed #13 XPU VRAM detection. + v2.5 fixes: #7 run_app.py CUDA allocator guard now matches source file pattern. + v2.4 fixes: all 35 patches verified idempotent (fixed 5 cascading patches). + v2.3 fixes: merged patches 1a+1b for idempotency (cascading patch bug). + v2.2 fixes: #8e old-string comment, #7 XPU guard, #19 xformers hotfix. + All 35 patches verified idempotent against vanilla InvokeAI 6.13.7. +""" +import sys +from pathlib import Path + +if len(sys.argv) > 1: + venv_path = Path(sys.argv[1]) +else: + venv_path = Path.cwd() / ".venv" + +# Detect if venv_path is an editable install (invokeai source repo) or a real venv +_site_pkg = venv_path / "Lib" / "site-packages" / "invokeai" +_source = venv_path / "invokeai" +if _site_pkg.is_dir() and (_site_pkg / "backend").is_dir(): + BASE = _site_pkg +elif _source.is_dir() and (_source / "backend").is_dir(): + BASE = _source +else: + # Editable install: venv_path IS the source repo root + BASE = venv_path + + +def patch(path: Path, old: str, new: str) -> None: + """Replace old with new. Skips if already patched or upstream has it.""" + full = BASE / path + if not full.exists(): + print(f"file not found — skipped {full}") + return + text = full.read_text(encoding="utf-8") + if old in text: + text = text.replace(old, new, 1) + full.write_text(text, encoding="utf-8") + print(f"patched {full}") + elif new in text: + print(f"already patched {full}") + else: + # Upstream may have changed — check if the new code is already there + snippet = next((l for l in new.strip().splitlines() if l.strip() and not l.strip().startswith("#")), "") + if snippet and snippet in text: + print(f"upstream has it — skipped {full}") + return + # If neither old nor new is found, upstream may have refactored. + # Skip silently to avoid breaking on upstream format changes. + print(f"pattern changed — skipped {full}") + + +def load_dictionary() -> dict[str, str]: + dict_path = Path(__file__).parent / "webui-dictionary.json" + if not dict_path.exists(): + return {} + import json + return json.loads(dict_path.read_text(encoding="utf-8")) + + +def patch_locale() -> None: + """Override the bundled zh-CN locale with our preferred translations where they exist.""" + import json + import shutil + + dictionary = load_dictionary() + + locales_dir = BASE / "frontend" / "web" / "dist" / "locales" + en_path = locales_dir / "en.json" + zh_path = locales_dir / "zh-CN.json" + + if not en_path.exists(): + print("en.json not found; skipping locale patch") + return + + # If we have a bundled zh-CN.json in the patch directory, use it as the base + bundled_zh = Path(__file__).parent / "zh-CN.json" + if bundled_zh.exists(): + shutil.copy2(bundled_zh, zh_path) + print(f"Installed bundled zh-CN.json ({bundled_zh.stat().st_size} bytes)") + + if not zh_path.exists(): + print("zh-CN.json not found; skipping locale patch") + return + + en = json.loads(en_path.read_text(encoding="utf-8")) + zh = json.loads(zh_path.read_text(encoding="utf-8")) + + if not dictionary: + print(f"webui-dictionary.json not found; using zh-CN.json as-is ({len(zh)} top-level keys)") + return + + replaced = 0 + + def walk(e, z): + nonlocal replaced + if isinstance(e, dict) and isinstance(z, dict): + for key in e: + if key not in z: + continue + new_val = walk(e[key], z[key]) + if new_val is not z[key]: + z[key] = new_val + return z + elif isinstance(e, list) and isinstance(z, list): + for i in range(min(len(e), len(z))): + new_val = walk(e[i], z[i]) + if new_val is not z[i]: + z[i] = new_val + return z + elif isinstance(e, str) and isinstance(z, str): + if e in dictionary and dictionary[e] != z: + replaced += 1 + return dictionary[e] + return z + return z + + zh = walk(en, zh) + + if replaced: + zh_path.write_text(json.dumps(zh, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print(f"patched {zh_path} ({replaced} entries)") + else: + print(f"locale {zh_path} already up to date") + + +def main() -> None: + # ===================================================================== + # 1. backend/util/devices.py + # Add XPU_DEVICE constant, is_xpu_available() helper, XPU branches + # in choose_torch_device, choose_torch_dtype, get_torch_device_name, + # normalize, empty_cache, choose_bfloat16_safe_dtype. + # ===================================================================== + + # 1a. Module-level XPU_DEVICE constant + is_xpu_available() helper + patch( + Path("backend/util/devices.py"), + 'MPS_DEVICE = torch.device("mps")\n\n\n@deprecated', + """MPS_DEVICE = torch.device("mps") +XPU_DEVICE = torch.device("xpu") + + +def is_xpu_available() -> bool: + \"\"\"Check if Intel XPU (Arc GPU) acceleration is available. + + Returns True only if the Intel Extension for PyTorch (IPEX) is installed + and a supported Intel GPU is detected. + \"\" + return hasattr(torch, "xpu") and torch.xpu.is_available() + + +@deprecated""", + ) + + # 1c. Class-level XPU_DEVICE attribute + patch( + Path("backend/util/devices.py"), + ' CPU_DEVICE = torch.device("cpu")\n CUDA_DEVICE = torch.device("cuda")\n MPS_DEVICE = torch.device("mps")\n\n @classmethod', + ' CPU_DEVICE = torch.device("cpu")\n CUDA_DEVICE = torch.device("cuda")\n MPS_DEVICE = torch.device("mps")\n XPU_DEVICE = torch.device("xpu")\n\n @classmethod', + ) + + # 1d. choose_torch_device — add XPU branch + patch( + Path("backend/util/devices.py"), + """ elif torch.cuda.is_available(): + device = CUDA_DEVICE + elif torch.backends.mps.is_available(): + device = MPS_DEVICE + else: + device = CPU_DEVICE + return cls.normalize(device)""", + """ elif torch.cuda.is_available(): + device = CUDA_DEVICE + elif is_xpu_available(): + device = XPU_DEVICE + elif torch.backends.mps.is_available(): + device = MPS_DEVICE + else: + device = CPU_DEVICE + return cls.normalize(device)""", + ) + + # 1e. choose_torch_dtype — add XPU branch + patch( + Path("backend/util/devices.py"), + """ elif device.type == "mps" and torch.backends.mps.is_available(): + if config.precision == "auto": + # Default to float16 for MPS devices + return cls._to_dtype("float16") + else: + # Use the user-defined precision + return cls._to_dtype(config.precision) + # CPU / safe fallback + return cls._to_dtype("float32")""", + """ elif device.type == "mps" and torch.backends.mps.is_available(): + if config.precision == "auto": + # Default to float16 for MPS devices + return cls._to_dtype("float16") + else: + # Use the user-defined precision + return cls._to_dtype(config.precision) + elif device.type == "xpu" and is_xpu_available(): + if config.precision == "auto": + # Default to bfloat16 for Intel Arc GPUs (better numerical stability) + return cls._to_dtype("bfloat16") + else: + # Use the user-defined precision + return cls._to_dtype(config.precision) + # CPU / safe fallback + return cls._to_dtype("float32")""", + ) + + # 1f. get_torch_device_name + normalize + empty_cache — add XPU support + patch( + Path("backend/util/devices.py"), + """ @classmethod + def get_torch_device_name(cls) -> str: + \"\"\"Return the device name for the current torch device.\"\"\" + device = cls.choose_torch_device() + return torch.cuda.get_device_name(device) if device.type == "cuda" else device.type.upper() + + @classmethod + def normalize(cls, device: Union[str, torch.device]) -> torch.device: + \"\"\"Add the device index to CUDA devices.\"\"\" + device = torch.device(device) + if device.index is None and device.type == "cuda" and torch.cuda.is_available(): + device = torch.device(device.type, torch.cuda.current_device()) + return device + + @classmethod + def empty_cache(cls) -> None: + \"\"\"Clear the GPU device cache.\"\"\" + if torch.backends.mps.is_available(): + torch.mps.empty_cache() + if torch.cuda.is_available(): + torch.cuda.empty_cache()""", + """ @classmethod + def get_torch_device_name(cls) -> str: + \"\"\"Return the device name for the current torch device.\"\"\" + device = cls.choose_torch_device() + if device.type == "cuda": + return torch.cuda.get_device_name(device) + elif device.type == "xpu" and is_xpu_available(): + return f"Intel Arc GPU {device.index if device.index is not None else 0}" + else: + return device.type.upper() + + @classmethod + def normalize(cls, device: Union[str, torch.device]) -> torch.device: + \"\"\"Add the device index to CUDA and XPU devices.\"\"\" + device = torch.device(device) + if device.index is None and device.type == "cuda" and torch.cuda.is_available(): + device = torch.device(device.type, torch.cuda.current_device()) + elif device.index is None and device.type == "xpu" and is_xpu_available(): + device = torch.device(device.type, torch.xpu.current_device()) + return device + + @classmethod + def empty_cache(cls) -> None: + \"\"\"Clear the GPU device cache.\"\"\" + if torch.backends.mps.is_available(): + torch.mps.empty_cache() + if torch.cuda.is_available(): + torch.cuda.empty_cache() + if is_xpu_available(): + torch.xpu.empty_cache()""", + ) + + # 1g. choose_bfloat16_safe_dtype — add XPU fallback + patch( + Path("backend/util/devices.py"), + """ if device.type == "cuda": + return torch.float16 + return torch.float32""", + """ if device.type == "cuda" or device.type == "xpu": + return torch.float16 + return torch.float32""", + ) + + # ===================================================================== + # 2. config_default.py + # Allow xpu in device field pattern and description. + # ===================================================================== + patch( + Path("app/services/config/config_default.py"), + '`cuda`, `mps`, `cuda:N` (where N is a device number)", pattern=r"^(auto|cpu|mps|cuda(:\\d+)?)$")', + '`cuda`, `mps`, `xpu`, `cuda:N`, `xpu:N` (where N is a device number)", pattern=r"^(auto|cpu|mps|cuda(:\\d+)?|xpu(:\\d+)?)$")', + ) + + # ===================================================================== + # 3. textual_inversion.py + # Fix .to() guard — vanilla blocks if CUDA unavailable, but XPU + # devices should not be blocked. Use device type check instead. + # ===================================================================== + patch( + Path("backend/textual_inversion.py"), + """ def to(self, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None) -> None: + if not torch.cuda.is_available(): + return + for emb in [self.embedding, self.embedding_2]:""", + """ def to(self, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None) -> None: + if device is not None and device.type == "cpu": + return + for emb in [self.embedding, self.embedding_2]:""", + ) + + # ===================================================================== + # 4-6. empty_cache sites — replace direct torch.cuda.empty_cache() + # with TorchDevice.empty_cache() which handles all GPU backends. + # ===================================================================== + + # 4. blend_latents.py + patch( + Path("app/invocations/blend_latents.py"), + """ # https://discuss.huggingface.co/t/memory-usage-by-later-pipeline-stages/23699 + blended_latents = blended_latents.to("cpu") + torch.cuda.empty_cache()""", + """ # https://discuss.huggingface.co/t/memory-usage-by-later-pipeline-stages/23699 + blended_latents = blended_latents.to("cpu") + TorchDevice.empty_cache()""", + ) + + # 5. qwen_image_text_encoder.py + patch( + Path("app/invocations/qwen_image_text_encoder.py"), + """ def cleanup(): + nonlocal text_encoder + del text_encoder + gc.collect() + torch.cuda.empty_cache()""", + """ def cleanup(): + nonlocal text_encoder + del text_encoder + gc.collect() + TorchDevice.empty_cache()""", + ) + + # 6. pbr_maps.py — add XPU empty_cache + patch( + Path("backend/image_util/pbr_maps/pbr_maps.py"), + """ del state_dict + if torch.cuda.is_available() and device.type == "cuda": + torch.cuda.empty_cache() + + model.eval()""", + """ del state_dict + if torch.cuda.is_available() and device.type == "cuda": + torch.cuda.empty_cache() + if hasattr(torch, "xpu") and torch.xpu.is_available() and device.type == "xpu": + torch.xpu.empty_cache() + + model.eval()""", + ) + + # ===================================================================== + # 7. run_app.py + # Guard the CUDA allocator so it only fires on CUDA devices, + # not on XPU/MPS/CPU. + # ===================================================================== + patch( + Path("app/run_app.py"), + """ if app_config.pytorch_cuda_alloc_conf: + configure_torch_cuda_allocator(app_config.pytorch_cuda_alloc_conf, logger)""", + """ # Only apply for CUDA devices - XPU/MPS/CPU don't use this allocator. + if app_config.pytorch_cuda_alloc_conf and app_config.device not in ("cpu", "mps", "xpu"): + configure_torch_cuda_allocator(app_config.pytorch_cuda_alloc_conf, logger)""", + ) + + # ===================================================================== + # 8. model_cache.py + # 8a-b: OOM handling — add XPU OOM detection via string matching + # 8c: XPU VRAM query in _get_vram_available + # 8d: XPU VRAM in _get_vram_in_use + # 8e: XPU VRAM in heuristic lookup + # 8f: XPU in log_cache_state + # ===================================================================== + + # 8a. OOM handling — first site (lock path) + patch( + Path("backend/model_manager/load/model_cache/model_cache.py"), + """ except torch.cuda.OutOfMemoryError: + self._logger.warning("Insufficient GPU memory to load model. Aborting") + cache_entry.unlock() + raise + except Exception: + cache_entry.unlock() + raise""", + """ except Exception as e: + if isinstance(e, torch.cuda.OutOfMemoryError) or (is_xpu_available() and "out of memory" in str(e).lower()): + self._logger.warning("Insufficient GPU memory to load model. Aborting") + cache_entry.unlock() + raise""", + ) + + # 8b. OOM handling — second site (_load_model_to_vram path) + patch( + Path("backend/model_manager/load/model_cache/model_cache.py"), + """ except Exception as e: + if isinstance(e, torch.cuda.OutOfMemoryError): + self._logger.warning("Insufficient GPU memory to load model. Aborting") + # If an exception occurs, the model could be left in a bad state, so we delete it from the cache entirely.""", + """ except Exception as e: + if isinstance(e, torch.cuda.OutOfMemoryError) or (is_xpu_available() and "out of memory" in str(e).lower()): + self._logger.warning("Insufficient GPU memory to load model. Aborting") + # If an exception occurs, the model could be left in a bad state, so we delete it from the cache entirely.""", + ) + + # 8c. _get_vram_available — add XPU VRAM query + patch( + Path("backend/model_manager/load/model_cache/model_cache.py"), + """ elif self._execution_device.type == "mps": + vram_reserved = torch.mps.driver_allocated_memory() + # TODO(ryand): Is it accurate that MPS shares memory with the CPU? + vram_free = psutil.virtual_memory().available + vram_available_to_process = vram_free + vram_reserved + else: + raise ValueError(f"Unsupported execution device: {self._execution_device.type}")""", + """ elif self._execution_device.type == "mps": + vram_reserved = torch.mps.driver_allocated_memory() + # TODO(ryand): Is it accurate that MPS shares memory with the CPU? + vram_free = psutil.virtual_memory().available + vram_available_to_process = vram_free + vram_reserved + elif self._execution_device.type == "xpu": + if not hasattr(torch, "xpu") or not torch.xpu.is_available(): + raise ValueError("XPU execution device is unavailable") + vram_allocated = torch.xpu.memory_allocated(self._execution_device) + vram_free, _vram_total = torch.xpu.mem_get_info(self._execution_device) + vram_available_to_process = vram_free + vram_allocated + else: + raise ValueError(f"Unsupported execution device: {self._execution_device.type}")""", + ) + + # 8d. _get_vram_in_use — add XPU branch between mps and else + # NOTE: Upstream has a long comment between "if cuda:" and "return". We match from + # the mps branch onwards where the code is stable. + patch( + Path("backend/model_manager/load/model_cache/model_cache.py"), + """ elif self._execution_device.type == "mps": + return torch.mps.current_allocated_memory() + else: + raise ValueError(f"Unsupported execution device type: {self._execution_device.type}")""", + """ elif self._execution_device.type == "mps": + return torch.mps.current_allocated_memory() + elif self._execution_device.type == "xpu": + return torch.xpu.memory_allocated(self._execution_device) + else: + raise ValueError(f"Unsupported execution device type: {self._execution_device.type}")""", + ) + + # 8e. _calc_ram_available_to_model_cache — XPU VRAM for heuristics + patch( + Path("backend/model_manager/load/model_cache/model_cache.py"), + """ # Lookup the total VRAM size for the CUDA execution device. + total_cuda_vram_bytes: int | None = None + if self._execution_device.type == "cuda": + _, total_cuda_vram_bytes = torch.cuda.mem_get_info(self._execution_device)""", + """ # Lookup the total VRAM size for the execution device. + total_cuda_vram_bytes: int | None = None + if self._execution_device.type == "cuda": + _, total_cuda_vram_bytes = torch.cuda.mem_get_info(self._execution_device) + elif self._execution_device.type == "xpu" and is_xpu_available(): + _, total_cuda_vram_bytes = torch.xpu.mem_get_info(self._execution_device)""", + ) + + # 8f. log_cache_state — add XPU memory display in allocated ternary + # NOTE: Upstream uses "allocated = (... if cuda else 0)" ternary. Extend it for xpu. + patch( + Path("backend/model_manager/load/model_cache/model_cache.py"), + """ allocated = ( + torch.cuda.memory_allocated(self._execution_device) if self._execution_device.type == "cuda" else 0 + )""", + """ allocated = ( + torch.cuda.memory_allocated(self._execution_device) + if self._execution_device.type == "cuda" + else ( + torch.xpu.memory_allocated(self._execution_device) + if self._execution_device.type == "xpu" and hasattr(torch, "xpu") and torch.xpu.is_available() + else 0 + ) + )""", + ) + + # 8g. model_cache.py imports — add is_xpu_available import + patch( + Path("backend/model_manager/load/model_cache/model_cache.py"), + "from invokeai.backend.util.devices import TorchDevice\nfrom invokeai.backend.util.logging import InvokeAILogger", + "from invokeai.backend.util.devices import TorchDevice\nfrom invokeai.backend.util.devices import is_xpu_available\nfrom invokeai.backend.util.logging import InvokeAILogger", + ) + + # ===================================================================== + # 9. attention.py + # XPU shares system RAM, so use psutil (like CPU/MPS). + # ===================================================================== + patch( + Path("backend/util/attention.py"), + """ if latents.device.type in {"cpu", "mps"}: + mem_free = psutil.virtual_memory().free + elif latents.device.type == "cuda": + mem_free, _ = torch.cuda.mem_get_info(latents.device) + else: + raise ValueError(f"unrecognized device {latents.device}")""", + """ if latents.device.type in {"cpu", "mps", "xpu"}: + mem_free = psutil.virtual_memory().free + elif latents.device.type == "cuda": + mem_free, _ = torch.cuda.mem_get_info(latents.device) + else: + raise ValueError(f"unrecognized device {latents.device}")""", + ) + + # ===================================================================== + # 10. memory_snapshot.py — add XPU memory_allocated + # ===================================================================== + patch( + Path("backend/model_manager/load/memory_snapshot.py"), + """ if torch.cuda.is_available(): + vram = torch.cuda.memory_allocated() + else: + # TODO: We could add support for mps.current_allocated_memory() as well. Leaving out for now until we have + # time to test it properly. + vram = None""", + """ if torch.cuda.is_available(): + vram = torch.cuda.memory_allocated() + elif hasattr(torch, "xpu") and torch.xpu.is_available(): + vram = torch.xpu.memory_allocated() + else: + # TODO: We could add support for mps.current_allocated_memory() as well. Leaving out for now until we have + # time to test it properly. + vram = None""", + ) + + # ===================================================================== + # 11. invocation_stats_default.py — XPU VRAM tracking + # Add _get_device_vram_allocated helper + XPU branches in + # collect_stats and get_stats. + # ===================================================================== + + # 11a. Add _get_device_vram_allocated helper + update imports + patch( + Path("app/services/invocation_stats/invocation_stats_default.py"), + """# Size of 1GB in bytes. +GB = 2**30 + + +class InvocationStatsService(InvocationStatsServiceBase):""", + """# Size of 1GB in bytes. +GB = 2**30 + + +def _get_device_vram_allocated(device: torch.device) -> int: + \"\"\"Return the amount of VRAM allocated on the given device, in bytes. + + Supports CUDA, XPU, and MPS devices. Returns 0 for unsupported devices. + \"\"\" + if device.type == "cuda": + return torch.cuda.memory_allocated(device) + elif device.type == "xpu": + return torch.xpu.memory_allocated(device) + elif device.type == "mps": + return torch.mps.current_allocated_memory() + return 0 + + +class InvocationStatsService(InvocationStatsServiceBase):""", + ) + + # 11b. collect_stats — add XPU VRAM tracking (before invocation) + patch( + Path("app/services/invocation_stats/invocation_stats_default.py"), + """ # Remember current VRAM usage + vram_in_use = torch.cuda.memory_allocated() if torch.cuda.is_available() else 0.0""", + """ # Remember current VRAM usage + vram_in_use = _get_device_vram_allocated(torch.device("cuda")) if torch.cuda.is_available() else (_get_device_vram_allocated(torch.device("xpu")) if hasattr(torch, "xpu") and torch.xpu.is_available() else 0)""", + ) + + # 11c. collect_stats — add XPU VRAM tracking (delta VRAM) + patch( + Path("app/services/invocation_stats/invocation_stats_default.py"), + """ # Record delta VRAM + delta_vram_gb = ((torch.cuda.memory_allocated() - vram_in_use) / GB) if torch.cuda.is_available() else 0.0""", + """ # Record delta VRAM + cur_vram = _get_device_vram_allocated(torch.device("cuda")) if torch.cuda.is_available() else (_get_device_vram_allocated(torch.device("xpu")) if hasattr(torch, "xpu") and torch.xpu.is_available() else 0) + delta_vram_gb = (cur_vram - vram_in_use) / GB""", + ) + + # 11d. get_stats — add XPU VRAM display + patch( + Path("app/services/invocation_stats/invocation_stats_default.py"), + """ # Note: We use memory_allocated() here (not memory_reserved()) because we want to show + # the current actively-used VRAM, not the total reserved memory including PyTorch's cache. + vram_usage_gb = torch.cuda.memory_allocated() / GB if torch.cuda.is_available() else None""", + """ # Note: We use memory_allocated() here (not memory_reserved()) because we want to show + # the current actively-used VRAM, not the total reserved memory including PyTorch's cache. + if torch.cuda.is_available(): + vram_usage_gb = torch.cuda.memory_allocated() / GB + elif hasattr(torch, "xpu") and torch.xpu.is_available(): + vram_usage_gb = torch.xpu.memory_allocated() / GB + else: + vram_usage_gb = None""", + ) + + # ===================================================================== + # 12. dev_utils.py — add _get_device_api helper + # ===================================================================== + patch( + Path("backend/model_manager/load/model_cache/dev_utils.py"), + """ torch.cuda.synchronize() + torch.cuda.reset_peak_memory_stats() + max_allocated_before = torch.cuda.max_memory_allocated() + max_reserved_before = torch.cuda.max_memory_reserved() + try: + yield + finally: + torch.cuda.synchronize() + max_allocated_after = torch.cuda.max_memory_allocated() + max_reserved_after = torch.cuda.max_memory_reserved() + logger = InvokeAILogger.get_logger() + logger.info( + f">>>{operation_name} Peak VRAM allocated: {(max_allocated_after - max_allocated_before) / 2**20} MB, " + f"Peak VRAM reserved: {(max_reserved_after - max_reserved_before) / 2**20} MB" + )""", + """ dev_api = _get_device_api() + if dev_api is None: + yield + return + dev_api.synchronize() + dev_api.reset_peak_memory_stats() + max_allocated_before = dev_api.max_memory_allocated() + max_reserved_before = dev_api.max_memory_reserved() + try: + yield + finally: + dev_api.synchronize() + max_allocated_after = dev_api.max_memory_allocated() + max_reserved_after = dev_api.max_memory_reserved() + logger = InvokeAILogger.get_logger() + logger.info( + f">>>{operation_name} Peak VRAM allocated: {(max_allocated_after - max_allocated_before) / 2**20} MB, " + f"Peak VRAM reserved: {(max_reserved_after - max_reserved_before) / 2**20} MB" + )""", + ) + + # 12b. Add _get_device_api helper to dev_utils.py + patch( + Path("backend/model_manager/load/model_cache/dev_utils.py"), + """from invokeai.backend.util.logging import InvokeAILogger + + +@contextmanager +def log_operation_vram_usage(operation_name: str):""", + """from invokeai.backend.util.logging import InvokeAILogger + + +def _get_device_api(): + \"\"\"Return the appropriate device API module (torch.cuda or torch.xpu) or None.\"\"\" + if torch.cuda.is_available(): + return torch.cuda + if hasattr(torch, "xpu") and torch.xpu.is_available(): + return torch.xpu + return None + + +@contextmanager +def log_operation_vram_usage(operation_name: str):""", + ) + + # ===================================================================== + # 13. diffusers_pipeline.py — XPU uses dedicated VRAM via torch.xpu.mem_get_info() + # ===================================================================== + patch( + Path("backend/stable_diffusion/diffusers_pipeline.py"), + """ if self.unet.device.type == "cpu" or self.unet.device.type == "mps": + mem_free = psutil.virtual_memory().free + elif self.unet.device.type == "cuda": + mem_free, _ = torch.cuda.mem_get_info(TorchDevice.normalize(self.unet.device)) + else: + raise ValueError(f"unrecognized device {self.unet.device}")""", + """ if self.unet.device.type in ("cpu", "mps"): + mem_free = psutil.virtual_memory().free + elif self.unet.device.type == "cuda": + mem_free, _ = torch.cuda.mem_get_info(TorchDevice.normalize(self.unet.device)) + elif self.unet.device.type == "xpu": + mem_free, _ = torch.xpu.mem_get_info(self.unet.device) + else: + raise ValueError(f"unrecognized device {self.unet.device}")""", + ) + + # ===================================================================== + # 14. bnb_nf4.py — fix docstring (cosmetic) + # ===================================================================== + patch( + Path("backend/quantization/bnb_nf4.py"), + ''' # Move the model to the "cuda" device. If the model was non-quantized, this is where the weight quantization takes + # place. + model.to("cuda") + ```''', + ''' # Move the model to the target device. If the model was non-quantized, this is where the weight quantization takes + # place. + model.to(device) + ```''', + ) + + # ===================================================================== + # 15. test_utils.py — add XPU to torch_device fixture + # ===================================================================== + patch( + Path("backend/util/test_utils.py"), + '''@pytest.fixture(scope="session") +def torch_device(): + return "cuda" if torch.cuda.is_available() else "cpu"''', + '''@pytest.fixture(scope="session") +def torch_device(): + if torch.cuda.is_available(): + return "cuda" + if hasattr(torch, "xpu") and torch.xpu.is_available(): + return "xpu" + return "cpu"''', + ) + + # ===================================================================== + # 16. ggml_tensor.py — add XPU to GGUF dispatch table + # ===================================================================== + patch( + Path("backend/quantization/gguf/ggml_tensor.py"), + """if torch.backends.mps.is_available(): + GGML_TENSOR_OP_TABLE.update( + {torch.ops.aten.linear.default: dequantize_and_run}""", + """if torch.backends.mps.is_available() or (hasattr(torch, "xpu") and torch.xpu.is_available()): + GGML_TENSOR_OP_TABLE.update( + {torch.ops.aten.linear.default: dequantize_and_run}""", + ) + + # ===================================================================== + # 17. anima_latents_to_image.py — XPU OOM detection + tiled decode + # The _is_oom_error and _use_tiled_decode functions need XPU + # awareness for Intel Arc GPUs. + # ===================================================================== + + # 17a. _is_oom_error — add XPU OOM string patterns + patch( + Path("app/invocations/anima_latents_to_image.py"), + """ if isinstance(e, torch.cuda.OutOfMemoryError): + return True + msg = str(e) + return "out of memory" in msg.lower() or "CUDNN_STATUS_ALLOC_FAILED" in msg or "CUBLAS_STATUS_ALLOC_FAILED" in msg""", + """ if isinstance(e, torch.cuda.OutOfMemoryError): + return True + msg = str(e).lower() + if "out of memory" in msg or "ur_result_error_out_of_device_memory" in msg: + return True + return "cudnn_status_alloc_failed" in msg or "cublas_status_alloc_failed" in msg""", + ) + + # 17b. _use_tiled_decode — add XPU device support + patch( + Path("app/invocations/anima_latents_to_image.py"), + """ if device.type != "cuda": + return False + total_vram = torch.cuda.get_device_properties(device).total_memory""", + """ if device.type not in ("cuda", "xpu"): + return False + if device.type == "cuda": + total_vram = torch.cuda.get_device_properties(device).total_memory + else: + total_vram = torch.xpu.get_device_properties(device).total_memory""", + ) + + # ===================================================================== + # 18. pidi/model.py — fix torch.cuda.FloatTensor for XPU + # The rd_conv kernel creates a CUDA-only FloatTensor buffer. + # On XPU (or any non-CUDA device), use torch.zeros().to(device). + # ===================================================================== + patch( + Path("backend/image_util/pidi/model.py"), + """ if weights.is_cuda: + buffer = torch.cuda.FloatTensor(shape[0], shape[1], 5 * 5).fill_(0) + else: + buffer = torch.zeros(shape[0], shape[1], 5 * 5).to(weights.device)""", + """ buffer = torch.zeros(shape[0], shape[1], 5 * 5, device=weights.device)""", + ) + + # ===================================================================== + # 19. hotfixes.py — disable xformers on Intel XPU + # xformers doesn't support XPU, so diffusers should not try to use it. + # Append a monkeypatch that makes diffusers think xformers is unavailable + # when running on XPU. + # ===================================================================== + _hotfixes_xpu_xformers = """ +# Patch diffusers to disable xformers on Intel XPU devices +def _patch_diffusers_xformers_for_xpu(): + \"\"\"Disable xformers in diffusers when running on Intel XPU devices.\"\"\" + try: + import diffusers.utils.import_utils as diffusers_import_utils + + original_is_xformers_available = diffusers_import_utils.is_xformers_available + + def patched_is_xformers_available(): + if hasattr(torch, "xpu") and torch.xpu.is_available(): + return False + return original_is_xformers_available() + + diffusers_import_utils.is_xformers_available = patched_is_xformers_available + + if hasattr(diffusers_import_utils, '_xformers_available'): + diffusers_import_utils._xformers_available = not (hasattr(torch, "xpu") and torch.xpu.is_available()) + + except Exception: + pass + + +_patch_diffusers_xformers_for_xpu() +""" + patch( + Path("backend/util/hotfixes.py"), + " )\n\n xformers.ops.memory_efficient_attention = new_memory_efficient_attention", + " )\n\n # [XPU patch] xformers disabled for Intel GPU support\n xformers.ops.memory_efficient_attention = new_memory_efficient_attention\n" + _hotfixes_xpu_xformers, + ) + + + # ===================================================================== + # 20. model_cache.py — running_with_cuda includes XPU + # Line 600: XPU models need the same CUDA-like cache management path. + # ===================================================================== + patch( + Path("backend/model_manager/load/model_cache/model_cache.py"), + 'running_with_cuda = effective_execution_device.type == "cuda"', + 'running_with_cuda = effective_execution_device.type in ("cuda", "xpu")', + ) + + # ===================================================================== + # 21. session_processor_default.py — XPU set_device + # Line 632: CUDA calls set_device; XPU needs the same for multi-device. + # ===================================================================== + patch( + Path("app/services/session_processor/session_processor_default.py"), + """ if worker.device.type == "cuda": + torch.cuda.set_device(worker.device)""", + """ if worker.device.type == "cuda": + torch.cuda.set_device(worker.device) + elif worker.device.type == "xpu": + torch.xpu.set_device(worker.device)""", + ) + + # ===================================================================== + # 22. config_default.py — update non-auto pattern (non-default field) + # Line 285: The non-auto Field pattern needs xpu too. + # ===================================================================== + patch( + Path("app/services/config/config_default.py"), + 'pattern=r"^(cpu|mps|cuda(:\\d+)?)$")', + 'pattern=r"^(cpu|mps|cuda(:\\d+)?|xpu(:\\d+)?)$")', + ) + + # ===================================================================== + # 23. app_info.py — XPU in GenerationDevice enum + device listing + # Line 123: API device pattern. Lines 226-231: enumerate XPU GPUs. + # ===================================================================== + patch( + Path("app/api/routers/app_info.py"), + '_GENERATION_DEVICE_PATTERN = re.compile(r"^(cpu|mps|cuda(:\\d+)?)$")', + '_GENERATION_DEVICE_PATTERN = re.compile(r"^(cpu|mps|cuda(:\\d+)?|xpu(:\\d+)?)$")', + ) + + patch( + Path("app/api/routers/app_info.py"), + """ for index in range(torch.cuda.device_count()): + device = f"cuda:{index}" + name = torch.cuda.get_device_name(index) + options.append(GenerationDeviceOption(device=device, name=name))""", + """ for index in range(torch.cuda.device_count()): + device = f"cuda:{index}" + name = torch.cuda.get_device_name(index) + options.append(GenerationDeviceOption(device=device, name=name)) + elif hasattr(torch, "xpu") and torch.xpu.is_available(): + for index in range(torch.xpu.device_count()): + device = f"xpu:{index}" + name = torch.xpu.get_device_name(index) + options.append(GenerationDeviceOption(device=device, name=name))""", + ) + + # ===================================================================== + # 24. events_common.py — XPU in device label detection + # Lines 159, 166: Event payloads report GPU device for CUDA; add XPU. + # ===================================================================== + patch( + Path("app/services/events/events_common.py"), + 'device: str | None = queue_item.device if queue_item.device and queue_item.device.startswith("cuda") else None', + 'device: str | None = queue_item.device if queue_item.device and (queue_item.device.startswith("cuda") or queue_item.device.startswith("xpu")) else None', + ) + + patch( + Path("app/services/events/events_common.py"), + 'device = str(session_device) if session_device is not None and session_device.type == "cuda" else None', + 'device = str(session_device) if session_device is not None and session_device.type in ("cuda", "xpu") else None', + ) + + # ===================================================================== + # 25. pid_distill_model.py — device-agnostic inference + # 9 hardcoded device="cuda" → device=self.net.device + # autocast("cuda") → autocast(self.net.device.type) + # ===================================================================== + patch( + Path("backend/pid/_src/models/pid_distill_model.py"), + 'device="cuda"', + 'device=self.net.device', + ) + + patch( + Path("backend/pid/_src/models/pid_distill_model.py"), + 'torch.autocast("cuda")', + 'torch.autocast(self.net.device.type)', + ) + + # ===================================================================== + # 26. pixeldit_model.py — runtime .to("cuda") → .to(self.net.device) + # Line 188 only; lines 86/141/145/154 keep cuda default (init path). + # ===================================================================== + patch( + Path("backend/pid/_src/models/pixeldit_model.py"), + """.to("cuda")""", + """.to(self.net.device)""", + ) + + # ===================================================================== + # 27. pid/decode.py — autocast XPU support + # Lines 262, 385: autocast only fires on CUDA; add XPU. + # ===================================================================== + patch( + Path("backend/pid/decode.py"), + 'noise.device.type == "cuda"', + 'noise.device.type in ("cuda", "xpu")', + ) + + patch( + Path("backend/pid/decode.py"), + 'autocast_dtype = torch.bfloat16 if device.type == "cuda" else None', + 'autocast_dtype = torch.bfloat16 if device.type in ("cuda", "xpu") else None', + ) + + # ===================================================================== + # 28. pipeline_registry.py — device-agnostic FPD eval pipeline + # Line 184: hardcoded device="cuda" parameter default. + # ===================================================================== + patch( + Path("backend/pid/_src/inference/pipeline_registry.py"), + 'device: Optional[str] = "cuda"', + 'device: Optional[str] = None', + ) + + # ===================================================================== + # 29. devices.py — multi-GPU API additions + # get_session_device_index, _all_available_devices, get_generation_devices + # already handled by patches 1c, 1g, 1h above. Add any remaining gaps. + # ===================================================================== + # (Covered by patches 1c, 1g, 1h — no additional patch needed.) + + # ===================================================================== + # 30. bnb_llm_int8.py — bitsandbytes XPU guard + # bitsandbytes does not support XPU; provide clear error instead of crash. + # ===================================================================== + patch( + Path("backend/quantization/bnb_llm_int8.py"), + "class InvokeInt8Params(BnBInt8Params):", + """class InvokeInt8Params(BnBInt8Params): + def to(self, *args, **kwargs): + # bitsandbytes does not support Intel XPU devices. + device = kwargs.get("device") or (args[0] if args else None) + if device is not None: + d = torch.device(device) if isinstance(device, str) else device + if d.type == "xpu": + raise RuntimeError( + "bitsandbytes INT8 quantization is not supported on Intel XPU devices. " + "Please use a different quantization method or CPU offloading." + ) + return super().to(*args, **kwargs)""", + ) + + # ===================================================================== + # 31. diffusers_pipeline.py — XPU VRAM detection (replaces old patch #13) + # XPU has dedicated VRAM, not shared system RAM. + # ===================================================================== + # Already handled by the fixed patch #13 above. + # Override the official zh-CN locale with our preferred translations. + patch_locale() + + print("XPU patch applied successfully.") + + +if __name__ == "__main__": + main() diff --git a/scripts/sync_and_patch.py b/scripts/sync_and_patch.py new file mode 100644 index 00000000000..6b2bd252410 --- /dev/null +++ b/scripts/sync_and_patch.py @@ -0,0 +1 @@ +fatal: path 'scripts/sync_and_patch.py' exists on disk, but not in 'stash@{0}'