Skip to content

Add Anima regional control support #2524

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Acly merged 8 commits into from
Aug 19, 2026
Merged

Add Anima regional control support #2524

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
da8f931
Add Anima regional control support
Sen-sou Jun 12, 2026
a702110
Fixed regional controlnet only usage
Sen-sou Jun 13, 2026
8e98cd4
Removed regional control mode and integrated it as segmentation contr…
Sen-sou Jun 27, 2026
050d178
Add explicit region segmentation control layer generation
Sen-sou Jun 28, 2026
9fa5755
Fill transparent segmentation controls with white
Sen-sou Jun 28, 2026
dc7dbc4
Use computed region bounds for segmentation control
Sen-sou Jun 28, 2026
9b5e085
cleanup
Sen-sou Jun 30, 2026
f51f547
lint
Sen-sou Jun 30, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion ai_diffusion/backend/resources.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,7 @@ def has_controlnet_inpaint(self):

@property
def supports_regions(self):
return self in [Arch.sd15, Arch.sdxl, Arch.illu, Arch.illu_v]
return self in [Arch.sd15, Arch.sdxl, Arch.illu, Arch.illu_v, Arch.anima]

@property
def supports_lcm(self):
Expand Down Expand Up @@ -738,6 +738,7 @@ def is_required(kind: ResourceKind, arch: Arch, identifier: ControlMode | Upscal
resource_id(ResourceKind.controlnet, Arch.illu, ControlMode.universal): ["union-sdxl", "xinsirunion"],
resource_id(ResourceKind.controlnet, Arch.illu_v, ControlMode.universal): ["union-sdxl", "xinsirunion"],
resource_id(ResourceKind.controlnet, Arch.anima, ControlMode.universal): ["anima*lllite*any"],
resource_id(ResourceKind.controlnet, Arch.anima, ControlMode.segmentation): ["anima-lllite-region-cn", "anima*lllite*region"],
resource_id(ResourceKind.controlnet, Arch.flux, ControlMode.universal): ["flux.1-dev-controlnet-union-pro-2.0", "flux.1-dev-controlnet-union-pro", "flux.1-dev-controlnet-union", "flux1devcontrolnetunion"],
resource_id(ResourceKind.controlnet, Arch.qwen, ControlMode.universal): ["qwen-image-instantx-controlnet-union"],
resource_id(ResourceKind.controlnet, Arch.sd15, ControlMode.scribble): ["control_v11p_sd15_scribble", "control_lora_rank128_v11p_sd15_scribble"],
Expand Down
74 changes: 73 additions & 1 deletion ai_diffusion/model/control.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,13 @@
from typing import Any, NamedTuple

from PyQt5.QtCore import QObject, Qt, QUuid, pyqtSignal
from PyQt5.QtGui import QColor

from .. import util
from ..backend import resources
from ..backend.api import ControlInput
from ..backend.resources import Arch, ControlMode, ResourceKind, resource_id
from ..image import Bounds, Extent, Image
from ..image import BlendMode, Bounds, Extent, Image
from ..layer import Layer, LayerType
from ..localization import translate as _
from ..util import PluginError
Expand All @@ -23,6 +24,20 @@
max_preset_value = 4
strength_multiplier = 50
clip_vision_extent = Extent(224, 224)
segmentation_colors = [
(120, 120, 120),
(180, 120, 120),
(120, 180, 120),
(120, 120, 180),
(180, 180, 120),
(180, 120, 180),
(120, 180, 180),
(220, 140, 100),
(140, 220, 100),
(100, 140, 220),
(220, 100, 140),
(100, 220, 140),
]

Check failure on line 40 in ai_diffusion/model/control.py

View workflow job for this annotation

GitHub Actions / check

ruff (mutable-class-default) 

ai_diffusion/model/control.py:27:27: mutable-class-default: Mutable default value for class attribute
  help: Consider initializing in `__init__` or annotating with `typing.ClassVar`

mode = Property(ControlMode.reference, persist=True, setter="set_mode")
layer_id = Property(QUuid(), persist=True)
Expand Down Expand Up @@ -125,6 +140,9 @@

image = layer.get_pixels(bounds, time)

if self.mode is ControlMode.segmentation:
image.make_opaque(background=Qt.GlobalColor.white)

if self.mode.is_lines or self.mode is ControlMode.stencil:
image.make_opaque(background=Qt.GlobalColor.white)

Expand All @@ -139,7 +157,61 @@
strength = self.strength / self.strength_multiplier
return ControlInput(self.mode, image, strength, (self.start, self.end))

def generate_segmentation(self):
assert self.mode is ControlMode.segmentation

ok, msg = self._model.document.check_color_mode()
if not ok and msg:
self._model.report_error(msg)
return

try:
bounds = Bounds.from_extent(self._model.document.extent)
image = self._segmentation_image_from_regions(bounds)
if image is None:
self._model.report_error(_("Text prompt regions have not been set up."))
return

layer = self._model.layers.create(f"[Control] {self.mode.text}", image, bounds)
self.layer_id = layer.id
except Exception as e:
self._model.report_error(util.log_error(e))
else:
self._model.clear_error()

def _segmentation_image_from_regions(self, bounds: Bounds):
from .region import RegionLink

image = Image.create(bounds.extent, fill=Qt.GlobalColor.white)
has_region_layer = False
root = self._model.active_regions

layers = [
layer
for layer in root.layers.all
if root.find_linked(layer, RegionLink.direct) is not None
and Bounds.intersection(bounds, layer.compute_bounds()).area > 0
]

for index, layer in enumerate(layers):
color = self.segmentation_colors[index % len(self.segmentation_colors)]
region_image = self._segmentation_region_image(layer, bounds, color)
image.draw_image(region_image, blend=BlendMode.alpha)
has_region_layer = True

return image if has_region_layer else None

def _segmentation_region_image(
self, layer: Layer, bounds: Bounds, color: tuple[int, int, int]
):
mask = layer.get_mask(bounds)
image = Image.create(bounds.extent, fill=QColor(*color, 255))
image._qimage.setAlphaChannel(mask._qimage)
return image

def generate(self):
if not self.can_generate:

@Acly Acly Jun 28, 2026

Copy link
Copy Markdown
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

any particular reason this was added?

@Sen-sou Sen-sou Jun 28, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

safeguard for unintended calls

@Acly Acly Jun 30, 2026

Copy link
Copy Markdown
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what unintended calls? if there is no way to hit this code please remove it.

@Sen-sou Sen-sou Jun 30, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yea sorry about that, i didnt check there were only two calls for this, and they would be hidden. Removed it.

return
self._generate_job = self._model.generate_control_layer(self)
self.has_active_job = True

Expand Down
38 changes: 37 additions & 1 deletion ai_diffusion/presets/control.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -199,5 +199,41 @@
"end": 1.0
}
]
},
"segmentation": {
"all": [
{
"strength": 0.7,
"start": 0.0,
"end": 0.5
},
{
"strength": 1.0,
"start": 0.0,
"end": 0.8
},
{
"strength": 1.0,
"start": 0.0,
"end": 1.0
}
],
"anima": [
{
"strength": 0.75,
"start": 0.0,
"end": 0.45
},
{
"strength": 1.0,
"start": 0.0,
"end": 0.45
},
{
"strength": 1.5,
"start": 0.0,
"end": 0.45
}
]
}
}
}
30 changes: 30 additions & 0 deletions ai_diffusion/ui/control.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,11 @@ def __init__(
)
self.generate_tool_button.clicked.connect(control.generate)

self.generate_regions_tool_button = _create_generate_regions_button(
self, Qt.ToolButtonStyle.ToolButtonIconOnly
)
self.generate_regions_tool_button.clicked.connect(control.generate_segmentation)

self.add_pose_tool_button = _create_add_pose_button(
self, Qt.ToolButtonStyle.ToolButtonIconOnly
)
Expand All @@ -91,6 +96,7 @@ def __init__(
bar_layout.addWidget(self.mode_select)
bar_layout.addWidget(self.layer_select, 3)
bar_layout.addWidget(self.generate_tool_button)
bar_layout.addWidget(self.generate_regions_tool_button)
bar_layout.addWidget(self.add_pose_tool_button)
bar_layout.addWidget(self.preset_slider, 1)
bar_layout.addWidget(self.error_text, 3)
Expand Down Expand Up @@ -129,6 +135,11 @@ def __init__(
)
self.generate_button.clicked.connect(control.generate)

self.generate_regions_button = _create_generate_regions_button(
self.extended_widget, Qt.ToolButtonStyle.ToolButtonTextBesideIcon
)
self.generate_regions_button.clicked.connect(control.generate_segmentation)

self.add_pose_button = _create_add_pose_button(
self.extended_widget, Qt.ToolButtonStyle.ToolButtonTextBesideIcon
)
Expand All @@ -141,6 +152,7 @@ def __init__(
actions_layout = QHBoxLayout()
actions_layout.addWidget(self.custom_checkbox, stretch=1)
actions_layout.addWidget(self.generate_button)
actions_layout.addWidget(self.generate_regions_button)
actions_layout.addWidget(self.add_pose_button)
extended_layout.addLayout(actions_layout)

Expand Down Expand Up @@ -228,6 +240,7 @@ def _add_pose_character(self):
def _update_visibility(self):
is_small = self.width() < 420
is_pose = self._control.mode is ControlMode.pose
is_segmentation = self._control.mode is ControlMode.segmentation
is_edit = root.active_model.arch.supports_edit

def controls():
Expand All @@ -236,6 +249,12 @@ def controls():
self.expand_button.setVisible(self._control.is_supported and not is_edit)
self.generate_button.setVisible(self._control.can_generate and is_small)
self.generate_tool_button.setVisible(self._control.can_generate and not is_small)
self.generate_regions_button.setVisible(
self._control.is_supported and is_segmentation and is_small
)
self.generate_regions_tool_button.setVisible(
self._control.is_supported and is_segmentation and not is_small
)
self.add_pose_button.setVisible(is_pose and is_small)
self.add_pose_tool_button.setVisible(is_pose and not is_small)
self.range_label.setVisible(self._control.has_range)
Expand Down Expand Up @@ -272,6 +291,8 @@ def _update_strength(self):
def _update_job_active(self):
self.generate_button.setEnabled(not self._control.has_active_job)
self.generate_tool_button.setEnabled(not self._control.has_active_job)
self.generate_regions_button.setEnabled(not self._control.has_active_job)
self.generate_regions_tool_button.setEnabled(not self._control.has_active_job)
self.layer_select.setEnabled(not self._control.has_active_job)

def _update_custom_values(self):
Expand Down Expand Up @@ -309,6 +330,15 @@ def _create_generate_button(parent, style: Qt.ToolButtonStyle):
return button


def _create_generate_regions_button(parent, style: Qt.ToolButtonStyle):
button = QToolButton(parent)
button.setToolButtonStyle(style)
button.setText(_("From Regions"))
button.setIcon(theme.icon("region-prompt"))
button.setToolTip(_("Generate segmentation control layer from current regions"))
return button


def _create_add_pose_button(parent, style: Qt.ToolButtonStyle):
button = QToolButton(parent)
button.setToolButtonStyle(style)
Expand Down
Loading