diff --git a/frontend/src/components/graph/AddNodeModal.tsx b/frontend/src/components/graph/AddNodeModal.tsx index 59e91ed89..65588a9d2 100644 --- a/frontend/src/components/graph/AddNodeModal.tsx +++ b/frontend/src/components/graph/AddNodeModal.tsx @@ -1,4 +1,4 @@ -import { useState, useMemo, useRef, useEffect } from "react"; +import { useState, useMemo, useRef, useEffect, useCallback } from "react"; import { Dialog, DialogContent, @@ -6,6 +6,7 @@ import { DialogTitle, DialogDescription, } from "../ui/dialog"; +import type { FlowNodeData } from "../../lib/graphUtils"; interface AddNodeModalProps { open: boolean; @@ -36,8 +37,10 @@ interface AddNodeModalProps { | "tempo" | "prompt_list" | "prompt_blend" - | "scheduler", - subType?: string + | "scheduler" + | "custom_node", + subType?: string, + extraData?: Partial ) => void; } @@ -67,12 +70,15 @@ interface NodeCatalogItem { | "tempo" | "prompt_list" | "prompt_blend" - | "scheduler"; + | "scheduler" + | "custom_node"; subType?: string; name: string; description: string; color: string; category: string; + /** Full definition for custom nodes (inputs/outputs/params). */ + customNodeDef?: Record; } const NODE_CATALOG: NodeCatalogItem[] = [ @@ -85,8 +91,8 @@ const NODE_CATALOG: NodeCatalogItem[] = [ }, { type: "pipeline", - name: "Pipeline", - description: "Processing pipeline node", + name: "Node", + description: "Video processing node (pick a model after dropping it)", color: "#60a5fa", category: "I/O", }, @@ -294,7 +300,15 @@ const NODE_CATALOG: NodeCatalogItem[] = [ }, ]; -const CATEGORIES = ["All", "I/O", "Values", "Controls", "UI", "Utility"]; +const CATEGORIES = [ + "All", + "I/O", + "Values", + "Controls", + "UI", + "Utility", + "Plugins", +]; interface TooltipState { text: string; @@ -388,10 +402,45 @@ export function AddNodeModal({ }: AddNodeModalProps) { const [searchText, setSearchText] = useState(""); const [activeCategory, setActiveCategory] = useState("All"); + const [customNodes, setCustomNodes] = useState([]); + + useEffect(() => { + if (!open) return; + fetch("/api/v1/nodes/definitions") + .then(r => r.json()) + .then(data => { + // The unified endpoint returns both pipelines (pipeline_meta != null) + // and plain custom nodes. Pipelines are still added via the hardcoded + // "Pipeline" catalog entry (placeholder + dropdown), so we filter + // them out of the plugin listing here to avoid duplication. + const items: NodeCatalogItem[] = (data.nodes ?? []) + // eslint-disable-next-line @typescript-eslint/no-explicit-any + .filter((n: any) => n.pipeline_meta == null) + // eslint-disable-next-line @typescript-eslint/no-explicit-any + .map((n: any) => ({ + type: "custom_node" as const, + subType: n.node_type_id, + name: n.display_name || n.node_type_id, + description: n.description || "", + color: "#9ca3af", + category: "Plugins", + customNodeDef: n, + })); + setCustomNodes(items); + }) + .catch(() => { + /* ignore — custom nodes just won't appear */ + }); + }, [open]); + + const fullCatalog = useMemo( + () => [...NODE_CATALOG, ...customNodes], + [customNodes] + ); const filteredItems = useMemo(() => { const lowerSearch = searchText.toLowerCase(); - return NODE_CATALOG.filter(item => { + return fullCatalog.filter(item => { const matchesSearch = !lowerSearch || item.name.toLowerCase().includes(lowerSearch) || @@ -400,14 +449,37 @@ export function AddNodeModal({ activeCategory === "All" || item.category === activeCategory; return matchesSearch && matchesCategory; }); - }, [searchText, activeCategory]); + }, [searchText, activeCategory, fullCatalog]); - const handleSelect = (item: NodeCatalogItem) => { - onSelectNodeType(item.type, item.subType); - onClose(); - setSearchText(""); - setActiveCategory("All"); - }; + const handleSelect = useCallback( + (item: NodeCatalogItem) => { + if (item.type === "custom_node" && item.customNodeDef) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const def = item.customNodeDef as any; + onSelectNodeType("custom_node", item.subType, { + customNodeTypeId: def.node_type_id, + customNodeDisplayName: def.display_name || def.node_type_id, + customNodeCategory: def.category || "", + customNodeInputs: def.inputs || [], + customNodeOutputs: def.outputs || [], + customNodeParamDefs: def.params || [], + customNodeParams: Object.fromEntries( + (def.params || []) + // eslint-disable-next-line @typescript-eslint/no-explicit-any + .filter((p: any) => p.default != null) + // eslint-disable-next-line @typescript-eslint/no-explicit-any + .map((p: any) => [p.name, p.default]) + ), + }); + } else { + onSelectNodeType(item.type, item.subType); + } + onClose(); + setSearchText(""); + setActiveCategory("All"); + }, + [onSelectNodeType, onClose] + ); const handleClose = () => { onClose(); diff --git a/frontend/src/components/graph/GraphEditor.tsx b/frontend/src/components/graph/GraphEditor.tsx index 15446cc80..18b71338a 100644 --- a/frontend/src/components/graph/GraphEditor.tsx +++ b/frontend/src/components/graph/GraphEditor.tsx @@ -55,6 +55,7 @@ import { TempoNode } from "./nodes/TempoNode"; import { PromptListNode } from "./nodes/PromptListNode"; import { PromptBlendNode } from "./nodes/PromptBlendNode"; import { SchedulerNode } from "./nodes/SchedulerNode"; +import { CustomNode } from "./nodes/CustomNode"; import { CustomEdge } from "./CustomEdge"; import { ContextMenu } from "./ContextMenu"; import { AddNodeModal } from "./AddNodeModal"; @@ -130,6 +131,7 @@ const nodeTypes = { prompt_list: PromptListNode, prompt_blend: PromptBlendNode, scheduler: SchedulerNode, + custom_node: CustomNode, }; const edgeTypes = { diff --git a/frontend/src/components/graph/contextMenuItems.tsx b/frontend/src/components/graph/contextMenuItems.tsx index 0d4903f9e..caeadd698 100644 --- a/frontend/src/components/graph/contextMenuItems.tsx +++ b/frontend/src/components/graph/contextMenuItems.tsx @@ -96,10 +96,10 @@ export function buildPaneMenuItems(deps: { keywords: ["input", "camera", "video"], }, { - label: "Pipeline", + label: "Node", icon: , onClick: () => handleNodeTypeSelect("pipeline"), - keywords: ["process", "effect", "filter"], + keywords: ["process", "effect", "filter", "pipeline"], }, { label: "Sink", diff --git a/frontend/src/components/graph/hooks/graph/useGraphPersistence.ts b/frontend/src/components/graph/hooks/graph/useGraphPersistence.ts index 8a2248d31..f31df93f8 100644 --- a/frontend/src/components/graph/hooks/graph/useGraphPersistence.ts +++ b/frontend/src/components/graph/hooks/graph/useGraphPersistence.ts @@ -7,8 +7,8 @@ import { parseHandleId, } from "../../../../lib/graphUtils"; import type { FlowNodeData } from "../../../../lib/graphUtils"; -import type { PluginInfo } from "../../../../lib/api"; -import { resolveWorkflow } from "../../../../lib/api"; +import type { PluginInfo, NodeDefinitionDto } from "../../../../lib/api"; +import { resolveWorkflow, fetchNodeDefinitions } from "../../../../lib/api"; import type { ScopeWorkflow, WorkflowResolutionPlan, @@ -65,6 +65,72 @@ function clearGraphFromLocalStorage(): void { } } +/** + * After loading or importing a workflow, fetch `/api/v1/nodes/definitions` + * and hydrate each custom_node flow node with its inputs/outputs/params/ + * display metadata. Saved workflows only persist `node_type_id` and + * `params`, so the port definitions have to be re-attached before render. + * User-supplied param values override the defaults from the definition. + * + * Accepts an AbortSignal so rapid reloads / unmounts can cancel an + * in-flight fetch before its setNodes callback stomps on newer state. + */ +function hydrateCustomNodeDefinitions( + nodes: Node[], + setNodes: React.Dispatch[]>>, + signal: AbortSignal +): void { + const customFlowNodes = nodes.filter( + n => n.data.nodeType === "custom_node" && !n.data.customNodeInputs + ); + if (customFlowNodes.length === 0) return; + fetchNodeDefinitions({ signal }) + .then(data => { + if (signal.aborted) return; + const defMap = new Map( + data.nodes.map(d => [d.node_type_id, d]) + ); + setNodes(prev => { + if (signal.aborted) return prev; + return prev.map(n => { + if ( + n.data.nodeType !== "custom_node" || + !n.data.customNodeTypeId || + n.data.customNodeInputs + ) { + return n; + } + const def = defMap.get(n.data.customNodeTypeId); + if (!def) return n; + return { + ...n, + data: { + ...n.data, + customNodeDisplayName: def.display_name, + customNodeCategory: def.category, + customNodeInputs: def.inputs ?? [], + customNodeOutputs: def.outputs ?? [], + customNodeParamDefs: def.params ?? [], + customNodeParams: { + ...Object.fromEntries( + (def.params ?? []) + .filter(p => p.default != null) + .map(p => [p.name, p.default] as const) + ), + // User-edited values take precedence over definition defaults + ...(n.data.customNodeParams || {}), + }, + }, + }; + }); + }); + }) + .catch((err: unknown) => { + if (err instanceof DOMException && err.name === "AbortError") return; + // custom nodes just won't be hydrated; render falls back to placeholders + }); +} + interface UseGraphPersistenceArgs { nodes: Node[]; edges: Edge[]; @@ -129,6 +195,25 @@ export function useGraphPersistence({ // to localStorage so we skip the expensive save when nothing changed. const lastSavedJsonRef = useRef(""); + // AbortController for in-flight custom-node hydration fetches. Aborted + // before each new hydrate and on unmount so a stale /api/v1/nodes/definitions + // response can't overwrite newer nodes state. + const hydrateAbortRef = useRef(null); + const startHydrate = useCallback( + (initialNodes: Node[]) => { + hydrateAbortRef.current?.abort(); + const controller = new AbortController(); + hydrateAbortRef.current = controller; + hydrateCustomNodeDefinitions(initialNodes, setNodes, controller.signal); + }, + [setNodes] + ); + useEffect(() => { + return () => { + hydrateAbortRef.current?.abort(); + }; + }, []); + const loadGraph = useCallback(() => { if (Object.keys(portsMap).length === 0) return; resetNavigationRef.current?.(); @@ -182,6 +267,8 @@ export function useGraphPersistence({ }, 0); } + startHydrate(enriched); + // Allow async side-effects (e.g. source mode restore) to settle // before re-enabling change notifications. setTimeout(() => { @@ -202,6 +289,7 @@ export function useGraphPersistence({ setEdges, setNodeParams, setNodes, + startHydrate, ]); useEffect(() => { @@ -374,6 +462,8 @@ export function useGraphPersistence({ } }, 0); } + + startHydrate(enriched); }, [ portsMap, @@ -384,6 +474,7 @@ export function useGraphPersistence({ setNodeParams, enrichDepsRef, resetNavigationRef, + startHydrate, ] ); diff --git a/frontend/src/components/graph/hooks/node/useNodeFactories.ts b/frontend/src/components/graph/hooks/node/useNodeFactories.ts index 36982fa76..60803a265 100644 --- a/frontend/src/components/graph/hooks/node/useNodeFactories.ts +++ b/frontend/src/components/graph/hooks/node/useNodeFactories.ts @@ -44,7 +44,8 @@ type NodeTypeKey = | "tempo" | "prompt_list" | "prompt_blend" - | "scheduler"; + | "scheduler" + | "custom_node"; interface NodeDefaults { /** The React Flow node `type` */ @@ -470,6 +471,15 @@ const NODE_DEFAULTS: Record = { ], }, }, + custom_node: { + type: "custom_node", + idPrefix: "custom", + defaultX: 300, + data: { + label: "Custom Node", + nodeType: "custom_node" as const, + }, + }, }; interface UseNodeFactoriesArgs { @@ -565,8 +575,10 @@ export function useNodeFactories({ | "tempo" | "prompt_list" | "prompt_blend" - | "scheduler", - subType?: string + | "scheduler" + | "custom_node", + subType?: string, + extraData?: Partial ) => { if (!pendingNodePosition) return; @@ -597,6 +609,12 @@ export function useNodeFactories({ outputSinkType: defaultType, outputSinkName: defaultNames[defaultType] || "Scope", }); + } else if (type === "custom_node") { + addNode("custom_node", pendingNodePosition, { + customNodeTypeId: subType, + label: subType || "Custom Node", + ...extraData, + }); } else { addNode(type as NodeTypeKey, pendingNodePosition); } diff --git a/frontend/src/components/graph/nodes/CustomNode.tsx b/frontend/src/components/graph/nodes/CustomNode.tsx new file mode 100644 index 000000000..a051674af --- /dev/null +++ b/frontend/src/components/graph/nodes/CustomNode.tsx @@ -0,0 +1,223 @@ +import { Handle, Position } from "@xyflow/react"; +import type { NodeProps, Node } from "@xyflow/react"; +import type { FlowNodeData } from "../../../lib/graphUtils"; +import { buildHandleId } from "../../../lib/graphUtils"; +import { useNodeData } from "../hooks/node/useNodeData"; +import { useNodeCollapse } from "../hooks/node/useNodeCollapse"; +import { NodeCard, NodeHeader, NodeBody, collapsedHandleStyle } from "../ui"; + +type CustomNodeType = Node; + +/* Port type -> color mapping for custom types */ +const PORT_COLORS: Record = { + audio: "#22c55e", + video: "#eeeeee", + number: "#38bdf8", + string: "#fbbf24", + boolean: "#34d399", + trigger: "#f97316", + latent: "#a855f7", + model: "#f59e0b", + vae: "#f59e0b", + clip: "#f59e0b", + conditioning: "#3b82f6", + semantic_hints: "#06b6d4", + config: "#6b7280", + curve: "#ec4899", + mask: "#ef4444", + lora: "#f472b6", +}; + +function portColor(portType: string): string { + return PORT_COLORS[portType] ?? "#9ca3af"; +} + +export function CustomNode({ id, data, selected }: NodeProps) { + const { updateData } = useNodeData(id); + const { collapsed, toggleCollapse } = useNodeCollapse(); + + const inputs = data.customNodeInputs ?? []; + const outputs = data.customNodeOutputs ?? []; + const params = data.customNodeParamDefs ?? []; + const displayName = + data.customTitle || + data.customNodeDisplayName || + data.customNodeTypeId || + "Custom Node"; + const category = data.customNodeCategory ?? ""; + const onParameterChange = data.onParameterChange as + | ((nodeId: string, key: string, value: unknown) => void) + | undefined; + + const setParam = (name: string, value: unknown) => { + updateData({ + customNodeParams: { + ...data.customNodeParams, + [name]: value, + }, + }); + onParameterChange?.(id, name, value); + }; + + return ( + + updateData({ customTitle: t })} + collapsed={collapsed} + onCollapseToggle={toggleCollapse} + /> + {!collapsed && ( + + {/* Show category badge */} + {category && ( +
+ + {category} + +
+ )} + {/* Show input ports */} + {inputs.length > 0 && ( +
+ {inputs.map(p => ( +
+ + {p.name} + + {p.port_type} + +
+ ))} +
+ )} + {/* Show output ports */} + {outputs.length > 0 && ( +
+ {outputs.map(p => ( +
+ + {p.port_type} + + {p.name} + +
+ ))} +
+ )} + {/* Parameter widgets (ComfyUI-style editable params) */} + {params.length > 0 && ( +
+ {params.map(p => { + const val = data.customNodeParams?.[p.name] ?? p.default ?? ""; + return ( +
+ + {p.description || p.name} + + {p.param_type === "select" && + Array.isArray(p.ui?.options) ? ( + + ) : p.param_type === "boolean" ? ( + setParam(p.name, e.target.checked)} + className="accent-blue-500" + /> + ) : p.param_type === "number" ? ( + setParam(p.name, Number(e.target.value))} + /> + ) : ( + setParam(p.name, e.target.value)} + /> + )} +
+ ); + })} +
+ )} +
+ )} + + {/* Input handles (left side) */} + {inputs.map((p, i) => ( + + ))} + + {/* Output handles (right side) */} + {outputs.map((p, i) => ( + 0 ? inputs.length * 18 + 4 : 0) + i * 18}px`, + width: 8, + height: 8, + ...(collapsed ? collapsedHandleStyle : {}), + }} + /> + ))} +
+ ); +} diff --git a/frontend/src/components/graph/nodes/PipelineNode.tsx b/frontend/src/components/graph/nodes/PipelineNode.tsx index 7b8408448..4cd461886 100644 --- a/frontend/src/components/graph/nodes/PipelineNode.tsx +++ b/frontend/src/components/graph/nodes/PipelineNode.tsx @@ -84,7 +84,7 @@ export function PipelineNode({ const supportsLoRA = data.supportsLoRA ?? false; const isStreaming = data.isStreaming ?? false; - const pipelineName = data.pipelineId || "Pipeline"; + const pipelineName = data.pipelineId || "Node"; // Inject unavailable pipelineId into options const isUnavailable = @@ -187,8 +187,8 @@ export function PipelineNode({ /> {!collapsed && ( - {/* Pipeline selector */} - + {/* Node type selector (video pipeline model) */} + { diff --git a/frontend/src/components/graph/nodes/SinkNode.tsx b/frontend/src/components/graph/nodes/SinkNode.tsx index 198695265..92dd2b166 100644 --- a/frontend/src/components/graph/nodes/SinkNode.tsx +++ b/frontend/src/components/graph/nodes/SinkNode.tsx @@ -23,6 +23,7 @@ export function SinkNode({ id, data, selected }: NodeProps) { const isPlaying = (data.isPlaying as boolean | undefined) ?? true; const onPlayPauseToggle = data.onPlayPauseToggle as (() => void) | undefined; const videoRef = useRef(null); + const audioRef = useRef(null); const [videoSize, setVideoSize] = useState<{ width: number; height: number; @@ -40,14 +41,26 @@ export function SinkNode({ id, data, selected }: NodeProps) { }, []); useEffect(() => { - if (videoRef.current && remoteStream instanceof MediaStream) { - videoRef.current.srcObject = remoteStream; + if (remoteStream instanceof MediaStream) { + if (videoRef.current) { + videoRef.current.srcObject = remoteStream; + } setHasAudioTrack(remoteStream.getAudioTracks().length > 0); setHasVideoTrack(remoteStream.getVideoTracks().length > 0); + // Also attach to audio element for reliable audio-only playback + if (audioRef.current) { + audioRef.current.srcObject = remoteStream; + audioRef.current.play().catch(() => {}); + } + const handleTrackAdded = () => { setHasAudioTrack(remoteStream.getAudioTracks().length > 0); setHasVideoTrack(remoteStream.getVideoTracks().length > 0); + if (audioRef.current && audioRef.current.srcObject !== remoteStream) { + audioRef.current.srcObject = remoteStream; + audioRef.current.play().catch(() => {}); + } }; remoteStream.addEventListener("addtrack", handleTrackAdded); return () => { @@ -60,8 +73,18 @@ export function SinkNode({ id, data, selected }: NodeProps) { if (videoRef.current) { videoRef.current.muted = isMuted; } + if (audioRef.current) { + audioRef.current.muted = isMuted; + } }, [isMuted]); + // Auto-unmute when audio arrives without video (audio-only workflow) + useEffect(() => { + if (hasAudioTrack && !hasVideoTrack) { + setIsMuted(false); + } + }, [hasAudioTrack, hasVideoTrack]); + const toggleMute = useCallback((e: React.MouseEvent) => { e.stopPropagation(); setIsMuted(prev => !prev); @@ -116,6 +139,13 @@ export function SinkNode({ id, data, selected }: NodeProps) { playsInline onResize={handleResize} /> + {/* Always-present audio element for reliable audio playback */} +