Repository navigation
fix(video_compose): resolve and stage edit_decisions.audio.music for Remotion renders #938
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from 1 commit
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -479,6 +479,15 @@ def _compose(self, inputs: dict[str, Any]) -> ToolResult: | |
| output_path = Path(inputs.get("output_path", "composed_output.mp4")) | ||
| output_path.parent.mkdir(parents=True, exist_ok=True) | ||
| audio_path = inputs.get("audio_path") | ||
| if not audio_path and edit_decisions: | ||
| music_cfg = ( | ||
| (edit_decisions.get("audio") or {}).get("music") | ||
| or edit_decisions.get("music") | ||
| ) | ||
| if isinstance(music_cfg, dict): | ||
| cand = music_cfg.get("src") or music_cfg.get("source") or music_cfg.get("path") | ||
| if cand and Path(cand).exists(): | ||
| audio_path = str(Path(cand).resolve()) | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [P1] This breaks the ffmpeg runtime. Setting Repro (
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Addressed in 5d34656 — dropped this block from _compose so clip audio is kept and video duration is not truncated. |
||
| subtitle_path = inputs.get("subtitle_path") | ||
| codec = inputs.get("codec", "libx264") | ||
| crf = inputs.get("crf", 23) | ||
|
|
@@ -855,6 +864,100 @@ def _cuts_to_cinematic_scenes(cuts: list[dict[str, Any]]) -> list[dict[str, Any] | |
|
|
||
| return scenes | ||
|
|
||
| @classmethod | ||
| def _resolve_music( | ||
| cls, | ||
| edit_decisions: dict[str, Any], | ||
| asset_lookup: dict[str, Any], | ||
| ) -> tuple[dict[str, Any] | None, str | None]: | ||
| """Resolve music configuration from edit_decisions and asset manifest. | ||
|
|
||
| Supports canonical `edit_decisions.audio.music` and legacy `edit_decisions.music`. | ||
| Resolves `asset_id` against `asset_lookup` to a local file path and adapts | ||
| snake_case fade settings (`fade_in_seconds`, `fade_out_seconds`) to the camelCase | ||
| properties (`fadeInSeconds`, `fadeOutSeconds`) expected by Remotion compositions. | ||
|
|
||
| Returns (resolved_music_dict, error_string). If an explicit asset_id is present | ||
| but cannot be found in asset_lookup, returns (None, error_string) to fail loudly. | ||
| """ | ||
| music_cfg = ( | ||
| (edit_decisions.get("audio") or {}).get("music") | ||
| or edit_decisions.get("music") | ||
| ) | ||
| if not music_cfg or not isinstance(music_cfg, dict): | ||
| return None, None | ||
|
|
||
| music_asset_id = music_cfg.get("asset_id") | ||
| music_source = ( | ||
| music_asset_id | ||
| or music_cfg.get("src") | ||
| or music_cfg.get("source") | ||
| or music_cfg.get("path") | ||
| ) | ||
| if not music_source: | ||
| return None, None | ||
|
|
||
| resolved_path = None | ||
| if music_source in asset_lookup: | ||
| resolved_path = asset_lookup[music_source].get("path", music_source) | ||
| elif Path(music_source).exists(): | ||
| resolved_path = str(Path(music_source).resolve()) | ||
| elif music_asset_id: | ||
| return None, f"Music asset_id {music_asset_id!r} not found in asset_manifest." | ||
| else: | ||
| resolved_path = music_source | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [P2] "Fails loudly" only covers an unknown Related: the Remotion-unavailable fallback in
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Addressed in 5d34656 — added Path(resolved_path).is_file() validation (including file:// URI support) so missing music files fail loudly with a clear error. Also forwarded resolved_edit_decisions in the Remotion-unavailable fallback in _render. |
||
|
|
||
| volume = music_cfg.get("volume", 0.15) | ||
| try: | ||
| volume = float(volume) | ||
| except (TypeError, ValueError): | ||
| volume = 0.15 | ||
|
|
||
| fade_in = music_cfg.get( | ||
| "fade_in_seconds", music_cfg.get("fadeInSeconds", 2.0) | ||
| ) | ||
| try: | ||
| fade_in = float(fade_in) | ||
| except (TypeError, ValueError): | ||
| fade_in = 2.0 | ||
|
|
||
| fade_out = music_cfg.get( | ||
| "fade_out_seconds", music_cfg.get("fadeOutSeconds", 3.0) | ||
| ) | ||
| try: | ||
| fade_out = float(fade_out) | ||
| except (TypeError, ValueError): | ||
| fade_out = 3.0 | ||
|
|
||
| resolved: dict[str, Any] = { | ||
| "src": str(resolved_path), | ||
| "volume": volume, | ||
| "fadeInSeconds": fade_in, | ||
| "fadeOutSeconds": fade_out, | ||
| "fade_in_seconds": fade_in, | ||
| "fade_out_seconds": fade_out, | ||
| } | ||
| if music_asset_id: | ||
| resolved["asset_id"] = music_asset_id | ||
| if "ducking" in music_cfg: | ||
| resolved["ducking"] = music_cfg["ducking"] | ||
| if "loop" in music_cfg: | ||
| resolved["loop"] = music_cfg["loop"] | ||
| if "trim_before_seconds" in music_cfg or "trimBeforeSeconds" in music_cfg: | ||
| resolved["trimBeforeSeconds"] = music_cfg.get( | ||
| "trim_before_seconds", music_cfg.get("trimBeforeSeconds") | ||
| ) | ||
| if "trim_after_seconds" in music_cfg or "trimAfterSeconds" in music_cfg: | ||
| resolved["trimAfterSeconds"] = music_cfg.get( | ||
| "trim_after_seconds", music_cfg.get("trimAfterSeconds") | ||
| ) | ||
| if "offset_seconds" in music_cfg or "offsetSeconds" in music_cfg: | ||
| resolved["offsetSeconds"] = music_cfg.get( | ||
| "offset_seconds", music_cfg.get("offsetSeconds") | ||
| ) | ||
|
|
||
| return resolved, None | ||
|
|
||
| @staticmethod | ||
| def _file_uri_to_raw_path(uri: str) -> str: | ||
| """Convert a ``file:`` URI to a raw filesystem path string. | ||
|
|
@@ -1684,9 +1787,21 @@ def _render(self, inputs: dict[str, Any]) -> ToolResult: | |
| resolved_cut["source"] = asset_lookup[source_id]["path"] | ||
| resolved_cuts.append(resolved_cut) | ||
|
|
||
| # Resolve music config from edit_decisions | ||
| resolved_music, music_err = self._resolve_music(edit_decisions, asset_lookup) | ||
| if music_err: | ||
| return ToolResult(success=False, error=music_err) | ||
|
|
||
| resolved_edit_decisions = dict(edit_decisions, cuts=resolved_cuts) | ||
| if resolved_music: | ||
| resolved_edit_decisions["music"] = resolved_music | ||
| audio_dict = dict(resolved_edit_decisions.get("audio") or {}) | ||
| audio_dict["music"] = resolved_music | ||
| resolved_edit_decisions["audio"] = audio_dict | ||
|
|
||
| # --- Pre-compose validation gate --- | ||
| scene_plan = inputs.get("scene_plan") | ||
| validation_block = self._pre_compose_validation(edit_decisions, resolved_cuts, scene_plan) | ||
| validation_block = self._pre_compose_validation(resolved_edit_decisions, resolved_cuts, scene_plan) | ||
| if validation_block is not None: | ||
| return validation_block | ||
|
|
||
|
|
@@ -1696,7 +1811,7 @@ def _render(self, inputs: dict[str, Any]) -> ToolResult: | |
| if render_runtime == "hyperframes": | ||
| return self._render_via_hyperframes( | ||
| inputs=inputs, | ||
| edit_decisions=edit_decisions, | ||
| edit_decisions=resolved_edit_decisions, | ||
| asset_manifest=asset_manifest, | ||
| resolved_cuts=resolved_cuts, | ||
| output_path=output_path, | ||
|
|
@@ -1706,15 +1821,15 @@ def _render(self, inputs: dict[str, Any]) -> ToolResult: | |
| # Caller explicitly asked for FFmpeg — don't auto-upgrade to Remotion. | ||
| return self._render_via_ffmpeg( | ||
| inputs=inputs, | ||
| edit_decisions=edit_decisions, | ||
| edit_decisions=resolved_edit_decisions, | ||
| resolved_cuts=resolved_cuts, | ||
| output_path=output_path, | ||
| profile=profile, | ||
| ) | ||
| # --- Explicit Remotion path (render_runtime == 'remotion') --- | ||
| if self._needs_remotion(resolved_cuts): | ||
| remotion_inputs: dict[str, Any] = { | ||
| "edit_decisions": dict(edit_decisions, cuts=resolved_cuts), | ||
| "edit_decisions": resolved_edit_decisions, | ||
| "output_path": str(output_path), | ||
| } | ||
| if profile: | ||
|
|
@@ -2040,6 +2155,22 @@ def _remotion_render(self, inputs: dict[str, Any]) -> ToolResult: | |
| # Deep-copy props so we don't mutate the original | ||
| props = json.loads(json.dumps(composition_data)) | ||
|
|
||
| # Adapt audio.music <-> top-level music and snake_case <-> camelCase for Remotion | ||
| music_block = (props.get("audio") or {}).get("music") or props.get("music") | ||
| if isinstance(music_block, dict): | ||
| if "fade_in_seconds" in music_block and "fadeInSeconds" not in music_block: | ||
| music_block["fadeInSeconds"] = music_block["fade_in_seconds"] | ||
| if "fade_out_seconds" in music_block and "fadeOutSeconds" not in music_block: | ||
| music_block["fadeOutSeconds"] = music_block["fade_out_seconds"] | ||
| if "source" in music_block and "src" not in music_block: | ||
| music_block["src"] = music_block["source"] | ||
| if "music" not in props: | ||
| props["music"] = music_block | ||
| if "audio" not in props or not isinstance(props["audio"], dict): | ||
| props["audio"] = {} | ||
| if "music" not in props["audio"]: | ||
| props["audio"]["music"] = music_block | ||
|
|
||
| # Build a custom themeConfig from the playbook's actual colors. | ||
| # This ensures every video gets a unique visual identity derived | ||
| # from its production decisions — not picked from a preset menu. | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
[P2] This test fails in CI. It goes through
_render->_needs_remotion->_remotion_available(), which requiresremotion-composer/node_modules. CI installs Python deps only, so the render falls back to ffmpeg and the fakerun_commandhitsStopIteration(empty error string). On a Python-only container: 12 passed, 1 failed in this file; withnode_modulesinstalled: 13 passed.Fix: add
monkeypatch.setattr(VideoCompose, "_remotion_available", lambda self: True)at the top of the test (the other render tests in this file call_remotion_renderdirectly, which is why they are unaffected).There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Addressed in 5d34656 — added monkeypatch for _remotion_available to return True so the test passes cleanly in Python-only CI environments without node_modules.