Skip to content

Improve the performance of dissolve transitions #1343

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
iniw wants to merge 7 commits into
base: master
Choose a base branch
from
Open

Improve the performance of dissolve transitions #1343

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
5b1a030
luma: don't convert opaque frames to rgba for fix_background_alpha
iniw Oct 13, 2026
38299b2
luma: dissolve opaque yuv422 frames in slices
iniw Oct 13, 2026
951c582
avformat: convert yuv420p to yuyv422 without libswscale
iniw Oct 13, 2026
3be22b7
avformat: copy the probed stream information to the audio context
iniw Oct 13, 2026
cfc6cc2
avformat: copy the probed stream information to the seek-test context
iniw Oct 13, 2026
4ac17a0
framework: copy large images on several threads
iniw Oct 13, 2026
25a34c4
avformat: find the error concealment frame in the image cache
iniw Oct 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 21 additions & 40 deletions src/modules/avformat/common.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -239,60 +239,41 @@ mlt_channel_layout mlt_get_channel_layout_or_default(const char *name, int chann
return layout;
}

const int *mlt_get_sws_coefficients(mlt_colorspace colorspace)
{
switch (colorspace) {
case mlt_colorspace_smpte170m:
case mlt_colorspace_bt470bg:
case mlt_colorspace_bt601:
return sws_getCoefficients(SWS_CS_ITU601);
case mlt_colorspace_smpte240m:
return sws_getCoefficients(SWS_CS_SMPTE240M);
case mlt_colorspace_bt709:
return sws_getCoefficients(SWS_CS_ITU709);
case mlt_colorspace_bt2020_ncl:
case mlt_colorspace_bt2020_cl:
return sws_getCoefficients(SWS_CS_BT2020);
default:
return sws_getCoefficients(SWS_CS_DEFAULT);
}
}

int mlt_set_luma_transfer(struct SwsContext *context,
mlt_colorspace src_colorspace,
mlt_colorspace dst_colorspace,
int src_full_range,
int dst_full_range)
{
const int *src_coefficients = sws_getCoefficients(SWS_CS_DEFAULT);
const int *dst_coefficients = sws_getCoefficients(SWS_CS_DEFAULT);
int brightness = 0;
int contrast = 1 << 16;
int saturation = 1 << 16;
int src_range = src_full_range ? 1 : 0;
int dst_range = dst_full_range ? 1 : 0;

switch (src_colorspace) {
case mlt_colorspace_smpte170m:
case mlt_colorspace_bt470bg:
case mlt_colorspace_bt601:
src_coefficients = sws_getCoefficients(SWS_CS_ITU601);
break;
case mlt_colorspace_smpte240m:
src_coefficients = sws_getCoefficients(SWS_CS_SMPTE240M);
break;
case mlt_colorspace_bt709:
src_coefficients = sws_getCoefficients(SWS_CS_ITU709);
break;
case mlt_colorspace_bt2020_ncl:
case mlt_colorspace_bt2020_cl:
src_coefficients = sws_getCoefficients(SWS_CS_BT2020);
default:
break;
}
switch (dst_colorspace) {
case mlt_colorspace_smpte170m:
case mlt_colorspace_bt470bg:
case mlt_colorspace_bt601:
dst_coefficients = sws_getCoefficients(SWS_CS_ITU601);
break;
case mlt_colorspace_smpte240m:
dst_coefficients = sws_getCoefficients(SWS_CS_SMPTE240M);
break;
case mlt_colorspace_bt709:
dst_coefficients = sws_getCoefficients(SWS_CS_ITU709);
break;
case mlt_colorspace_bt2020_ncl:
case mlt_colorspace_bt2020_cl:
dst_coefficients = sws_getCoefficients(SWS_CS_BT2020);
default:
break;
}
return sws_setColorspaceDetails(context,
src_coefficients,
mlt_get_sws_coefficients(src_colorspace),
src_range,
dst_coefficients,
mlt_get_sws_coefficients(dst_colorspace),
dst_range,
brightness,
contrast,
Expand Down
1 change: 1 addition & 0 deletions src/modules/avformat/common.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ mlt_channel_layout av_channel_layout_to_mlt(AVChannelLayout *layout);
mlt_channel_layout av_channel_layout_to_mlt(int64_t layout);
#endif
mlt_channel_layout mlt_get_channel_layout_or_default(const char *name, int channels);
const int *mlt_get_sws_coefficients(mlt_colorspace colorspace);
int mlt_set_luma_transfer(struct SwsContext *context,
mlt_colorspace src_colorspace,
mlt_colorspace dst_colorspace,
Expand Down
84 changes: 82 additions & 2 deletions src/modules/avformat/producer_avformat.c
View file
Open in desktop

@bmatherly bmatherly Oct 14, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As Dan said, we can not put conversion code in this file. But if SWS is not returning correct chroma location, we should figure that out. Is it not calling mlt_get_sws_flags() as the scaler does. Also, if creating multiple sws contexts is expensive, maybe we could find a way to preserve them - like adding the conversion to the filtergraph that is sometimes created. I'm not sure - I haven't thought about it that much.

Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,9 @@

// System header files
#include <limits.h>
#ifdef __SSE2__
#include <emmintrin.h>
#endif
#include <math.h>
#include <pthread.h>
#include <stdatomic.h>
Expand Down Expand Up @@ -2241,6 +2244,72 @@ static int sliced_h_pix_fmt_conv_proc(int id, int idx, int jobs, void *cookie)
return 0;
}

/** Convert progressive 8-bit 4:2:0 to packed 4:2:2 without libswscale.
*
* In 4:2:0 video such as H.264 and MPEG-2, each chroma line lies between two luma
* lines, so each output line mixes the nearest chroma line (3/4) and the next nearest
* one (1/4). For this conversion, libswscale runs its general scaler, which is much
* slower than this loop, and it places the chroma half a line too low.
*/
static int sliced_yuv420p_to_yuyv422_proc(int id, int idx, int jobs, void *cookie)
{
struct sliced_pix_fmt_conv_t *ctx = (struct sliced_pix_fmt_conv_t *) cookie;
AVFrame *frame = ctx->frame;
int half_width = ctx->width / 2;
int chroma_height = (ctx->height + 1) / 2;
int start;
int lines = mlt_slices_size_slice(jobs, idx, ctx->height, &start);

for (int y = start; y < start + lines; y++) {
int near_line = y / 2;
int far_line = (y & 1) ? FFMIN(near_line + 1, chroma_height - 1) : FFMAX(near_line - 1, 0);
const uint8_t *restrict luma = frame->data[0] + y * frame->linesize[0];
const uint8_t *restrict u_near = frame->data[1] + near_line * frame->linesize[1];
const uint8_t *restrict u_far = frame->data[1] + far_line * frame->linesize[1];
const uint8_t *restrict v_near = frame->data[2] + near_line * frame->linesize[2];
const uint8_t *restrict v_far = frame->data[2] + far_line * frame->linesize[2];
uint8_t *restrict out = ctx->out_data[0] + y * ctx->out_stride[0];

int x = 0;
#ifdef __SSE2__
// 32 pixels at a time: mix 16 chroma samples of each plane in 16 bits,
// then interleave them with the luma.
const __m128i zero = _mm_setzero_si128();
const __m128i two = _mm_set1_epi16(2);
for (; x + 16 <= half_width; x += 16) {
__m128i un = _mm_loadu_si128((const __m128i *) (u_near + x));
__m128i uf = _mm_loadu_si128((const __m128i *) (u_far + x));
__m128i vn = _mm_loadu_si128((const __m128i *) (v_near + x));
__m128i vf = _mm_loadu_si128((const __m128i *) (v_far + x));
#define MIX(n, f) \
_mm_srli_epi16(_mm_add_epi16(_mm_add_epi16(_mm_add_epi16(n, n), n), _mm_add_epi16(f, two)), 2)
__m128i u
= _mm_packus_epi16(MIX(_mm_unpacklo_epi8(un, zero), _mm_unpacklo_epi8(uf, zero)),
MIX(_mm_unpackhi_epi8(un, zero), _mm_unpackhi_epi8(uf, zero)));
__m128i v
= _mm_packus_epi16(MIX(_mm_unpacklo_epi8(vn, zero), _mm_unpacklo_epi8(vf, zero)),
MIX(_mm_unpackhi_epi8(vn, zero), _mm_unpackhi_epi8(vf, zero)));
#undef MIX
__m128i uv_lo = _mm_unpacklo_epi8(u, v);
__m128i uv_hi = _mm_unpackhi_epi8(u, v);
__m128i y_lo = _mm_loadu_si128((const __m128i *) (luma + 2 * x));
__m128i y_hi = _mm_loadu_si128((const __m128i *) (luma + 2 * x + 16));
_mm_storeu_si128((__m128i *) (out + 4 * x), _mm_unpacklo_epi8(y_lo, uv_lo));
_mm_storeu_si128((__m128i *) (out + 4 * x + 16), _mm_unpackhi_epi8(y_lo, uv_lo));
_mm_storeu_si128((__m128i *) (out + 4 * x + 32), _mm_unpacklo_epi8(y_hi, uv_hi));
_mm_storeu_si128((__m128i *) (out + 4 * x + 48), _mm_unpackhi_epi8(y_hi, uv_hi));
}
#endif
for (; x < half_width; x++) {
out[4 * x] = luma[2 * x];
out[4 * x + 1] = (3 * u_near[x] + u_far[x] + 2) >> 2;
out[4 * x + 2] = luma[2 * x + 1];
out[4 * x + 3] = (3 * v_near[x] + v_far[x] + 2) >> 2;
}
}
return 0;
}

static mlt_colorspace convert_image_yuvp(producer_avformat self,
mlt_profile profile,
AVFrame *frame,
Expand Down Expand Up @@ -2598,8 +2667,19 @@ static void convert_image(producer_avformat self,
c = (width + ctx.slice_w - 1) / ctx.slice_w;
int last_slice_w = width - ctx.slice_w * (c - 1);

if (sliced && (last_slice_w % 8) == 0
&& !(ctx.src_format == AV_PIX_FMT_YUV422P && last_slice_w % 16)) {
#if LIBAVUTIL_VERSION_INT >= ((58 << 16) + (7 << 8) + 100)
int interlaced = frame->flags & AV_FRAME_FLAG_INTERLACED;
#else
int interlaced = frame->interlaced_frame;
#endif
if (sliced && !interlaced && width % 2 == 0
&& (src_pix_fmt == AV_PIX_FMT_YUV420P || src_pix_fmt == AV_PIX_FMT_YUVJ420P)
&& self->full_range == dst_full_range
&& mlt_get_sws_coefficients(ctx.src_colorspace)
== mlt_get_sws_coefficients(ctx.dst_colorspace)) {
mlt_slices_run_normal(0, sliced_yuv420p_to_yuyv422_proc, &ctx);
} else if (sliced && (last_slice_w % 8) == 0
&& !(ctx.src_format == AV_PIX_FMT_YUV422P && last_slice_w % 16)) {
#if LIBAVUTIL_VERSION_INT >= ((58 << 16) + (7 << 8) + 100)
c *= (frame->flags & AV_FRAME_FLAG_INTERLACED) ? 2 : 1;
#else
Expand Down
11 changes: 11 additions & 0 deletions src/modules/core/filter_resize.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,17 @@ static uint8_t *frame_resize_image(mlt_frame frame, int owidth, int oheight, mlt
// Now update the frame
mlt_frame_set_image(frame, output, owidth * (oheight + 1) * bpp, mlt_pool_release);

// An opaque image without an alpha mask gets one, so that the padding can be
// transparent. This gives the same result as padding an rgba image, without
// converting the image to rgba and back.
if (format == mlt_image_yuv422 && !alpha
&& mlt_properties_get_int(properties, "fix_background_alpha")) {

@bmatherly bmatherly Oct 14, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This creates an undocumented coupling between transtion_luma and filter_resize. And "fix_background_alpha" doesn't mean anything in this filter. Can you think of a different way to accomplish this? I wonder if we could just infer that the alpha should be created if resize_alpha != 255. In fact, if we did this, it would remove the need for the fix_background_alpha property on transition_luma.

@ddennedy ddennedy Oct 15, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I agree, but the default for "resize_alpha" is 0 since it does not exist by default. Only certain things set it to 255. So, the logic should be to create the alpha if it does not exist and the "resize_alpha" property exists on the frame. And use the property value instead of hard-code 255 in the memset.

The "fix_background_alpha" property exists on transition luma to version the behavior. Are you saying that 3.5 years is long enough, and the fidelity of old projects no longer relevant?

@bmatherly bmatherly Oct 15, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think the "fix" property has gone on lone enough. If there are old projects that currently do not set fix_background_alpha=1, then they are getting inconsistent results depending on whether an upstream service is requesting RGB or YUV. In my opinion, that is a bug that should be fixed.

Current Behavior:

  • If an upstream service is requesting RGB, then they are getting transparent alpha every time.
  • If an upstream service is requesting YUV, then they are getting opaque or transparent depending on fix_background_alpha.

Proposed Behavior:

  • The transition should operate on transparent background regardless of the format requested

That could be accomplished by always setting "resize_alpha=0" in the transition and then conditionally creating the alpha in filter_resize if the "resize_alpha" property exists and not 255.

alpha_size = iwidth * iheight;
alpha = mlt_pool_alloc(alpha_size);
memset(alpha, 255, alpha_size);
mlt_frame_set_alpha(frame, alpha, alpha_size, mlt_pool_release);
}

// We should resize the alpha too
if (format != mlt_image_rgba && alpha && alpha_size >= iwidth * iheight) {
alpha = resize_alpha(alpha, owidth, oheight, iwidth, iheight, alpha_value);
Expand Down
126 changes: 97 additions & 29 deletions src/modules/core/transition_luma.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,60 @@ static int dissolve_slice_yuv422(int id, int index, int count, void *context)
return 0;
}

// Gives a frame an opaque alpha mask, as converting it to rgba and back does.
static uint8_t *add_opaque_alpha(mlt_frame frame, int width, int height)
{
int size = width * height;
uint8_t *alpha = mlt_pool_alloc(size);
memset(alpha, 255, size);
mlt_frame_set_alpha(frame, alpha, size, mlt_pool_release);
return alpha;
}

struct dissolve_opaque_context
{
uint8_t *dst_image;
uint8_t *src_image;
uint8_t *dst_alpha;
uint8_t *src_alpha;
int dst_width;
int src_width;
int height;
int mix;
};

static int dissolve_opaque_slice_yuv422(int id, int index, int count, void *context)
{
struct dissolve_opaque_context ctx = *((struct dissolve_opaque_context *) context);
int slice_start, slice_height = mlt_slices_size_slice(count, index, ctx.height, &slice_start);

ctx.dst_image += slice_start * (ctx.dst_width << 1);
ctx.src_image += slice_start * (ctx.src_width << 1);
if (ctx.dst_alpha)
ctx.dst_alpha += slice_start * ctx.dst_width;
if (ctx.src_alpha)
ctx.src_alpha += slice_start * ctx.src_width;

for (int i = 0; i < slice_height; i++) {
composite_line_yuv(ctx.dst_image,

@bmatherly bmatherly Oct 14, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is the only difference between this function and dissolve_slice_yuv422 that it calls composite_line_yuv() instead of composite_line_yuv_float()? If so, what is the difference? Could everything use the same dissolve slice function and simplify this whole file?

ctx.src_image,
ctx.src_width,
ctx.src_alpha,
ctx.dst_alpha,
ctx.mix,
NULL,
0,
0);
ctx.dst_image += ctx.dst_width << 1;
ctx.src_image += ctx.src_width << 1;
if (ctx.dst_alpha)
ctx.dst_alpha += ctx.dst_width;
if (ctx.src_alpha)
ctx.src_alpha += ctx.src_width;
}
return 0;
}

static inline int dissolve_yuv422(mlt_frame frame,
mlt_frame that,
float weight,
Expand All @@ -138,29 +192,36 @@ static inline int dissolve_yuv422(mlt_frame frame,
int fix_background_alpha)
{
int ret = 0;
int i = height + 1;
int width_src = width, height_src = height;
mlt_image_format format = (fix_background_alpha && mlt_frame_has_convert_image(frame))
? mlt_image_rgba
: mlt_image_yuv422;
mlt_image_format format = mlt_image_yuv422;
uint8_t *p_src, *p_dest;
uint8_t *alpha_src;
uint8_t *alpha_dst;
int mix = weight * (1 << 16);

if (fix_background_alpha) {
// Tell filter_resize to make the padding transparent (see filter_resize.c).
mlt_properties_set_int(MLT_FRAME_PROPERTIES(frame), "fix_background_alpha", 1);
mlt_properties_set_int(MLT_FRAME_PROPERTIES(that), "fix_background_alpha", 1);
}

mlt_frame_get_image(frame, &p_dest, &format, &width, &height, 1);
if (fix_background_alpha && mlt_frame_has_convert_image(frame))
mlt_frame_convert_image(frame, &p_dest, &format, mlt_image_yuv422);
alpha_dst = mlt_frame_get_alpha(frame);
if (fix_background_alpha && mlt_frame_has_convert_image(that))
format = mlt_image_rgba;
format = mlt_image_yuv422;
mlt_frame_get_image(that, &p_src, &format, &width_src, &height_src, 0);
if (mlt_frame_has_convert_image(that))
mlt_frame_convert_image(that, &p_src, &format, mlt_image_yuv422);
alpha_src = mlt_frame_get_alpha(that);
int is_translucent = (alpha_dst && !is_opaque(alpha_dst, width, height))
|| (alpha_src && !is_opaque(alpha_src, width_src, height_src));

// An opaque frame without an alpha mask must get one when the other frame is
// translucent, so that the alpha of the result is computed (see alpha_over).
if (fix_background_alpha && is_translucent) {
if (!alpha_dst)
alpha_dst = add_opaque_alpha(frame, width, height);
if (!alpha_src)
alpha_src = add_opaque_alpha(that, width_src, height_src);
}

// Pick the lesser of two evils ;-)
width_src = width_src > width ? width : width_src;
height_src = height_src > height ? height : height_src;
Expand All @@ -175,15 +236,15 @@ static inline int dissolve_yuv422(mlt_frame frame,
.weight = weight};
mlt_slices_run_normal(threads, dissolve_slice_yuv422, &context);
} else {
while (--i) {
composite_line_yuv(p_dest, p_src, width_src, alpha_src, alpha_dst, mix, NULL, 0, 0);
p_src += width_src << 1;
p_dest += width << 1;
if (alpha_src)
alpha_src += width_src;
if (alpha_dst)
alpha_dst += width;
}
struct dissolve_opaque_context context = {.dst_image = p_dest,
.src_image = p_src,
.dst_alpha = alpha_dst,
.src_alpha = alpha_src,
.dst_width = width,
.src_width = width_src,
.height = height,
.mix = mix};
mlt_slices_run_normal(threads, dissolve_opaque_slice_yuv422, &context);
}

return ret;
Expand Down Expand Up @@ -341,29 +402,27 @@ static void luma_composite_yuv422(mlt_frame a_frame,
{
int width_src = *width, height_src = *height;
int width_dest = *width, height_dest = *height;
mlt_image_format format_src = (fix_background_alpha && mlt_frame_has_convert_image(a_frame))
? mlt_image_rgba
: mlt_image_yuv422;
mlt_image_format format_dest = (fix_background_alpha && mlt_frame_has_convert_image(b_frame))
? mlt_image_rgba
: mlt_image_yuv422;
mlt_image_format format_src = mlt_image_yuv422;
mlt_image_format format_dest = mlt_image_yuv422;
uint8_t *p_src, *p_dest;
uint8_t *alpha_src, *alpha_dest;
int i, j;
int stride_src;
int stride_dest;

if (fix_background_alpha) {
// Tell filter_resize to make the padding transparent (see filter_resize.c).
mlt_properties_set_int(MLT_FRAME_PROPERTIES(a_frame), "fix_background_alpha", 1);
mlt_properties_set_int(MLT_FRAME_PROPERTIES(b_frame), "fix_background_alpha", 1);
}

if (mlt_properties_get(&a_frame->parent, "distort"))
mlt_properties_set(&b_frame->parent,
"distort",
mlt_properties_get(&a_frame->parent, "distort"));
mlt_frame_get_image(a_frame, &p_dest, &format_dest, &width_dest, &height_dest, 1);
if (fix_background_alpha && mlt_frame_has_convert_image(a_frame))
mlt_frame_convert_image(a_frame, &p_dest, &format_dest, mlt_image_yuv422);
alpha_dest = mlt_frame_get_alpha(a_frame);
mlt_frame_get_image(b_frame, &p_src, &format_src, &width_src, &height_src, 0);
if (fix_background_alpha && mlt_frame_has_convert_image(b_frame))
mlt_frame_convert_image(b_frame, &p_src, &format_src, mlt_image_yuv422);
alpha_src = mlt_frame_get_alpha(b_frame);

if (*width == 0 || *height == 0)
Expand All @@ -372,6 +431,15 @@ static void luma_composite_yuv422(mlt_frame a_frame,
int is_translucent = (alpha_dest && !is_opaque(alpha_dest, width_dest, height_dest))
|| (alpha_src && !is_opaque(alpha_src, width_src, height_src));

// An opaque frame without an alpha mask must get one when the other frame is
// translucent, so that the alpha of the result is computed (see alpha_over).
if (fix_background_alpha && is_translucent) {
if (!alpha_dest)
alpha_dest = add_opaque_alpha(a_frame, width_dest, height_dest);
if (!alpha_src)
alpha_src = add_opaque_alpha(b_frame, width_src, height_src);
}

// Pick the lesser of two evils ;-)
width_src = width_src > width_dest ? width_dest : width_src;
height_src = height_src > height_dest ? height_dest : height_src;
Expand Down