From a8675ed43ee963ff97bd9ec3d280165635ae3a24 Mon Sep 17 00:00:00 2001 From: Eugene Morozov Date: Fri, 30 Oct 2026 12:13:05 -0700 Subject: [PATCH] NAT-551: Fix edit-list inspection and frame-rate precision --- .../standardization/MediaSampleInspector.kt | 5 ++- .../StandardInputTimelineInspector.kt | 42 ++++++++++++++----- .../standardization/MediaSampleFactsTests.kt | 18 ++++++++ .../StandardInputTimelineInspectorTests.kt | 38 +++++++++++++++++ 4 files changed, 91 insertions(+), 12 deletions(-) diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/MediaSampleInspector.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/MediaSampleInspector.kt index 1ba860d4..084a42d5 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/MediaSampleInspector.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/MediaSampleInspector.kt @@ -113,7 +113,10 @@ internal object VideoSampleFactsReader { ordered.sort() val intervals = LongArray(ordered.size - 1) { ordered[it + 1] - ordered[it] } if (intervals.any { it <= 0 }) return base - val span = (ordered.last() - ordered.first()) / 1e6 + // Keep proven container precision at policy boundaries such as 60/120 fps. + // Rounding each endpoint to microseconds first can turn exactly 60 into 60.000004. + val span = presented.maxOf { maxOf(timeline.startSeconds, time(it)) } - + presented.minOf { maxOf(timeline.startSeconds, time(it)) } if (!span.isFinite() || span <= 0) return base val frameInterval = span / (ordered.size - 1) val constant = intervals.all { abs(it / 1e6 - frameInterval) <= maxOf(2e-6, frameInterval * 0.001) } diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspector.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspector.kt index 50e04a79..6a5ccac3 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspector.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspector.kt @@ -6,6 +6,7 @@ import java.io.File import java.io.RandomAccessFile import java.util.concurrent.CancellationException import kotlin.math.abs +import kotlin.math.round internal sealed interface AudioVideoStartOffset { data object NotApplicable : AudioVideoStartOffset @@ -121,9 +122,11 @@ internal data class IsoTrackTimeline( val effectivePresentationSeconds: List, val hasEdit: Boolean, val timescale: Long, + /** Media timestamp minus effective timestamp, including any leading empty-edit gap. */ val mediaStartSeconds: Double, /** Decode-order indices of samples whose presentation intervals overlap the effective edit. */ val effectiveSampleIndices: List, + val leadingGapRoundingSeconds: Double = 0.0, ) /** A single table-to-platform timestamp convention must agree for the entire track. */ @@ -132,6 +135,7 @@ internal class PlatformTimestampMatcher(private val timing: IsoTrackTimeline) { private var raw = true private var clamped = true private var firstSampleShifted = true + private var roundedGapShifted = true fun matches(index: Int, actual: Double): Boolean { val effective = timing.presentationSeconds[index] @@ -144,7 +148,10 @@ internal class PlatformTimestampMatcher(private val timing: IsoTrackTimeline) { clamped = clamped && near(maxOf(0.0, effective)) // Android Q limits the video shift to the first composition time and clamps early CTS. firstSampleShifted = firstSampleShifted && near(maxOf(0.0, unshifted - legacyShift)) - return shifted || raw || clamped || firstSampleShifted + // Some extractors round a leading movie-time gap to the nearest media tick. + // Require that same shift for every sample rather than widening timestamp tolerances. + roundedGapShifted = roundedGapShifted && near(effective + timing.leadingGapRoundingSeconds) + return shifted || raw || clamped || firstSampleShifted || roundedGapShifted } } @@ -235,9 +242,10 @@ internal object IsoTimelineReader { } val ends = LongArray(sampleCount) { checkedAdd(starts[it], deltas[it]) } require(starts.distinct().size == sampleCount) - // Single unit-rate edits cover phone encoder priming and simple trims. Complex edits are - // unproven here, including empty leading edits; do not guess how the platform applies them. + // Prove one unit-rate media edit, optionally preceded by one empty edit for an A/V gap. + // Other edit sequences remain unproven; the full platform scan must match this timeline. var mediaStart = 0.0 + var leadingGap = 0.0 var editDuration: Double? = null val editBoxes = trackChildren.filter { it.type == "edts" } require(editBoxes.size <= 1) @@ -248,31 +256,43 @@ internal object IsoTimelineReader { val version = input.readUnsignedByte() require(version in 0..1 && input.readUnsignedByte() == 0 && input.readUnsignedShort() == 0) val count = uint() - if (count > 1) throw SampleScanStop(SampleScanStatus.Unsupported) + if (count > 2) throw SampleScanStop(SampleScanStatus.Unsupported) require(list.end - list.payload == 8 + count * (if (version == 0) 12 else 20)) - if (count == 1L) { + fun entry(): Pair { val duration = if (version == 0) uint() else input.readLong() val time = if (version == 0) input.readInt().toLong() else input.readLong() - require(duration > 0 && time >= 0 && input.readInt() == 65536) + require(duration > 0 && input.readInt() == 65536) + return duration to time + } + if (count == 2L) { + val (duration, time) = entry() + if (time != -1L) throw SampleScanStop(SampleScanStatus.Unsupported) + leadingGap = duration.toDouble() / movieScale + } + if (count > 0) { + val (duration, time) = entry() + require(time >= 0) mediaStart = time.toDouble() / scale editDuration = duration.toDouble() / movieScale } } val rawStart = starts.min().toDouble() / scale val rawEnd = ends.max().toDouble() / scale - val start = maxOf(0.0, rawStart - mediaStart) - val end = editDuration ?: rawEnd + val start = maxOf(0.0, rawStart - mediaStart) + leadingGap + val end = (editDuration ?: rawEnd) + leadingGap require(start.isFinite() && end.isFinite() && end > start) if (editDuration != null) { // Table coverage must reach the edit endpoint; movie/track timescale rounding is bounded. require(rawStart <= mediaStart + 2.0 / scale && - rawEnd + 2.0 / scale + 1.0 / movieScale >= mediaStart + end) + rawEnd + 2.0 / scale + 1.0 / movieScale >= mediaStart + editDuration) } else require(rawStart >= 0) - val platformTimes = DoubleArray(sampleCount) { starts[it].toDouble() / scale - mediaStart } + val platformTimes = DoubleArray(sampleCount) { starts[it].toDouble() / scale - mediaStart + leadingGap } val effectiveIndices = starts.indices.filter { ends[it].toDouble() / scale > mediaStart && platformTimes[it] < end } val effectiveTimes = effectiveIndices.map { maxOf(start, platformTimes[it]) }.sorted() require(effectiveTimes.isNotEmpty()) - trackId to IsoTrackTimeline(platformTimes, start, end, rawEnd - mediaStart, effectiveTimes, editDuration != null, scale, mediaStart, effectiveIndices) + trackId to IsoTrackTimeline(platformTimes, start, end, rawEnd - mediaStart + leadingGap, effectiveTimes, + editDuration != null, scale, mediaStart - leadingGap, effectiveIndices, + round(leadingGap * scale) / scale - leadingGap) }.also { require(it.map { track -> track.first }.distinct().size == it.size) }.toMap() } } diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/MediaSampleFactsTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/MediaSampleFactsTests.kt index 97375687..8bde2052 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/MediaSampleFactsTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/MediaSampleFactsTests.kt @@ -221,6 +221,24 @@ class MediaSampleFactsTests : AbsRobolectricTest() { IsoTrackTimeline(times.toDoubleArray(), 0.0, end, fullEnd, presented.map { maxOf(0.0, times[it]) }.sorted(), true, 1000, 1.0, presented) + @Test fun exactPolicyBoundaryRatesUseContainerPrecisionBeforeMicrosecondRounding() { + for (rate in listOf(60.0, 120.0, 60.01)) { + val times = List(300) { it / rate } + val time = editedTimeline(times, 300 / rate, times.indices.toList()).copy(hasEdit = false, + mediaStartSeconds = 0.0, timescale = 60000) + val facts = VideoSampleFactsReader.read(List(300) { CompressedVideoSample(100000, it == 0, idr) }, + time, compliantFacts(dimensions = Dimensions(3840, 2160))) + assertEquals(rate, facts.frameRate.valueOrNull!!, 1e-12) + assertEquals(known(Cadence.Constant), facts.cadence) + if (rate == 60.0) { + val plan = StandardInputPlanner().plan(facts, options(resolution = com.mux.video.upload.internal.MaximumResolution.Preset3840x2160), fullCapabilities()) + assertFalse(plan.evaluation.nonCompliantRequirements.contains(PolicyRequirement.FrameRate)) + val conversion = (plan.action as StandardInputAction.Convert).conversion + assertEquals(OutputCadence.PreserveSourceTimestamps, conversion.outputCadence) + assertEquals(60.0, conversion.outputFrameRate, 1e-12) + } + } + } @Test fun effectiveEditCountsPrerollBytesAndRetainsTheIdrProof() { val samples = listOf(sample(-1_000_000, true, 1000), sample(0, bytes = 100), sample(1_000_000, bytes = 100)) val time = editedTimeline(listOf(-1.0, 0.0, 1.0), 2.0, listOf(1, 2)) diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspectorTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspectorTests.kt index 5d34b5a4..a47ad5f7 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspectorTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspectorTests.kt @@ -60,6 +60,44 @@ class StandardInputTimelineInspectorTests : AbsRobolectricTest() { assertEquals(2.0, time.endSeconds, 0.0) } } + @Test fun leadingEmptyAudioEditPreservesStartOffsetAndPlatformTimestampConventions() { + for (version in 0..1) { + fun entry(duration: Long, start: Long) = + (if (version == 0) int(duration.toInt()) + int(start.toInt()) else long(duration) + long(start)) + int(65536) + val edits = box("edts", box("elst", byteArrayOf(version.toByte(), 0, 0, 0) + int(2) + + entry(56, -1) + entry(30000, 0))) + val source = file(track() + track(kind = "soun", edits = edits), scale = 10000) + try { + val tracks = IsoTimelineReader.read(source, 250000) + val audio = tracks.getValue(2) + assertEquals(0.0056, audio.startSeconds - tracks.getValue(1).startSeconds, 1e-12) + assertEquals(3.0056, audio.endSeconds, 1e-12) + assertEquals(3.0, audio.endSeconds - audio.startSeconds, 1e-12) + audio.effectivePresentationSeconds.zip(listOf(0.0056, 1.0056, 2.0056)).forEach { + (actual, expected) -> assertEquals(expected, actual, 1e-12) + } + for (shift in listOf(0.0, 0.0056, 0.006)) { + val matcher = PlatformTimestampMatcher(audio) + assertTrue(audio.presentationSeconds.indices.all { matcher.matches(it, it.toDouble() + shift) }) + } + val mixed = PlatformTimestampMatcher(audio) + assertTrue(mixed.matches(0, 0.0056)) + assertFalse(mixed.matches(1, 1.006)) + } finally { source.delete() } + } + } + @Test fun multipleMediaEditsAndRepeatedEmptyEditsRemainUnproven() { + for (starts in listOf(listOf(0, 1000), listOf(-1, -1))) { + val edits = box("edts", box("elst", ByteArray(4) + int(2) + + starts.flatMap { (int(1000) + int(it) + int(65536)).toList() }.toByteArray())) + val source = file(track(edits = edits)) + try { + try { IsoTimelineReader.read(source, 250000); fail("Unsupported edit sequence was proven") } + catch (_: IllegalArgumentException) { } + catch (stop: SampleScanStop) { assertEquals(SampleScanStatus.Unsupported, stop.status) } + } finally { source.delete() } + } + } @Test fun movieAndMediaTimescalesAreIndependentAndTrimsCannotClaimMissingTail() { assertEquals(2.0, read(track(edits = edit(180000, 1000)), scale = 90000).endSeconds, 0.0) invalid(track(edits = edit(4000, 0)))