From b5e43878498d380bb86a49e3ac1019f1f3ffaed4 Mon Sep 17 00:00:00 2001 From: Eugene Morozov Date: Fri, 2 Oct 2026 12:48:02 -0700 Subject: [PATCH 01/49] Inspect container and track metadata for Standard Input --- library/build.gradle | 10 + .../MediaMetadataFixtureTests.kt | 152 +++++++++ .../internal/standardization/CodecMetadata.kt | 213 ++++++++++++ .../standardization/IsoContainerMetadata.kt | 145 ++++++++ .../standardization/MediaMetadataInspector.kt | 316 ++++++++++++++++++ .../IsoContainerMetadataTests.kt | 84 +++++ .../MediaMetadataReaderTests.kt | 245 ++++++++++++++ scripts/README.md | 32 ++ scripts/run-metadata-fixtures.py | 145 ++++++++ 9 files changed, 1342 insertions(+) create mode 100644 library/src/androidTest/java/com/mux/video/upload/internal/standardization/MediaMetadataFixtureTests.kt create mode 100644 library/src/main/java/com/mux/video/upload/internal/standardization/CodecMetadata.kt create mode 100644 library/src/main/java/com/mux/video/upload/internal/standardization/IsoContainerMetadata.kt create mode 100644 library/src/main/java/com/mux/video/upload/internal/standardization/MediaMetadataInspector.kt create mode 100644 library/src/test/java/com/mux/video/upload/internal/standardization/IsoContainerMetadataTests.kt create mode 100644 library/src/test/java/com/mux/video/upload/internal/standardization/MediaMetadataReaderTests.kt create mode 100644 scripts/README.md create mode 100644 scripts/run-metadata-fixtures.py diff --git a/library/build.gradle b/library/build.gradle index d7eff249..04491ed5 100644 --- a/library/build.gradle +++ b/library/build.gradle @@ -32,6 +32,13 @@ android { kotlinOptions { jvmTarget = '1.8' } + // Target only the instrumentation APK at a current API level. + testOptions { targetSdk = 36 } + + // External fixture assets are staged outside product Git by the manual test runner. + if (project.hasProperty("mediaFixtureAssets")) { + sourceSets.androidTest.assets.srcDir(project.property("mediaFixtureAssets")) + } buildFeatures { buildConfig true } @@ -93,6 +100,9 @@ dependencies { implementation 'androidx.core:core-ktx:1.13.1' implementation "org.jetbrains.kotlinx:kotlinx-coroutines-android:1.8.1" + androidTestImplementation 'androidx.test:runner:1.6.2' + androidTestImplementation 'androidx.test.ext:junit:1.2.1' + testImplementation 'junit:junit:4.13.2' testImplementation 'org.jetbrains.kotlinx:kotlinx-coroutines-test:1.8.1' testImplementation 'androidx.test.ext:junit:1.2.1' diff --git a/library/src/androidTest/java/com/mux/video/upload/internal/standardization/MediaMetadataFixtureTests.kt b/library/src/androidTest/java/com/mux/video/upload/internal/standardization/MediaMetadataFixtureTests.kt new file mode 100644 index 00000000..1490ea6b --- /dev/null +++ b/library/src/androidTest/java/com/mux/video/upload/internal/standardization/MediaMetadataFixtureTests.kt @@ -0,0 +1,152 @@ +package com.mux.video.upload.internal.standardization + +import android.os.Build +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import org.json.JSONArray +import org.json.JSONObject +import org.junit.Assert.* +import org.junit.Test +import org.junit.runner.RunWith +import java.io.File +import java.security.MessageDigest + +@RunWith(AndroidJUnit4::class) +class MediaMetadataFixtureTests { + @Test fun inspectExternalCatalogAndMeasureMetadata() { + val instrumentation = InstrumentationRegistry.getInstrumentation() + val assets = instrumentation.context.assets + val catalog = JSONObject(assets.open("fixtures/catalog.json").bufferedReader().use { it.readText() }) + val fixtures = catalog.getJSONArray("fixtures") + val results = JSONArray() + val directory = instrumentation.targetContext.getExternalFilesDir("metadata-fixtures")!! + val fixtureId = InstrumentationRegistry.getArguments().getString("fixtureId") + require(fixtureId != null) { "Use scripts/run-metadata-fixtures.py to stage a verified fixture" } + run { + for (i in 0 until fixtures.length()) { + val fixture = fixtures.getJSONObject(i) + if (fixture.getString("id") != fixtureId) continue + val file = File(directory, fixture.getString("canonicalFilename")) + assertEquals(fixture.getLong("sizeBytes"), file.length()) + val digest = MessageDigest.getInstance("SHA-256") + file.inputStream().use { input -> + val buffer = ByteArray(64 * 1024) + while (true) { val count = input.read(buffer); if (count < 0) break; digest.update(buffer, 0, count) } + } + assertEquals(fixture.getString("sha256"), digest.digest().joinToString("") { "%02x".format(it) }) + val inspector = MediaMetadataInspector() + val initial = inspector.inspect(file) + val row = JSONObject().put("id", fixture.getString("id")).put("sha256", fixture.getString("sha256")) + if (initial is MetadataInspectionResult.Success) { + assertFalse("Expected malformed fixture rejection", fixture.optBoolean("expectedMetadataFailure")) + val inspection = initial.inspection + assertDeferredFacts(inspection.facts) + val expected = fixture.getJSONObject("facts") + println("METADATA " + fixture.getString("id") + " " + inspection) + verifyMetadata(fixture, expected, inspection) + val times = mutableListOf() + repeat(10) { + val repeated = inspector.inspect(file) as MetadataInspectionResult.Success + assertEquals(inspection.facts, repeated.inspection.facts) + times.add(repeated.inspection.elapsedNanos) + } + times.sort() + row.put("status", "success").put("initialMs", inspection.elapsedNanos / 1e6) + .put("medianMs", (times[4] + times[5]) / 2e6).put("p90Ms", times[8] / 1e6) + .put("maxMs", times.last() / 1e6).put("facts", inspection.facts.toString()) + .put("tracks", JSONArray(inspection.tracks.map { it.toString() })) + .put("isoTracks", inspection.isoTracks.toString()) + } else { + row.put("status", initial.toString()) + assertTrue("Required fixture unreadable: " + fixture.getString("id"), + fixture.optBoolean("expectedMetadataFailure", false) || + fixture.getJSONObject("facts").optString("videoCodec") == "prores") + } + results.put(row) + file.delete() + } + val report = JSONObject().put("api", Build.VERSION.SDK_INT).put("model", Build.MODEL) + .put("catalogId", catalog.getString("catalogId")).put("sampleScanPerformed", false) + .put("iterations", 10).put("results", results) + File(instrumentation.targetContext.filesDir, "metadata-results.json").writeText(report.toString(2)) + } + } + + private fun verifyMetadata(fixture: JSONObject, expected: JSONObject, inspection: MediaMetadataInspection) { + val facts = inspection.facts + val strict = fixture.getString("id").startsWith("synthetic-standard") + val expectedCodec = when (expected.optString("videoCodec")) { + "h264" -> VideoCodec.H264 + "hevc" -> VideoCodec.Hevc + else -> VideoCodec.Other + } + if (facts.videoCodec != MediaFact.Unknown && !fixture.getString("id").contains("dolby-vision")) + assertEquals(MediaFact.Known(expectedCodec), facts.videoCodec) + for ((key, value) in listOf("encodedDimensions" to facts.encodedDimensions, "displayDimensions" to facts.displayDimensions)) { + val dimensions = expected.optJSONArray(key) ?: continue + if (strict || value != MediaFact.Unknown) + assertEquals(fixture.getString("id") + " " + key, + MediaFact.Known(Dimensions(dimensions.getInt(0), dimensions.getInt(1))), value) + } + val range = if (fixture.getString("id").contains("dolby-vision")) DynamicRange.DolbyVision + else when (expected.optString("dynamicRange")) { + "sdr" -> DynamicRange.Sdr + "hlg" -> DynamicRange.Hlg + "pq" -> DynamicRange.Pq + "dolbyVision" -> DynamicRange.DolbyVision + else -> null + } + if (range != null && (strict || facts.dynamicRange != MediaFact.Unknown)) + assertEquals(fixture.getString("id") + " range", MediaFact.Known(range), facts.dynamicRange) + if (expected.has("orientationDegrees") && facts.rotationDegrees != MediaFact.Unknown) { + val degrees = expected.getInt("orientationDegrees") + assertEquals(MediaFact.Known((degrees % 360 + 360) % 360), facts.rotationDegrees) + } + if (fixture.getString("id").contains("dolby-vision")) + assertEquals(MediaFact.Known(DynamicRange.DolbyVision), facts.dynamicRange) + if (strict) { + assertEquals(MediaFact.Known(PixelFormat(expected.getInt("bitDepth"), ChromaSubsampling.Yuv420)), facts.pixelFormat) + assertEquals(MediaFact.Known(expected.getInt("orientationDegrees")), facts.rotationDegrees) + val layout = when (expected.getString("audioLayout")) { + "5.1" -> AudioChannelLayout.FivePointOne + else -> AudioChannelLayout.Stereo + } + assertEquals(MediaFact.Known(listOf(AudioTrack(MediaFact.Known(AudioFormat.Aac(layout))))), facts.audioTracks) + } + expected.optJSONArray("audioLayouts")?.let { layouts -> + assertEquals(layouts.length(), facts.audioTracks.valueOrNull!!.size) + for (i in 0 until layouts.length()) { + val layout = AudioChannelLayout.valueOf(layouts.getString(i)) + assertEquals(MediaFact.Known(AudioFormat.Aac(layout)), facts.audioTracks.valueOrNull!![i].format) + } + assertEquals(listOf(0, 1, 2).map { MediaFact.Known(it) }, inspection.tracks.map { it.containerIndex }) + } + } + + private fun assertDeferredFacts(facts: MediaFacts) { + assertEquals(MediaFact.Unknown, facts.gopStructure) + assertEquals(MediaFact.Unknown, facts.cadence) + assertEquals(MediaFact.Unknown, facts.timestamps) + assertEquals(MediaFact.Unknown, facts.frameRate) + assertEquals(MediaFact.Unknown, facts.averageBitrate) + assertEquals(MediaFact.Unknown, facts.maximumGopBitrate) + assertEquals(MediaFact.Unknown, facts.maximumGopByteSize) + assertEquals(MediaFact.Unknown, facts.maximumKeyframeIntervalSeconds) + assertEquals(MediaFact.Unknown, facts.durationSeconds) + assertEquals(MediaFact.Unknown, facts.audioVideoStartOffsetSeconds) + assertEquals(MediaFact.Unknown, facts.editList) + } + + @Test fun rejectMissingEmptyAndMalformedContainers() { + val context = InstrumentationRegistry.getInstrumentation().targetContext + val file = File(context.cacheDir, "metadata-invalid.mp4") + try { + file.delete() + assertTrue(MediaMetadataInspector().inspect(file) is MetadataInspectionResult.Failure) + file.writeBytes(byteArrayOf()) + assertTrue(MediaMetadataInspector().inspect(file) is MetadataInspectionResult.Failure) + file.writeBytes(ByteArray(64) { 42 }) + assertTrue(MediaMetadataInspector().inspect(file) is MetadataInspectionResult.Failure) + } finally { file.delete() } + } +} diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/CodecMetadata.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/CodecMetadata.kt new file mode 100644 index 00000000..b2af00b1 --- /dev/null +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/CodecMetadata.kt @@ -0,0 +1,213 @@ +package com.mux.video.upload.internal.standardization + +import androidx.media3.common.util.ParsableBitArray +import androidx.media3.container.NalUnitUtil +import androidx.media3.container.ParsableNalUnitBitArray + +/** Color codes use Media3/Android's normalized constants, not ISO code points. */ +internal data class ColorMetadata( + val standard: MediaFact = MediaFact.Unknown, + val transfer: MediaFact = MediaFact.Unknown, + val range: MediaFact = MediaFact.Unknown, +) + +internal data class CodecMetadata( + val profile: MediaFact = MediaFact.Unknown, + val pixelFormat: MediaFact = MediaFact.Unknown, + val color: ColorMetadata = ColorMetadata(), + val pixelAspectRatio: MediaFact = MediaFact.Unknown, +) + +/** Only codec initialization data is read here. No compressed media samples are inspected. */ +internal object CodecMetadataReader { + const val MAX_CONFIGURATION_BYTES = 64 * 1024 + + fun video(codec: VideoCodec?, configurations: List): CodecMetadata { + if (configurations.sumOf { it.size.toLong() } > MAX_CONFIGURATION_BYTES) return CodecMetadata() + val results = configurations.flatMap(::annexBNalUnits).mapNotNull { nal -> + try { + when { + codec == VideoCodec.H264 && nal.isNotEmpty() && nal[0].toInt() and 31 == 7 -> avc(nal) + codec == VideoCodec.Hevc && nal.size >= 2 && (nal[0].toInt() and 126) shr 1 == 33 -> hevc(nal) + else -> null + } + } catch (_: RuntimeException) { + // Truncated/unsupported initialization data never becomes compliant evidence. + CodecMetadata() + } + } + // Several SPS descriptions may signal changing formats; do not choose an arbitrary one. + return results.distinct().singleOrNull() ?: CodecMetadata() + } + + private fun avc(nal: ByteArray): CodecMetadata { + val sps = NalUnitUtil.parseSpsNalUnit(nal, 0, nal.size) + val bits = ParsableNalUnitBitArray(nal, 1, nal.size) + val profile = bits.readBits(8) + bits.skipBits(16) + bits.readUnsignedExpGolombCodedInt() // seq_parameter_set_id + val chroma = when (profile) { + 66, 77, 88 -> 1 + 100, 110, 122, 244, 44, 83, 86, 118, 128, 138 -> bits.readUnsignedExpGolombCodedInt() + else -> return CodecMetadata() + } + return CodecMetadata( + profile = MediaFact.Known(when (profile) { + 66 -> VideoProfile.H264Baseline + 77 -> VideoProfile.H264Main + 100 -> VideoProfile.H264High + else -> VideoProfile.Other + }), + pixelFormat = pixels(sps.bitDepthLumaMinus8, sps.bitDepthChromaMinus8, chroma), + color = color(sps.colorSpace, sps.colorTransfer, sps.colorRange), + pixelAspectRatio = positive(sps.pixelWidthHeightRatio.toDouble()), + ) + } + + private fun hevc(nal: ByteArray): CodecMetadata { + // Single-layer SPS only; multilayer configuration needs separate evidence. + if ((nal[0].toInt() and 1) != 0 || (nal[1].toInt() and 248) != 0) return CodecMetadata() + if (!boundedHevcSyntax(nal)) return CodecMetadata() + val sps = NalUnitUtil.parseH265SpsNalUnit(nal, 0, nal.size, null) + val profile = sps.profileTierLevel ?: return CodecMetadata() + return CodecMetadata( + profile = MediaFact.Known(when (if (profile.generalProfileSpace == 0) profile.generalProfileIdc else -1) { + 1 -> VideoProfile.HevcMain + 2 -> VideoProfile.HevcMain10 + else -> VideoProfile.Other + }), + pixelFormat = pixels(sps.bitDepthLumaMinus8, sps.bitDepthChromaMinus8, sps.chromaFormatIdc), + color = color(sps.colorSpace, sps.colorTransfer, sps.colorRange), + pixelAspectRatio = positive(sps.pixelWidthHeightRatio.toDouble()), + ) + } + + /** Bound allocation-driving SPS fields before calling the shared parser. */ + private fun boundedHevcSyntax(nal: ByteArray): Boolean { + val bits = ParsableNalUnitBitArray(nal, 2, nal.size) + bits.skipBits(4) + val layers = bits.readBits(3) + if (layers > 6) return false + bits.skipBit() + bits.skipBits(96) // general profile/tier/level + val profilePresent = BooleanArray(layers) + val levelPresent = BooleanArray(layers) + for (i in 0 until layers) { + profilePresent[i] = bits.readBit() + levelPresent[i] = bits.readBit() + } + if (layers > 0) bits.skipBits((8 - layers) * 2) + for (i in 0 until layers) { + if (profilePresent[i]) bits.skipBits(88) + if (levelPresent[i]) bits.skipBits(8) + } + if (bits.readUnsignedExpGolombCodedInt() !in 0..15) return false + val chroma = bits.readUnsignedExpGolombCodedInt() + if (chroma !in 0..3) return false + if (chroma == 3) bits.skipBit() + repeat(2) { if (bits.readUnsignedExpGolombCodedInt() !in 1..65535) return false } + if (bits.readBit()) repeat(4) { bits.readUnsignedExpGolombCodedInt() } + repeat(2) { if (bits.readUnsignedExpGolombCodedInt() !in 0..8) return false } + val pocBits = bits.readUnsignedExpGolombCodedInt() + if (pocBits !in 0..12) return false + val start = if (bits.readBit()) 0 else layers + for (i in start..layers) repeat(3) { bits.readUnsignedExpGolombCodedInt() } + repeat(6) { bits.readUnsignedExpGolombCodedInt() } + if (bits.readBit() && bits.readBit()) { + for (sizeId in 0..3) { + for (matrixId in 0..5 step if (sizeId == 3) 3 else 1) { + if (!bits.readBit()) bits.readUnsignedExpGolombCodedInt() else { + if (sizeId > 1) bits.readSignedExpGolombCodedInt() + repeat(minOf(64, 1 shl (4 + 2 * sizeId))) { bits.readSignedExpGolombCodedInt() } + } + } + } + } + bits.skipBits(2) + if (bits.readBit()) { + bits.skipBits(8) + repeat(2) { bits.readUnsignedExpGolombCodedInt() } + bits.skipBit() + } + val sets = bits.readUnsignedExpGolombCodedInt() + if (sets !in 0..64) return false + for (i in 0 until sets) { + // Predicted reference sets need a fuller bounded syntax reader; leave them unknown for now. + if (i > 0 && bits.readBit()) return false + val negative = bits.readUnsignedExpGolombCodedInt() + val positive = bits.readUnsignedExpGolombCodedInt() + if (negative !in 0..16 || positive !in 0..16 || negative + positive > 16) return false + repeat(negative + positive) { bits.readUnsignedExpGolombCodedInt(); bits.skipBit() } + } + if (bits.readBit()) { + val count = bits.readUnsignedExpGolombCodedInt() + if (count !in 0..32) return false + repeat(count) { bits.skipBits(pocBits + 5) } + } + return true + } + + private fun pixels(luma: Int, chroma: Int, subsampling: Int): MediaFact = + if (luma != chroma || luma !in 0..8 || subsampling !in 0..3) MediaFact.Unknown + else MediaFact.Known(PixelFormat(luma + 8, + if (subsampling == 1) ChromaSubsampling.Yuv420 else ChromaSubsampling.Other)) + + private fun color(standard: Int, transfer: Int, range: Int) = ColorMetadata( + positiveCode(standard), positiveCode(transfer), positiveCode(range)) + + /** MediaExtractor supplies AVC/HEVC CSD in Annex B form, with three- or four-byte prefixes. */ + private fun annexBNalUnits(data: ByteArray): List { + val starts = mutableListOf>() + var i = 0 + while (i + 2 < data.size) { + val prefix = when { + data[i] != 0.toByte() || data[i + 1] != 0.toByte() -> 0 + data[i + 2] == 1.toByte() -> 3 + i + 3 < data.size && data[i + 2] == 0.toByte() && data[i + 3] == 1.toByte() -> 4 + else -> 0 + } + if (prefix > 0) { starts.add(i to prefix); i += prefix } else i++ + } + return starts.mapIndexed { index, (start, prefix) -> + data.copyOfRange(start + prefix, starts.getOrNull(index + 1)?.first ?: data.size) + } + } + + /** Channel count alone cannot prove a surround layout. PCE/extension configurations stay unknown. */ + fun aacLayout(configuration: ByteArray?, channelCount: Int?): MediaFact { + if (configuration == null || configuration.size !in 2..MAX_CONFIGURATION_BYTES) return MediaFact.Unknown + return try { + val bits = ParsableBitArray(configuration) + val objectType = bits.readBits(5) + if (objectType !in 1..4) return MediaFact.Unknown + val frequencyIndex = bits.readBits(4) + if (frequencyIndex == 15) { + if (bits.readBits(24) <= 0) return MediaFact.Unknown + } else if (frequencyIndex > 12) return MediaFact.Unknown + val channels = bits.readBits(4) + val expectedCount = when (channels) { 1 -> 1; 2 -> 2; 3 -> 3; 4 -> 4; 5 -> 5; 6 -> 6; 7 -> 8; else -> null } + ?: return MediaFact.Unknown + if (channelCount != null && channelCount != expectedCount) return MediaFact.Unknown + MediaFact.Known(when (channels) { + 1 -> AudioChannelLayout.Mono + 2 -> AudioChannelLayout.Stereo + 6 -> AudioChannelLayout.FivePointOne + else -> AudioChannelLayout.Other + }) + } catch (_: RuntimeException) { MediaFact.Unknown } + } +} + +internal fun positive(value: Double?): MediaFact = + if (value != null && value.isFinite() && value > 0) MediaFact.Known(value) else MediaFact.Unknown + +internal fun positiveCode(value: Int?): MediaFact = + if (value != null && value > 0) MediaFact.Known(value) else MediaFact.Unknown + +/** Missing evidence may be filled; contradictory evidence stays unknown. */ +internal fun reconcile(a: MediaFact, b: MediaFact): MediaFact = when { + a == MediaFact.Unknown -> b + b == MediaFact.Unknown -> a + a == b -> a + else -> MediaFact.Unknown +} diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/IsoContainerMetadata.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/IsoContainerMetadata.kt new file mode 100644 index 00000000..d380b20e --- /dev/null +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/IsoContainerMetadata.kt @@ -0,0 +1,145 @@ +package com.mux.video.upload.internal.standardization + +import java.io.File +import java.io.RandomAccessFile + +internal data class IsoTrackMetadata( + val handler: String?, + val trackId: MediaFact, + val sampleEntries: List, + val hasDolbyVisionConfiguration: Boolean, + val rotationDegrees: MediaFact, + val hasSimpleSampleGeometry: Boolean, +) { + val hasSimpleOrientation: Boolean get() = rotationDegrees != MediaFact.Unknown +} + +/** Seeks past sample payloads/tables; reads only bounded track and sample-description metadata. */ +internal object IsoContainerMetadataReader { + private const val MAX_BOXES = 4096 + private const val MAX_ENTRIES = 16 + private data class Box(val type: String, val payload: Long, val end: Long) + + fun read(file: File): MediaFact> = try { + RandomAccessFile(file, "r").use { input -> + val reader = Reader(input) + val moov = reader.boxes(0, input.length()).singleOrNull { it.type == "moov" } + ?: return MediaFact.Unknown + val tracks = reader.boxes(moov.payload, moov.end).filter { it.type == "trak" } + if (tracks.size !in 1..64) return MediaFact.Unknown + val metadata = tracks.map { reader.track(it) } + val ids = metadata.mapNotNull { it.trackId.valueOrNull } + require(ids.distinct().size == ids.size) + MediaFact.Known(metadata) + } + } catch (_: Exception) { MediaFact.Unknown } + + private class Reader(val input: RandomAccessFile) { + var boxCount = 0 + fun boxes(start: Long, end: Long, allowQuickTimeTerminator: Boolean = false): List { + require(start <= end && end <= input.length()) + val result = mutableListOf() + var position = start + while (position < end) { + if (allowQuickTimeTerminator && end - position == 4L) { + input.seek(position) + require(input.readInt() == 0) + break + } + require(end - position >= 8 && ++boxCount <= MAX_BOXES) + input.seek(position) + val size32 = input.readInt().toLong() and 0xffffffffL + val type = fourCc() + val header = if (size32 == 1L) 16 else 8 + val size = when (size32) { 0L -> end - position; 1L -> input.readLong(); else -> size32 } + require(size >= header && size <= end - position) + result.add(Box(type, position + header, position + size)) + position += size + } + return result + } + + fun track(track: Box): IsoTrackMetadata { + val children = boxes(track.payload, track.end) + val tkhd = children.singleOrNull { it.type == "tkhd" } + val mdia = children.single { it.type == "mdia" } + val media = boxes(mdia.payload, mdia.end) + val handler = media.singleOrNull { it.type == "hdlr" }?.let { + require(it.end - it.payload >= 12) + input.seek(it.payload + 8) + fourCc() + } + val trackId = tkhd?.let { + input.seek(it.payload) + val version = input.readUnsignedByte() + val offset = when (version) { 0 -> 12; 1 -> 20; else -> -1 } + if (offset < 0 || it.end - it.payload < offset + 4) MediaFact.Unknown else { + input.seek(it.payload + offset) + positiveCode(input.readInt()) + } + } ?: MediaFact.Unknown + val orientation = tkhd?.let(::orientation) ?: MediaFact.Unknown + if (handler !in listOf("vide", "soun")) + return IsoTrackMetadata(handler, trackId, emptyList(), false, orientation, false) + val minf = media.single { it.type == "minf" } + val stbl = boxes(minf.payload, minf.end).single { it.type == "stbl" } + val stsd = boxes(stbl.payload, stbl.end).single { it.type == "stsd" } + require(stsd.end - stsd.payload >= 8) + input.seek(stsd.payload + 4) + val count = input.readInt() + require(count in 1..MAX_ENTRIES) + val entries = boxes(stsd.payload + 8, stsd.end) + require(entries.size == count) + var dolby = false + var simpleGeometry = true + if (handler == "vide") { + for (entry in entries) { + if (entry.type in listOf("dvhe", "dvh1", "dvav", "dva1")) dolby = true + if (entry.type in listOf("hvc1", "hev1", "avc1", "avc3", "dvhe", "dvh1", "dvav", "dva1")) { + require(entry.end - entry.payload >= 78) + val extensions = boxes(entry.payload + 78, entry.end, allowQuickTimeTerminator = true) + if (extensions.any { it.type in listOf("dvcC", "dvvC", "dvwC") }) dolby = true + if (extensions.any { it.type == "clap" }) simpleGeometry = false + val aspects = extensions.filter { it.type == "pasp" } + if (aspects.size > 1) simpleGeometry = false + for (aspect in aspects) { + require(aspect.end - aspect.payload == 8L) + input.seek(aspect.payload) + val horizontal = input.readInt().toLong() and 0xffffffffL + val vertical = input.readInt().toLong() and 0xffffffffL + // Retain uncertain anamorphic/clean-aperture geometry for a later richer reader. + if (horizontal == 0L || horizontal != vertical) simpleGeometry = false + } + } + } + } + return IsoTrackMetadata(handler, trackId, entries.map { it.type }, dolby, orientation, simpleGeometry) + } + + private fun orientation(box: Box): MediaFact { + input.seek(box.payload) + val version = input.readUnsignedByte() + val offset = when (version) { 0 -> 40; 1 -> 52; else -> return MediaFact.Unknown } + require(box.end - box.payload >= offset + 36) + input.seek(box.payload + offset) + val matrix = IntArray(9) { input.readInt() } + // Unit rotations plus translation only. Scale, shear, reflection and perspective stay unproven. + if (matrix[2] != 0 || matrix[5] != 0 || matrix[8] != 0x40000000) return MediaFact.Unknown + val axes = listOf(matrix[0], matrix[1], matrix[3], matrix[4]) + // Normalize the container transform to the shared clockwise convention. + return when (axes) { + listOf(65536, 0, 0, 65536) -> MediaFact.Known(0) + listOf(0, 65536, -65536, 0) -> MediaFact.Known(270) + listOf(-65536, 0, 0, -65536) -> MediaFact.Known(180) + listOf(0, -65536, 65536, 0) -> MediaFact.Known(90) + else -> MediaFact.Unknown + } + } + + private fun fourCc(): String { + val bytes = ByteArray(4) + input.readFully(bytes) + return String(bytes, Charsets.US_ASCII) + } + } +} diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/MediaMetadataInspector.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/MediaMetadataInspector.kt new file mode 100644 index 00000000..a7081d7b --- /dev/null +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/MediaMetadataInspector.kt @@ -0,0 +1,316 @@ +package com.mux.video.upload.internal.standardization + +import android.media.MediaCodecInfo.CodecProfileLevel +import android.media.MediaExtractor +import android.media.MediaFormat +import android.os.Build +import java.io.File +import java.io.IOException +import java.nio.ByteBuffer +import kotlin.math.roundToInt + +internal enum class ContainerKind { IsoBaseMedia, Matroska, Other } +internal enum class TrackKind { Video, Audio, Other, Unknown } +internal enum class MetadataFailure { Unreadable, Malformed, LimitExceeded } + +internal sealed interface MetadataInspectionResult { + data class Success(val inspection: MediaMetadataInspection) : MetadataInspectionResult + data class Failure(val reason: MetadataFailure) : MetadataInspectionResult +} + +internal data class VideoTrackMetadata( + val codec: MediaFact, + val profile: MediaFact, + val encodedDimensions: MediaFact, + val displayDimensions: MediaFact, + val rotationDegrees: MediaFact, + val reportedPlatformRotationDegrees: MediaFact, + val configuration: CodecMetadata, + val platformColor: ColorMetadata, + val dynamicRange: MediaFact, +) + +internal data class AudioTrackMetadata( + val format: MediaFact, + val channelCount: MediaFact, + val sampleRate: MediaFact, + val encoderDelaySamples: MediaFact, + val encoderPaddingSamples: MediaFact, +) + +/** Reported metadata, not sample measurements or an effective presentation timeline. */ +internal data class TrackMetadata( + val extractorIndex: Int, + val containerIndex: MediaFact, + val sourceTrackId: MediaFact, + val kind: TrackKind, + val mimeType: MediaFact, + val reportedDurationSeconds: MediaFact, + val reportedFrameRate: MediaFact, + val reportedBitrate: MediaFact, + val video: VideoTrackMetadata? = null, + val audio: AudioTrackMetadata? = null, +) + +internal data class MediaMetadataInspection( + val container: MediaFact, + val tracks: List, + val isoTracks: MediaFact>, + val facts: MediaFacts, + val elapsedNanos: Long, +) + +/** File-only metadata inspection. Owns its extractor; never exports or scans samples. */ +internal class MediaMetadataInspector { + fun inspect(file: File): MetadataInspectionResult { + val started = System.nanoTime() + if (!file.isFile || !file.canRead() || file.length() == 0L) + return MetadataInspectionResult.Failure(MetadataFailure.Unreadable) + val extractor = MediaExtractor() + return try { + val container = container(file) + extractor.setDataSource(file.absolutePath) + if (extractor.trackCount !in 1..MAX_TRACKS) + return MetadataInspectionResult.Failure(MetadataFailure.LimitExceeded) + val iso = if (container == MediaFact.Known(ContainerKind.IsoBaseMedia)) + IsoContainerMetadataReader.read(file) else MediaFact.Unknown + val tracks = (0 until extractor.trackCount).map { index -> + val format = extractor.getTrackFormat(index) + val orientation = if (format.string(MediaFormat.KEY_MIME)?.startsWith("video/") == true) + iso.valueOrNull?.filter { it.handler == "vide" }?.singleOrNull()?.rotationDegrees else null + val track = MediaTrackMetadataReader.read(index, format, containerRotation = orientation) + val containers = iso.valueOrNull + val id = track.sourceTrackId.valueOrNull + val match = if (id != null) containers?.indexOfFirst { it.trackId.valueOrNull == id } else null + val ordered = containers?.getOrNull(index)?.handler + val sameKind = (ordered == "vide" && track.kind == TrackKind.Video) || + (ordered == "soun" && track.kind == TrackKind.Audio) + val containerIndex = when { + match != null && match >= 0 && + ((containers!![match].handler == "vide" && track.kind == TrackKind.Video) || + (containers[match].handler == "soun" && track.kind == TrackKind.Audio)) -> MediaFact.Known(match) + id == null && containers?.size == extractor.trackCount && sameKind -> MediaFact.Known(index) + else -> MediaFact.Unknown + } + track.copy(containerIndex = containerIndex) + } + val containerFacts = MediaContainerMetadataReader.facts(container, iso, tracks) + MetadataInspectionResult.Success(MediaMetadataInspection( + container, tracks, iso, containerFacts, System.nanoTime() - started)) + } catch (_: IOException) { + MetadataInspectionResult.Failure(MetadataFailure.Unreadable) + } catch (_: RuntimeException) { + MetadataInspectionResult.Failure(MetadataFailure.Malformed) + } finally { + extractor.release() + } + } + + private fun container(file: File): MediaFact { + val header = ByteArray(12) + val count = file.inputStream().use { it.read(header) } + return when { + count >= 8 && String(header, 4, 4, Charsets.US_ASCII) in listOf("ftyp", "moov", "mdat", "free", "wide") -> + MediaFact.Known(ContainerKind.IsoBaseMedia) + count >= 4 && header.take(4) == listOf(0x1a.toByte(), 0x45.toByte(), 0xdf.toByte(), 0xa3.toByte()) -> + MediaFact.Known(ContainerKind.Matroska) + count >= 8 -> MediaFact.Known(ContainerKind.Other) + else -> MediaFact.Unknown + } + } + + companion object { private const val MAX_TRACKS = 64 } +} + +/** Reconcile physical container tracks with extractor views, including Dolby Vision base-layer views. */ +internal object MediaContainerMetadataReader { + fun facts(container: MediaFact, iso: MediaFact>, + tracks: List): MediaFacts { + val dolby = iso.valueOrNull?.any { it.hasDolbyVisionConfiguration } == true || + tracks.any { it.video?.dynamicRange == MediaFact.Known(DynamicRange.DolbyVision) } + if (container != MediaFact.Known(ContainerKind.IsoBaseMedia)) return MediaTrackMetadataReader.facts(tracks) + val containers = iso.valueOrNull + ?: return MediaFacts(dynamicRange = if (dolby) MediaFact.Known(DynamicRange.DolbyVision) else MediaFact.Unknown) + if (containers.any { it.handler == null }) return MediaFacts() + val videoIndices = containers.indices.filter { containers[it].handler == "vide" } + val audioIndices = containers.indices.filter { containers[it].handler == "soun" } + val audioViews = tracks.filter { it.kind == TrackKind.Audio }.groupBy { it.containerIndex.valueOrNull } + val orderedAudio = if (audioViews.keys == audioIndices.toSet() && audioViews.values.all { it.size == 1 }) { + MediaFact.Known(audioIndices.map { index -> + val format = if (containers[index].sampleEntries.size == 1) audioViews[index]!!.single().audio!!.format + else MediaFact.Unknown + AudioTrack(format) + }) + } else MediaFact.Unknown + val base = MediaFacts(videoTrackCount = MediaFact.Known(videoIndices.size), audioTracks = orderedAudio, + dynamicRange = if (dolby) MediaFact.Known(DynamicRange.DolbyVision) else MediaFact.Unknown) + val index = videoIndices.singleOrNull() ?: return base + val description = containers[index] + if (description.sampleEntries.size != 1) return base + val views = tracks.filter { it.kind == TrackKind.Video } + if (views.any { it.containerIndex.valueOrNull != index }) return base + val selected = if (views.size == 1) views.single() else { + if (!dolby || views.size != 2 || views.count { it.mimeType.valueOrNull == "video/dolby-vision" } != 1) return base + val codec = when (description.sampleEntries.single()) { + "hvc1", "hev1", "dvh1", "dvhe" -> VideoCodec.Hevc + "avc1", "avc3", "dva1", "dvav" -> VideoCodec.H264 + else -> return base + } + views.singleOrNull { it.video?.codec == MediaFact.Known(codec) } ?: return base + } + val video = selected.video ?: return base + return base.copy( + videoCodec = video.codec, videoProfile = video.profile, encodedDimensions = video.encodedDimensions, + displayDimensions = if (description.hasSimpleOrientation && description.hasSimpleSampleGeometry) + video.displayDimensions else MediaFact.Unknown, + rotationDegrees = if (description.hasSimpleOrientation) video.rotationDegrees else MediaFact.Unknown, + pixelFormat = video.configuration.pixelFormat, + dynamicRange = if (dolby) MediaFact.Known(DynamicRange.DolbyVision) else video.dynamicRange, + ) + } +} + +internal object MediaTrackMetadataReader { + fun read(index: Int, format: MediaFormat, apiLevel: Int = Build.VERSION.SDK_INT, + containerRotation: MediaFact? = null): TrackMetadata { + val mime = format.string(MediaFormat.KEY_MIME) + val kind = when { + mime == null -> TrackKind.Unknown + mime.startsWith("video/") -> TrackKind.Video + mime.startsWith("audio/") -> TrackKind.Audio + else -> TrackKind.Other + } + return TrackMetadata( + index, MediaFact.Unknown, positiveCode(format.int("track-id")), kind, mime?.let { MediaFact.Known(it) } ?: MediaFact.Unknown, + positive(format.long(MediaFormat.KEY_DURATION)?.toDouble()?.div(1_000_000)), + positive(format.number(MediaFormat.KEY_FRAME_RATE)), + positiveCode(format.int(MediaFormat.KEY_BIT_RATE)), + video = if (kind == TrackKind.Video) video(format, mime, apiLevel, containerRotation) else null, + audio = if (kind == TrackKind.Audio) audio(format, mime) else null, + ) + } + + fun facts(tracks: List): MediaFacts { + if (tracks.any { it.kind == TrackKind.Unknown }) return MediaFacts() + val videos = tracks.filter { it.kind == TrackKind.Video } + val audio = tracks.mapNotNull { it.audio }.map { AudioTrack(it.format) } + val base = MediaFacts(videoTrackCount = MediaFact.Known(videos.size), audioTracks = MediaFact.Known(audio)) + val video = videos.singleOrNull()?.video ?: return base + return base.copy( + videoCodec = video.codec, videoProfile = video.profile, + encodedDimensions = video.encodedDimensions, displayDimensions = video.displayDimensions, + rotationDegrees = video.rotationDegrees, pixelFormat = video.configuration.pixelFormat, + dynamicRange = video.dynamicRange, + // Duration, nominal rate and declared bitrate require later sample/timeline proof. + ) + } + + private fun video(format: MediaFormat, mime: String?, apiLevel: Int, + containerRotation: MediaFact?): VideoTrackMetadata { + val codec = when (mime) { + "video/avc" -> MediaFact.Known(VideoCodec.H264) + "video/hevc" -> MediaFact.Known(VideoCodec.Hevc) + null -> MediaFact.Unknown + else -> MediaFact.Known(VideoCodec.Other) + } + val csd = (0..2).mapNotNull { format.bytes("csd-$it") } + val config = CodecMetadataReader.video(codec.valueOrNull, csd) + val encoded = dimensions(format.int(MediaFormat.KEY_WIDTH), format.int(MediaFormat.KEY_HEIGHT)) + val platformRotation = format.int(MediaFormat.KEY_ROTATION)?.let { + if (it % 90 == 0) MediaFact.Known((it % 360 + 360) % 360) else MediaFact.Unknown + } ?: MediaFact.Unknown + val rotation = containerRotation ?: platformRotation + val platformColor = if (apiLevel >= 24) ColorMetadata( + positiveCode(format.int(MediaFormat.KEY_COLOR_STANDARD)), + positiveCode(format.int(MediaFormat.KEY_COLOR_TRANSFER)), + positiveCode(format.int(MediaFormat.KEY_COLOR_RANGE)), + ) else ColorMetadata() + val transfer = reconcile(platformColor.transfer, config.color.transfer) + val range = if (mime == "video/dolby-vision") MediaFact.Known(DynamicRange.DolbyVision) + else dynamicRange(transfer) + return VideoTrackMetadata(codec, + reconcile(platformProfile(codec.valueOrNull, format.int(MediaFormat.KEY_PROFILE)), config.profile), + encoded, display(format, encoded, rotation, config.pixelAspectRatio), rotation, platformRotation, + config, platformColor, range) + } + + private fun audio(format: MediaFormat, mime: String?): AudioTrackMetadata { + val count = positiveCode(format.int(MediaFormat.KEY_CHANNEL_COUNT)) + val audio = when (mime) { + "audio/mp4a-latm" -> CodecMetadataReader.aacLayout(format.bytes("csd-0"), count.valueOrNull) + .valueOrNull?.let { MediaFact.Known(AudioFormat.Aac(it)) } ?: MediaFact.Unknown + null -> MediaFact.Unknown + else -> MediaFact.Known(AudioFormat.OtherCodec) + } + return AudioTrackMetadata(audio, count, positiveCode(format.int(MediaFormat.KEY_SAMPLE_RATE)), + nonNegative(format.int(MediaFormat.KEY_ENCODER_DELAY)), nonNegative(format.int(MediaFormat.KEY_ENCODER_PADDING))) + } + + private fun display(format: MediaFormat, encoded: MediaFact, rotation: MediaFact, + headerAspect: MediaFact): MediaFact { + val size = encoded.valueOrNull ?: return MediaFact.Unknown + val degrees = rotation.valueOrNull ?: return MediaFact.Unknown + val crop = listOf("crop-left", "crop-right", "crop-top", "crop-bottom").map { format.int(it) } + val visible = if (crop.all { it == null }) size else { + if (crop.any { it == null }) return MediaFact.Unknown + val (left, right, top, bottom) = crop.map { it!! } + if (left < 0 || top < 0 || right < left || bottom < top || right >= size.width || bottom >= size.height) + return MediaFact.Unknown + Dimensions(right - left + 1, bottom - top + 1) + } + val sarWidth = format.int("sar-width") + val sarHeight = format.int("sar-height") + val platformAspect = if (sarWidth != null && sarHeight != null && sarWidth > 0 && sarHeight > 0) + MediaFact.Known(sarWidth.toDouble() / sarHeight) else MediaFact.Unknown + if ((sarWidth != null || sarHeight != null) && platformAspect == MediaFact.Unknown) return MediaFact.Unknown + val aspect = reconcile(platformAspect, headerAspect).valueOrNull ?: return MediaFact.Unknown + val width = visible.width * aspect + if (!width.isFinite() || width < 1 || width > Int.MAX_VALUE) return MediaFact.Unknown + val display = Dimensions(width.roundToInt(), visible.height) + return MediaFact.Known(if (degrees == 90 || degrees == 270) Dimensions(display.height, display.width) else display) + } + + private fun dimensions(width: Int?, height: Int?): MediaFact = + if (width != null && height != null && width > 0 && height > 0) MediaFact.Known(Dimensions(width, height)) + else MediaFact.Unknown + + private fun platformProfile(codec: VideoCodec?, profile: Int?): MediaFact = + if (profile == null) MediaFact.Unknown else MediaFact.Known(when (codec) { + VideoCodec.H264 -> when (profile) { + CodecProfileLevel.AVCProfileBaseline, CodecProfileLevel.AVCProfileConstrainedBaseline -> VideoProfile.H264Baseline + CodecProfileLevel.AVCProfileMain -> VideoProfile.H264Main + CodecProfileLevel.AVCProfileHigh, CodecProfileLevel.AVCProfileConstrainedHigh -> VideoProfile.H264High + else -> VideoProfile.Other + } + VideoCodec.Hevc -> when (profile) { + CodecProfileLevel.HEVCProfileMain -> VideoProfile.HevcMain + CodecProfileLevel.HEVCProfileMain10, CodecProfileLevel.HEVCProfileMain10HDR10, + CodecProfileLevel.HEVCProfileMain10HDR10Plus -> VideoProfile.HevcMain10 + else -> VideoProfile.Other + } + else -> VideoProfile.Other + }) + + private fun dynamicRange(transfer: MediaFact): MediaFact = when (transfer.valueOrNull) { + MediaFormat.COLOR_TRANSFER_SDR_VIDEO, MediaFormat.COLOR_TRANSFER_LINEAR -> MediaFact.Known(DynamicRange.Sdr) + MediaFormat.COLOR_TRANSFER_HLG -> MediaFact.Known(DynamicRange.Hlg) + MediaFormat.COLOR_TRANSFER_ST2084 -> MediaFact.Known(DynamicRange.Pq) + else -> MediaFact.Unknown + } +} + +private fun MediaFormat.int(key: String): Int? = try { if (containsKey(key)) getInteger(key) else null } catch (_: RuntimeException) { null } +private fun MediaFormat.long(key: String): Long? = try { if (containsKey(key)) getLong(key) else null } catch (_: RuntimeException) { null } +private fun MediaFormat.string(key: String): String? = try { if (containsKey(key)) getString(key) else null } catch (_: RuntimeException) { null } +private fun MediaFormat.number(key: String): Double? = try { if (containsKey(key)) getFloat(key).toDouble() else null } + catch (_: RuntimeException) { int(key)?.toDouble() } +private fun MediaFormat.bytes(key: String): ByteArray? { + return try { + val buffer: ByteBuffer = getByteBuffer(key)?.duplicate() ?: return null + if (buffer.remaining() !in 1..CodecMetadataReader.MAX_CONFIGURATION_BYTES) null + else ByteArray(buffer.remaining()).also { buffer.get(it) } + } catch (_: RuntimeException) { null } +} + +private fun nonNegative(value: Int?): MediaFact = + if (value != null && value >= 0) MediaFact.Known(value) else MediaFact.Unknown diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/IsoContainerMetadataTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/IsoContainerMetadataTests.kt new file mode 100644 index 00000000..98d9d306 --- /dev/null +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/IsoContainerMetadataTests.kt @@ -0,0 +1,84 @@ +package com.mux.video.upload.internal.standardization + +import org.junit.Assert.* +import org.junit.Test +import java.io.File +import java.nio.ByteBuffer + +class IsoContainerMetadataTests { + private fun int(value: Int) = ByteBuffer.allocate(4).putInt(value).array() + private fun box(type: String, payload: ByteArray) = int(payload.size + 8) + type.toByteArray() + payload + private fun track(handler: String = "vide", entry: String = "hvc1", extension: ByteArray = byteArrayOf(), + matrix: IntArray = intArrayOf(65536, 0, 0, 0, 65536, 0, 0, 0, 0x40000000)): ByteArray { + val tkhd = ByteArray(84) + for (i in matrix.indices) int(matrix[i]).copyInto(tkhd, 40 + i * 4) + val hdlr = ByteArray(8) + handler.toByteArray() + val sampleEntry = box(entry, ByteArray(if (handler == "vide") 78 else 28) + extension) + val stsd = box("stsd", ByteArray(4) + int(1) + sampleEntry) + return box("trak", box("tkhd", tkhd) + + box("mdia", box("hdlr", hdlr) + box("minf", box("stbl", stsd)))) + } + + private fun read(data: ByteArray): MediaFact> { + val file = File.createTempFile("iso-metadata", ".mp4") + return try { file.writeBytes(data); IsoContainerMetadataReader.read(file) } finally { file.delete() } + } + + @Test fun readsContainerOrderAndDolbyConfigurationInOrdinaryHevcEntry() { + val tracks = read(box("ftyp", "isom".toByteArray()) + box("moov", + track(extension = box("dvvC", byteArrayOf(1, 0, 0, 0))) + track("soun", "mp4a"))).valueOrNull!! + assertEquals(listOf("vide", "soun"), tracks.map { it.handler }) + assertEquals(listOf("hvc1"), tracks[0].sampleEntries) + assertTrue(tracks[0].hasDolbyVisionConfiguration) + assertTrue(tracks[0].hasSimpleOrientation) + } + + @Test fun readsDolbySampleEntryEvenWithoutConfigurationChild() { + assertTrue(read(box("moov", track(entry = "dvh1"))).valueOrNull!![0].hasDolbyVisionConfiguration) + } + + @Test fun doesNotClaimSimpleGeometryForReflectionShearOrScale() { + for (matrix in listOf( + intArrayOf(-65536, 0, 0, 0, 65536, 0, 0, 0, 0x40000000), + intArrayOf(65536, 100, 0, 0, 65536, 0, 0, 0, 0x40000000), + intArrayOf(131072, 0, 0, 0, 65536, 0, 0, 0, 0x40000000))) { + assertFalse(read(box("moov", track(matrix = matrix))).valueOrNull!![0].hasSimpleOrientation) + } + } + + @Test fun recognizesQuarterRotationWithTranslation() { + val matrix = intArrayOf(0, 65536, 0, -65536, 0, 0, 1920 * 65536, 0, 0x40000000) + assertTrue(read(box("moov", track(matrix = matrix))).valueOrNull!![0].hasSimpleOrientation) + } + + @Test fun cleanApertureAndAnamorphicContainerGeometryStayUnproven() { + assertFalse(read(box("moov", track(extension = box("clap", ByteArray(32))))) + .valueOrNull!![0].hasSimpleSampleGeometry) + assertFalse(read(box("moov", track(extension = box("pasp", int(2) + int(1))))) + .valueOrNull!![0].hasSimpleSampleGeometry) + assertTrue(read(box("moov", track(extension = box("pasp", int(1) + int(1))))) + .valueOrNull!![0].hasSimpleSampleGeometry) + } + + @Test fun malformedSizesAndBoxLimitsStayUnknown() { + for (data in listOf(byteArrayOf(), int(4096) + "moov".toByteArray(), + int(4) + "moov".toByteArray(), int(1) + "moov".toByteArray() + ByteArray(8) { -1 })) { + assertEquals(MediaFact.Unknown, read(data)) + } + val many = ByteArray(4097 * 8) + val free = box("free", byteArrayOf()) + for (i in 0..4096) free.copyInto(many, i * 8) + assertEquals(MediaFact.Unknown, read(many)) + } + + @Test fun permitsOnlyZeroQuickTimeSampleDescriptionTerminator() { + assertTrue(read(box("moov", track(extension = box("dvvC", ByteArray(4)) + ByteArray(4)))) + .valueOrNull!![0].hasDolbyVisionConfiguration) + assertEquals(MediaFact.Unknown, read(box("moov", track(extension = int(1))))) + } + + @Test fun noTrackOrDuplicateMovieBoxDoesNotInventAbsence() { + assertEquals(MediaFact.Unknown, read(box("moov", byteArrayOf()))) + assertEquals(MediaFact.Unknown, read(box("moov", track()) + box("moov", track()))) + } +} diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/MediaMetadataReaderTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/MediaMetadataReaderTests.kt new file mode 100644 index 00000000..de84bd3f --- /dev/null +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/MediaMetadataReaderTests.kt @@ -0,0 +1,245 @@ +package com.mux.video.upload.internal.standardization + +import android.media.MediaFormat +import com.mux.exoplayeradapter.AbsRobolectricTest +import org.junit.Assert.* +import org.junit.Test +import org.robolectric.annotation.Config +import java.nio.ByteBuffer + +@Config(sdk = [23]) +class MediaMetadataReaderTests : AbsRobolectricTest() { + private fun hex(value: String) = value.chunked(2).map { it.toInt(16).toByte() }.toByteArray() + // SPS initialization bytes from checksum-verified shared synthetic fixtures, not media samples. + private val avc = hex("000000016742c028da01e0089f97016a02020280000003008000001e078c1950") + private val hevcSdr = hex("000001420101016000000300900000030000030078a003c0801107cb965654a4c2f016a020202080000003008000000f04") + private val hevcHlg = hex("00000001420101022000000300900000030000030078a003c0801107cad965654a4c2f016a12241208000003000800000300f040") + private val hevcPq = hex("00000001420101022000000300900000030000030078a003c0801107cad965654a4c2f016a12201208000003000800000300f040") + + private fun video(mime: String = "video/avc", csd: ByteArray? = avc): MediaFormat = + MediaFormat.createVideoFormat(mime, 1920, 1080).apply { + setInteger(MediaFormat.KEY_ROTATION, 0) + csd?.let { setByteBuffer("csd-0", ByteBuffer.wrap(it)) } + } + + private fun facts(format: MediaFormat, api: Int = 23) = + MediaTrackMetadataReader.facts(listOf(MediaTrackMetadataReader.read(0, format, api))) + + @Test fun readsAvcConfigurationAndGeometryOnApi23() { + val f = facts(video()) + assertEquals(MediaFact.Known(VideoProfile.H264Baseline), f.videoProfile) + assertEquals(MediaFact.Known(PixelFormat(8, ChromaSubsampling.Yuv420)), f.pixelFormat) + assertEquals(MediaFact.Known(DynamicRange.Sdr), f.dynamicRange) + assertEquals(MediaFact.Known(Dimensions(1920, 1080)), f.displayDimensions) + } + + @Test fun readsHevcMainMain10AndHdrFromConfigurationOnApi23() { + for ((csd, profile, depth, range) in listOf( + listOf(hevcSdr, VideoProfile.HevcMain, 8, DynamicRange.Sdr), + listOf(hevcHlg, VideoProfile.HevcMain10, 10, DynamicRange.Hlg), + listOf(hevcPq, VideoProfile.HevcMain10, 10, DynamicRange.Pq))) { + val f = facts(video("video/hevc", csd as ByteArray)) + assertEquals(MediaFact.Known(profile), f.videoProfile) + assertEquals(MediaFact.Known(PixelFormat(depth as Int, ChromaSubsampling.Yuv420)), f.pixelFormat) + assertEquals(MediaFact.Known(range), f.dynamicRange) + } + } + + @Test fun missingColorIsUnknownAndApi23IgnoresPlatformColor() { + val format = video(csd = null).apply { setInteger("color-transfer", 3) } + assertEquals(MediaFact.Unknown, facts(format).dynamicRange) + assertEquals(MediaFact.Known(DynamicRange.Sdr), facts(format, 24).dynamicRange) + format.setInteger("color-transfer", 0) + assertEquals(MediaFact.Unknown, facts(format, 24).dynamicRange) + } + + @Test fun conflictingColorAndProfileRemainUnknown() { + val format = video().apply { + setInteger("color-transfer", 7) + setInteger(MediaFormat.KEY_PROFILE, 8) // High contradicts the baseline SPS. + } + val f = facts(format, 24) + assertEquals(MediaFact.Unknown, f.dynamicRange) + assertEquals(MediaFact.Unknown, f.videoProfile) + } + + @Test fun dolbyVisionMimeOverridesBaseTransfer() { + val format = video("video/dolby-vision").apply { setInteger("color-transfer", 6) } + assertEquals(MediaFact.Known(DynamicRange.DolbyVision), facts(format, 24).dynamicRange) + } + + @Test fun metadataRateBitrateAndDurationDoNotBecomeSampleFacts() { + val format = video().apply { + setFloat(MediaFormat.KEY_FRAME_RATE, 29.97f) + setInteger(MediaFormat.KEY_BIT_RATE, 123456) + setLong(MediaFormat.KEY_DURATION, 2000000) + setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1) + } + val track = MediaTrackMetadataReader.read(0, format) + assertEquals(29.97, track.reportedFrameRate.valueOrNull!!, 0.001) + assertEquals(MediaFact.Known(123456), track.reportedBitrate) + assertEquals(MediaFact.Known(2.0), track.reportedDurationSeconds) + val f = MediaTrackMetadataReader.facts(listOf(track)) + assertEquals(MediaFact.Unknown, f.frameRate) + assertEquals(MediaFact.Unknown, f.averageBitrate) + assertEquals(MediaFact.Unknown, f.durationSeconds) + assertEquals(MediaFact.Unknown, f.maximumKeyframeIntervalSeconds) + assertEquals(MediaFact.Unknown, f.gopStructure) + assertEquals(MediaFact.Unknown, f.editList) + assertEquals(MediaFact.Unknown, f.timestamps) + } + + @Test fun normalizesSignedRotationAndSwapsPortraitAxes() { + val format = video().apply { setInteger(MediaFormat.KEY_ROTATION, -90) } + val f = facts(format) + assertEquals(MediaFact.Known(270), f.rotationDegrees) + assertEquals(MediaFact.Known(Dimensions(1080, 1920)), f.displayDimensions) + } + + @Test fun missingOrNonQuarterRotationDoesNotGuessDisplay() { + val missing = MediaFormat.createVideoFormat("video/avc", 1920, 1080) + assertEquals(MediaFact.Unknown, facts(missing).rotationDegrees) + assertEquals(MediaFact.Unknown, facts(missing).displayDimensions) + val format = video().apply { setInteger(MediaFormat.KEY_ROTATION, 45) } + assertEquals(MediaFact.Unknown, facts(format).displayDimensions) + } + + @Test fun appliesCropAndPixelAspectRatio() { + val format = video(csd = null).apply { + setInteger("crop-left", 10); setInteger("crop-right", 1909) + setInteger("crop-top", 0); setInteger("crop-bottom", 1079) + setInteger("sar-width", 2); setInteger("sar-height", 1) + setInteger(MediaFormat.KEY_ROTATION, 90) + } + assertEquals(MediaFact.Known(Dimensions(1080, 3800)), facts(format).displayDimensions) + format.setInteger("crop-right", 1920) + assertEquals(MediaFact.Unknown, facts(format).displayDimensions) + } + + @Test fun conflictingAspectRatioIsUnknown() { + val format = video().apply { setInteger("sar-width", 2); setInteger("sar-height", 1) } + assertEquals(MediaFact.Unknown, facts(format).displayDimensions) + } + + @Test fun malformedAndOversizedCsdStayUnknown() { + for (data in listOf(hex("00000167"), ByteArray(CodecMetadataReader.MAX_CONFIGURATION_BYTES + 1))) { + val f = facts(video(csd = data)) + assertEquals(MediaFact.Unknown, f.pixelFormat) + assertEquals(MediaFact.Unknown, f.dynamicRange) + } + assertEquals(CodecMetadata(), CodecMetadataReader.video(VideoCodec.H264, listOf(avc, ByteArray(65536)))) + } + + @Test fun disagreeingSpsDescriptionsStayUnknownAndBuffersAreNotMutated() { + val format = video("video/hevc", hevcHlg) + format.setByteBuffer("csd-1", ByteBuffer.wrap(hevcPq)) + val before = format.getByteBuffer("csd-0")!!.position() + assertEquals(MediaFact.Unknown, facts(format).pixelFormat) + assertEquals(before, format.getByteBuffer("csd-0")!!.position()) + } + + private fun audio(channels: Int, config: String? = null) = + MediaFormat.createAudioFormat("audio/mp4a-latm", 48000, channels).apply { + config?.let { setByteBuffer("csd-0", ByteBuffer.wrap(hex(it))) } + } + + @Test fun aacLayoutRequiresConfigurationEvidence() { + for ((count, config, layout) in listOf( + Triple(1, "1188", AudioChannelLayout.Mono), + Triple(2, "1190", AudioChannelLayout.Stereo), + Triple(6, "11b0", AudioChannelLayout.FivePointOne))) { + val track = MediaTrackMetadataReader.read(1, audio(count, config)) + assertEquals(MediaFact.Known(AudioFormat.Aac(layout)), track.audio!!.format) + } + assertEquals(MediaFact.Unknown, MediaTrackMetadataReader.read(1, audio(6)).audio!!.format) + assertEquals(MediaFact.Unknown, MediaTrackMetadataReader.read(1, audio(6, "1190")).audio!!.format) + assertEquals(MediaFact.Unknown, MediaTrackMetadataReader.read(1, audio(6, "1180")).audio!!.format) + } + + @Test fun audioTracksRetainContainerOrderAndMultipleVideoDoesNotSelectOne() { + val tracks = listOf(MediaTrackMetadataReader.read(0, video()), + MediaTrackMetadataReader.read(1, audio(2, "1190")), MediaTrackMetadataReader.read(2, audio(1, "1188"))) + assertEquals(MediaFact.Known(listOf(AudioTrack(MediaFact.Known(AudioFormat.Aac(AudioChannelLayout.Stereo))), + AudioTrack(MediaFact.Known(AudioFormat.Aac(AudioChannelLayout.Mono))))), + MediaTrackMetadataReader.facts(tracks).audioTracks) + val f = MediaTrackMetadataReader.facts(tracks + MediaTrackMetadataReader.read(3, video("video/hevc", hevcSdr))) + assertEquals(MediaFact.Known(2), f.videoTrackCount) + assertEquals(MediaFact.Unknown, f.videoCodec) + assertEquals(MediaFact.Unknown, f.displayDimensions) + } + + @Test fun readsExplicitAacFrequencyAndRejectsZeroFrequency() { + assertEquals(MediaFact.Known(AudioChannelLayout.Stereo), + CodecMetadataReader.aacLayout(hex("17805dc010"), 2)) + assertEquals(MediaFact.Unknown, CodecMetadataReader.aacLayout(hex("1780000010"), 2)) + // AAC configuration is not NAL-escaped; this byte sequence must not lose its 0x03 byte. + assertEquals(MediaFact.Known(AudioChannelLayout.Stereo), + CodecMetadataReader.aacLayout(hex("1780000310"), 2)) + } + + @Test fun audioDelayAndPaddingRemainObservationsIncludingExplicitZero() { + val format = audio(2, "1190").apply { + setInteger(MediaFormat.KEY_ENCODER_DELAY, 1024) + setInteger(MediaFormat.KEY_ENCODER_PADDING, 0) + } + val track = MediaTrackMetadataReader.read(0, format) + assertEquals(MediaFact.Known(1024), track.audio!!.encoderDelaySamples) + assertEquals(MediaFact.Known(0), track.audio.encoderPaddingSamples) + assertEquals(MediaFact.Unknown, MediaTrackMetadataReader.facts(listOf(track)).audioVideoStartOffsetSeconds) + } + + @Test fun boundsAllocationDrivingHevcReferenceCounts() { + fun ue(value: Int): String { + val binary = (value + 1).toString(2) + return "0".repeat(binary.length - 1) + binary + } + val bits = "00000001" + "00000001" + "0".repeat(88) + ue(0) + ue(1) + + ue(1920) + ue(1080) + "0" + ue(0).repeat(3) + "0" + ue(0).repeat(9) + + "0000" + ue(1) + ue(100000000) + ue(0) + val raw = bits.padEnd((bits.length + 7) / 8 * 8, '0').chunked(8).map { it.toInt(2).toByte() } + val escaped = mutableListOf(0x42, 1) + var zeroCount = 0 + for (byte in raw) { + if (zeroCount == 2 && byte.toInt() and 255 <= 3) { escaped.add(3); zeroCount = 0 } + escaped.add(byte) + zeroCount = if (byte == 0.toByte()) zeroCount + 1 else 0 + } + val csd = byteArrayOf(0, 0, 0, 1) + escaped.toByteArray() + assertEquals(CodecMetadata(), CodecMetadataReader.video(VideoCodec.Hevc, listOf(csd))) + } + + private fun isoTrack(handler: String, entry: String, id: Int, dolby: Boolean = false) = + IsoTrackMetadata(handler, MediaFact.Known(id), listOf(entry), dolby, MediaFact.Known(0), true) + + @Test fun reconcilesDolbyViewsAsOnePhysicalTrackWithoutLosingBaseGeometry() { + val iso = MediaFact.Known(listOf(isoTrack("vide", "hvc1", 1, true), isoTrack("soun", "mp4a", 2))) + val views = listOf( + MediaTrackMetadataReader.read(0, video("video/dolby-vision", null)).copy(containerIndex = MediaFact.Known(0)), + MediaTrackMetadataReader.read(1, video("video/hevc", hevcHlg)).copy(containerIndex = MediaFact.Known(0)), + MediaTrackMetadataReader.read(2, audio(2, "1190")).copy(containerIndex = MediaFact.Known(1))) + val f = MediaContainerMetadataReader.facts(MediaFact.Known(ContainerKind.IsoBaseMedia), iso, views) + assertEquals(MediaFact.Known(1), f.videoTrackCount) + assertEquals(MediaFact.Known(VideoCodec.Hevc), f.videoCodec) + assertEquals(MediaFact.Known(Dimensions(1920, 1080)), f.displayDimensions) + assertEquals(MediaFact.Known(DynamicRange.DolbyVision), f.dynamicRange) + assertEquals(MediaFact.Known(listOf(AudioTrack(MediaFact.Known(AudioFormat.Aac(AudioChannelLayout.Stereo))))), f.audioTracks) + } + + @Test fun normalizesAudioByContainerIdentityAndKeepsMissingTracksUnknown() { + val iso = MediaFact.Known(listOf(isoTrack("vide", "avc1", 1), isoTrack("soun", "mp4a", 2), isoTrack("soun", "mp4a", 3))) + val views = listOf(MediaTrackMetadataReader.read(0, video()).copy(containerIndex = MediaFact.Known(0)), + MediaTrackMetadataReader.read(1, audio(1, "1188")).copy(containerIndex = MediaFact.Known(2)), + MediaTrackMetadataReader.read(2, audio(2, "1190")).copy(containerIndex = MediaFact.Known(1))) + val f = MediaContainerMetadataReader.facts(MediaFact.Known(ContainerKind.IsoBaseMedia), iso, views) + assertEquals(listOf(AudioChannelLayout.Stereo, AudioChannelLayout.Mono), + f.audioTracks.valueOrNull!!.map { (it.format.valueOrNull as AudioFormat.Aac).layout }) + assertEquals(MediaFact.Unknown, MediaContainerMetadataReader.facts( + MediaFact.Known(ContainerKind.IsoBaseMedia), iso, views.dropLast(1)).audioTracks) + } + + @Test fun unknownMimePreventsClaimingCompleteTrackStructure() { + val f = MediaTrackMetadataReader.facts(listOf(MediaTrackMetadataReader.read(0, video()), + MediaTrackMetadataReader.read(1, MediaFormat()))) + assertEquals(MediaFacts(), f) + } +} diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 00000000..1765275c --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,32 @@ +# Metadata fixture instrumentation + +Run the library's metadata inspector on a local emulator using the shared Swift fixture catalog and an external media directory. Python 3.9+, JDK 17, an Android SDK with `adb`, and FFmpeg are required. No media binary is stored in this repository. + +```sh +export JAVA_HOME=/path/to/jdk-17 +export ANDROID_HOME=/path/to/android-sdk +python3 scripts/run-metadata-fixtures.py \ + --catalog /path/to/swift-upload-sdk/Tests/MediaFixtures/catalog.json \ + --media /path/to/external-media \ + --serial emulator-5554 \ + --evidence /tmp/android-metadata-evidence +``` + +Use an already-running local emulator with sufficient free storage. The runner does not create an emulator, clear existing app data, or use physical devices. It installs a library test APK, transfers one verified fixture at a time into that test app's external-files directory, and removes each staged input afterward. The test app remains installed for inspection. Fixtures too large for available storage are recorded as skipped; a skip is not successful inspection evidence. + +The runner verifies the shared catalog's file sizes and SHA-256 values, packages only the catalog as test assets, and verifies each transferred file again before inspection. Temporary build assets and generated Android cases remain outside product Git. The evidence directory contains the run catalog, FFmpeg version, build log, per-fixture test logs, and `metadata-results.json`. + +The shared catalog stays unchanged. The run catalog adds two Android cases: distinguishable stereo and mono audio tracks with the second marked default, and a malformed container. This establishes container-order inspection, not Mux primary/default ingest selection. The library test target SDK is set separately from the shipped library's consumer requirements. + +Each readable fixture records a first inspection and ten repeated metadata inspections. Reported median, p90 and maximum exclude file transfer, fixture hashing and copying. The first call is not a cold-disk measurement: checksum validation has already read the file. These are local debug-emulator observations, not physical-device performance or release acceptance limits. + +The inspector exposes typed normalized facts alongside per-track metadata and container evidence. It keeps extractor indices separate from physical container indices; Dolby Vision may expose both a Dolby view and an HEVC base view of one physical track. Container matrices provide normalized orientation while raw Android rotation remains visible. Reported duration, nominal rate, bitrate, encoder delay and padding remain observations, not effective-timeline, cadence, or bitrate measurements. + +Compressed samples are not read by this implementation. GOP/IDR, sample timestamps/cadence, measured rates/bitrates, A/V start offset and effective duration/edit-list facts remain unknown. Unproven configurations, conflicting evidence, complex geometry and absent color stay unknown. HEVC multilayer/predicted reference-set configuration, AAC PCE/extension layouts, anamorphic container geometry and clean-aperture extensions need additional bounded metadata support before those properties can be claimed. API 23 color handling has unit/Robolectric coverage; this runner proves only the selected emulator's real platform behavior. + +The ordinary unit/build check is: + +```sh +./gradlew :library:testDebugUnitTest :library:testReleaseUnitTest \ + :library:assembleRelease :app:assembleRelease --console=plain +``` diff --git a/scripts/run-metadata-fixtures.py b/scripts/run-metadata-fixtures.py new file mode 100644 index 00000000..40838f56 --- /dev/null +++ b/scripts/run-metadata-fixtures.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +"""Run metadata-only instrumentation from an external Swift fixture catalog on a local emulator.""" +import argparse +import hashlib +import json +import os +import re +from pathlib import Path +import shutil +import subprocess +import tempfile + + +def run(command, **kwargs): + try: + return subprocess.run(command, check=True, **kwargs) + except subprocess.CalledProcessError as error: + if error.stdout: print(error.stdout) + if error.stderr: print(error.stderr) + raise + + +def sha256(path): + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--catalog", type=Path, required=True) + parser.add_argument("--media", type=Path, required=True) + parser.add_argument("--serial", required=True, help="An already-running local emulator") + parser.add_argument("--evidence", type=Path, required=True, help="External evidence directory") + args = parser.parse_args() + repo = Path(__file__).resolve().parents[1] + adb = str(Path(os.environ["ANDROID_HOME"]) / "platform-tools/adb") + prefix = [adb, "-s", args.serial] + qemu = run(prefix + ["shell", "getprop", "ro.kernel.qemu"], capture_output=True, text=True).stdout.strip() + if qemu != "1": + parser.error("This runner requires a local emulator; physical-device participation needs separate authorization.") + evidence = args.evidence.resolve() + product_root = Path(run(["git", "-C", str(repo), "rev-parse", "--path-format=absolute", "--git-common-dir"], + capture_output=True, text=True).stdout.strip()).parent + if evidence.is_relative_to(product_root): + parser.error("Keep fixture binaries and evidence outside the product checkout.") + evidence.mkdir(parents=True, exist_ok=True) + catalog = json.loads(args.catalog.read_text()) + if catalog["schemaVersion"] != 1: + parser.error("Unsupported shared catalog version") + names = set() + ids = set() + for fixture in catalog["fixtures"]: + fixture_id = fixture["id"] + if not re.fullmatch(r"[a-z0-9][a-z0-9-]+", fixture_id) or fixture_id in ids: + parser.error("Fixture IDs must be unique safe identifiers") + ids.add(fixture_id) + name = fixture["canonicalFilename"] + if Path(name).name != name or name in names: + parser.error("Fixture filenames must be unique basenames") + names.add(name) + source = args.media / name + if source.stat().st_size != fixture["sizeBytes"] or sha256(source) != fixture["sha256"]: + parser.error("Fixture integrity failed: " + fixture["id"]) + print("Verified", len(names), "external fixtures", flush=True) + stage = Path(tempfile.mkdtemp(prefix="android-metadata-assets-")) + package = "com.mux.video.upload.test" + device_dir = "/sdcard/Android/data/" + package + "/files/metadata-fixtures" + try: + assets = stage / "assets/fixtures" + assets.mkdir(parents=True) + multi = stage / "android-distinct-multi-audio.mp4" + source = args.media / "standard-h264-sdr-1080p30-aac-stereo.mp4" + run(["ffmpeg", "-v", "error", "-nostdin", "-y", "-i", str(source), + "-f", "lavfi", "-i", "sine=frequency=880:sample_rate=48000:duration=3", + "-map", "0:v:0", "-map", "0:a:0", "-map", "1:a:0", "-c:v", "copy", + "-c:a", "aac", "-ac:a:0", "2", "-ac:a:1", "1", "-disposition:a:0", "0", + "-disposition:a:1", "default", "-shortest", str(multi)]) + if "android-distinct-multi-audio" in ids or "android-malformed-container" in ids: + parser.error("Android overlay fixture IDs already exist in the source catalog") + source_fixture = next(f for f in catalog["fixtures"] if f["canonicalFilename"] == source.name) + catalog["fixtures"].append({ + "id": "android-distinct-multi-audio", "canonicalFilename": multi.name, + "sizeBytes": multi.stat().st_size, "sha256": sha256(multi), + "materialization": {"kind": "derived", "sourceSha256": source_fixture["sha256"]}, + "facts": {"videoCodec": "h264", "encodedDimensions": [1920, 1080], + "displayDimensions": [1920, 1080], "dynamicRange": "sdr", + "audioLayouts": ["Stereo", "Mono"]}, + }) + malformed = stage / "android-malformed-container.mp4" + malformed.write_bytes(bytes([42]) * 64) + catalog["fixtures"].append({ + "id": "android-malformed-container", "canonicalFilename": malformed.name, + "sizeBytes": malformed.stat().st_size, "sha256": sha256(malformed), + "materialization": {"kind": "generated"}, "facts": {}, + "expectedMetadataFailure": True, + }) + (assets / "catalog.json").write_text(json.dumps(catalog, indent=2)) + (evidence / "run-catalog.json").write_text(json.dumps(catalog, indent=2)) + (evidence / "ffmpeg-version.txt").write_text(run(["ffmpeg", "-version"], capture_output=True, text=True).stdout) + env = dict(os.environ, ANDROID_SERIAL=args.serial) + with (evidence / "build.log").open("w") as log: + run([str(repo / "gradlew"), ":library:assembleDebugAndroidTest", + "-PmediaFixtureAssets=" + str(stage / "assets"), "--console=plain"], + cwd=repo, env=env, stdout=log, stderr=subprocess.STDOUT) + apk = repo / "library/build/outputs/apk/androidTest/debug/library-debug-androidTest.apk" + run(prefix + ["install", "-r", "-t", str(apk)], capture_output=True) + run(prefix + ["shell", "mkdir", "-p", device_dir]) + results = [] + for fixture in catalog["fixtures"]: + # Leave space for emulator services; never clear unrelated apps or data. + available = int(run(prefix + ["shell", "df", "-k", "/data"], + capture_output=True, text=True).stdout.splitlines()[-1].split()[3]) * 1024 + if fixture["sizeBytes"] + 128 * 1024 * 1024 > available: + results.append({"id": fixture["id"], "status": "insufficientEmulatorStorage"}) + continue + local = stage / fixture["canonicalFilename"] if fixture["id"].startswith("android-") else args.media / fixture["canonicalFilename"] + remote = device_dir + "/" + fixture["canonicalFilename"] + try: + run(prefix + ["push", str(local), remote], capture_output=True) + output = run(prefix + ["shell", "am", "instrument", "-w", "-r", + "-e", "class", "com.mux.video.upload.internal.standardization.MediaMetadataFixtureTests", + "-e", "fixtureId", fixture["id"], package + "/androidx.test.runner.AndroidJUnitRunner"], + capture_output=True, text=True).stdout + (evidence / (fixture["id"] + ".log")).write_text(output) + if "FAILURES!!!" in output or "OK (2 tests)" not in output: + raise RuntimeError("Instrumentation failed: " + fixture["id"]) + report = run(prefix + ["exec-out", "run-as", package, "cat", "files/metadata-results.json"], + capture_output=True).stdout + data = json.loads(report) + results.extend(data["results"]) + data["results"] = results + (evidence / "metadata-results.json").write_text(json.dumps(data, indent=2)) + print("Inspected", fixture["id"], flush=True) + finally: + run(prefix + ["shell", "rm", "-f", remote]) + print("Evidence:", evidence, flush=True) + finally: + shutil.rmtree(stage) + + +if __name__ == "__main__": + main() From 047d2d99f7883efd5e47ecded152941c04c50637 Mon Sep 17 00:00:00 2001 From: Eugene Morozov Date: Fri, 2 Oct 2026 12:54:56 -0700 Subject: [PATCH 02/49] Keep metadata fixture validation tooling external --- library/build.gradle | 10 -- .../MediaMetadataFixtureTests.kt | 152 ------------------ scripts/README.md | 32 ---- scripts/run-metadata-fixtures.py | 145 ----------------- 4 files changed, 339 deletions(-) delete mode 100644 library/src/androidTest/java/com/mux/video/upload/internal/standardization/MediaMetadataFixtureTests.kt delete mode 100644 scripts/README.md delete mode 100644 scripts/run-metadata-fixtures.py diff --git a/library/build.gradle b/library/build.gradle index 04491ed5..d7eff249 100644 --- a/library/build.gradle +++ b/library/build.gradle @@ -32,13 +32,6 @@ android { kotlinOptions { jvmTarget = '1.8' } - // Target only the instrumentation APK at a current API level. - testOptions { targetSdk = 36 } - - // External fixture assets are staged outside product Git by the manual test runner. - if (project.hasProperty("mediaFixtureAssets")) { - sourceSets.androidTest.assets.srcDir(project.property("mediaFixtureAssets")) - } buildFeatures { buildConfig true } @@ -100,9 +93,6 @@ dependencies { implementation 'androidx.core:core-ktx:1.13.1' implementation "org.jetbrains.kotlinx:kotlinx-coroutines-android:1.8.1" - androidTestImplementation 'androidx.test:runner:1.6.2' - androidTestImplementation 'androidx.test.ext:junit:1.2.1' - testImplementation 'junit:junit:4.13.2' testImplementation 'org.jetbrains.kotlinx:kotlinx-coroutines-test:1.8.1' testImplementation 'androidx.test.ext:junit:1.2.1' diff --git a/library/src/androidTest/java/com/mux/video/upload/internal/standardization/MediaMetadataFixtureTests.kt b/library/src/androidTest/java/com/mux/video/upload/internal/standardization/MediaMetadataFixtureTests.kt deleted file mode 100644 index 1490ea6b..00000000 --- a/library/src/androidTest/java/com/mux/video/upload/internal/standardization/MediaMetadataFixtureTests.kt +++ /dev/null @@ -1,152 +0,0 @@ -package com.mux.video.upload.internal.standardization - -import android.os.Build -import androidx.test.ext.junit.runners.AndroidJUnit4 -import androidx.test.platform.app.InstrumentationRegistry -import org.json.JSONArray -import org.json.JSONObject -import org.junit.Assert.* -import org.junit.Test -import org.junit.runner.RunWith -import java.io.File -import java.security.MessageDigest - -@RunWith(AndroidJUnit4::class) -class MediaMetadataFixtureTests { - @Test fun inspectExternalCatalogAndMeasureMetadata() { - val instrumentation = InstrumentationRegistry.getInstrumentation() - val assets = instrumentation.context.assets - val catalog = JSONObject(assets.open("fixtures/catalog.json").bufferedReader().use { it.readText() }) - val fixtures = catalog.getJSONArray("fixtures") - val results = JSONArray() - val directory = instrumentation.targetContext.getExternalFilesDir("metadata-fixtures")!! - val fixtureId = InstrumentationRegistry.getArguments().getString("fixtureId") - require(fixtureId != null) { "Use scripts/run-metadata-fixtures.py to stage a verified fixture" } - run { - for (i in 0 until fixtures.length()) { - val fixture = fixtures.getJSONObject(i) - if (fixture.getString("id") != fixtureId) continue - val file = File(directory, fixture.getString("canonicalFilename")) - assertEquals(fixture.getLong("sizeBytes"), file.length()) - val digest = MessageDigest.getInstance("SHA-256") - file.inputStream().use { input -> - val buffer = ByteArray(64 * 1024) - while (true) { val count = input.read(buffer); if (count < 0) break; digest.update(buffer, 0, count) } - } - assertEquals(fixture.getString("sha256"), digest.digest().joinToString("") { "%02x".format(it) }) - val inspector = MediaMetadataInspector() - val initial = inspector.inspect(file) - val row = JSONObject().put("id", fixture.getString("id")).put("sha256", fixture.getString("sha256")) - if (initial is MetadataInspectionResult.Success) { - assertFalse("Expected malformed fixture rejection", fixture.optBoolean("expectedMetadataFailure")) - val inspection = initial.inspection - assertDeferredFacts(inspection.facts) - val expected = fixture.getJSONObject("facts") - println("METADATA " + fixture.getString("id") + " " + inspection) - verifyMetadata(fixture, expected, inspection) - val times = mutableListOf() - repeat(10) { - val repeated = inspector.inspect(file) as MetadataInspectionResult.Success - assertEquals(inspection.facts, repeated.inspection.facts) - times.add(repeated.inspection.elapsedNanos) - } - times.sort() - row.put("status", "success").put("initialMs", inspection.elapsedNanos / 1e6) - .put("medianMs", (times[4] + times[5]) / 2e6).put("p90Ms", times[8] / 1e6) - .put("maxMs", times.last() / 1e6).put("facts", inspection.facts.toString()) - .put("tracks", JSONArray(inspection.tracks.map { it.toString() })) - .put("isoTracks", inspection.isoTracks.toString()) - } else { - row.put("status", initial.toString()) - assertTrue("Required fixture unreadable: " + fixture.getString("id"), - fixture.optBoolean("expectedMetadataFailure", false) || - fixture.getJSONObject("facts").optString("videoCodec") == "prores") - } - results.put(row) - file.delete() - } - val report = JSONObject().put("api", Build.VERSION.SDK_INT).put("model", Build.MODEL) - .put("catalogId", catalog.getString("catalogId")).put("sampleScanPerformed", false) - .put("iterations", 10).put("results", results) - File(instrumentation.targetContext.filesDir, "metadata-results.json").writeText(report.toString(2)) - } - } - - private fun verifyMetadata(fixture: JSONObject, expected: JSONObject, inspection: MediaMetadataInspection) { - val facts = inspection.facts - val strict = fixture.getString("id").startsWith("synthetic-standard") - val expectedCodec = when (expected.optString("videoCodec")) { - "h264" -> VideoCodec.H264 - "hevc" -> VideoCodec.Hevc - else -> VideoCodec.Other - } - if (facts.videoCodec != MediaFact.Unknown && !fixture.getString("id").contains("dolby-vision")) - assertEquals(MediaFact.Known(expectedCodec), facts.videoCodec) - for ((key, value) in listOf("encodedDimensions" to facts.encodedDimensions, "displayDimensions" to facts.displayDimensions)) { - val dimensions = expected.optJSONArray(key) ?: continue - if (strict || value != MediaFact.Unknown) - assertEquals(fixture.getString("id") + " " + key, - MediaFact.Known(Dimensions(dimensions.getInt(0), dimensions.getInt(1))), value) - } - val range = if (fixture.getString("id").contains("dolby-vision")) DynamicRange.DolbyVision - else when (expected.optString("dynamicRange")) { - "sdr" -> DynamicRange.Sdr - "hlg" -> DynamicRange.Hlg - "pq" -> DynamicRange.Pq - "dolbyVision" -> DynamicRange.DolbyVision - else -> null - } - if (range != null && (strict || facts.dynamicRange != MediaFact.Unknown)) - assertEquals(fixture.getString("id") + " range", MediaFact.Known(range), facts.dynamicRange) - if (expected.has("orientationDegrees") && facts.rotationDegrees != MediaFact.Unknown) { - val degrees = expected.getInt("orientationDegrees") - assertEquals(MediaFact.Known((degrees % 360 + 360) % 360), facts.rotationDegrees) - } - if (fixture.getString("id").contains("dolby-vision")) - assertEquals(MediaFact.Known(DynamicRange.DolbyVision), facts.dynamicRange) - if (strict) { - assertEquals(MediaFact.Known(PixelFormat(expected.getInt("bitDepth"), ChromaSubsampling.Yuv420)), facts.pixelFormat) - assertEquals(MediaFact.Known(expected.getInt("orientationDegrees")), facts.rotationDegrees) - val layout = when (expected.getString("audioLayout")) { - "5.1" -> AudioChannelLayout.FivePointOne - else -> AudioChannelLayout.Stereo - } - assertEquals(MediaFact.Known(listOf(AudioTrack(MediaFact.Known(AudioFormat.Aac(layout))))), facts.audioTracks) - } - expected.optJSONArray("audioLayouts")?.let { layouts -> - assertEquals(layouts.length(), facts.audioTracks.valueOrNull!!.size) - for (i in 0 until layouts.length()) { - val layout = AudioChannelLayout.valueOf(layouts.getString(i)) - assertEquals(MediaFact.Known(AudioFormat.Aac(layout)), facts.audioTracks.valueOrNull!![i].format) - } - assertEquals(listOf(0, 1, 2).map { MediaFact.Known(it) }, inspection.tracks.map { it.containerIndex }) - } - } - - private fun assertDeferredFacts(facts: MediaFacts) { - assertEquals(MediaFact.Unknown, facts.gopStructure) - assertEquals(MediaFact.Unknown, facts.cadence) - assertEquals(MediaFact.Unknown, facts.timestamps) - assertEquals(MediaFact.Unknown, facts.frameRate) - assertEquals(MediaFact.Unknown, facts.averageBitrate) - assertEquals(MediaFact.Unknown, facts.maximumGopBitrate) - assertEquals(MediaFact.Unknown, facts.maximumGopByteSize) - assertEquals(MediaFact.Unknown, facts.maximumKeyframeIntervalSeconds) - assertEquals(MediaFact.Unknown, facts.durationSeconds) - assertEquals(MediaFact.Unknown, facts.audioVideoStartOffsetSeconds) - assertEquals(MediaFact.Unknown, facts.editList) - } - - @Test fun rejectMissingEmptyAndMalformedContainers() { - val context = InstrumentationRegistry.getInstrumentation().targetContext - val file = File(context.cacheDir, "metadata-invalid.mp4") - try { - file.delete() - assertTrue(MediaMetadataInspector().inspect(file) is MetadataInspectionResult.Failure) - file.writeBytes(byteArrayOf()) - assertTrue(MediaMetadataInspector().inspect(file) is MetadataInspectionResult.Failure) - file.writeBytes(ByteArray(64) { 42 }) - assertTrue(MediaMetadataInspector().inspect(file) is MetadataInspectionResult.Failure) - } finally { file.delete() } - } -} diff --git a/scripts/README.md b/scripts/README.md deleted file mode 100644 index 1765275c..00000000 --- a/scripts/README.md +++ /dev/null @@ -1,32 +0,0 @@ -# Metadata fixture instrumentation - -Run the library's metadata inspector on a local emulator using the shared Swift fixture catalog and an external media directory. Python 3.9+, JDK 17, an Android SDK with `adb`, and FFmpeg are required. No media binary is stored in this repository. - -```sh -export JAVA_HOME=/path/to/jdk-17 -export ANDROID_HOME=/path/to/android-sdk -python3 scripts/run-metadata-fixtures.py \ - --catalog /path/to/swift-upload-sdk/Tests/MediaFixtures/catalog.json \ - --media /path/to/external-media \ - --serial emulator-5554 \ - --evidence /tmp/android-metadata-evidence -``` - -Use an already-running local emulator with sufficient free storage. The runner does not create an emulator, clear existing app data, or use physical devices. It installs a library test APK, transfers one verified fixture at a time into that test app's external-files directory, and removes each staged input afterward. The test app remains installed for inspection. Fixtures too large for available storage are recorded as skipped; a skip is not successful inspection evidence. - -The runner verifies the shared catalog's file sizes and SHA-256 values, packages only the catalog as test assets, and verifies each transferred file again before inspection. Temporary build assets and generated Android cases remain outside product Git. The evidence directory contains the run catalog, FFmpeg version, build log, per-fixture test logs, and `metadata-results.json`. - -The shared catalog stays unchanged. The run catalog adds two Android cases: distinguishable stereo and mono audio tracks with the second marked default, and a malformed container. This establishes container-order inspection, not Mux primary/default ingest selection. The library test target SDK is set separately from the shipped library's consumer requirements. - -Each readable fixture records a first inspection and ten repeated metadata inspections. Reported median, p90 and maximum exclude file transfer, fixture hashing and copying. The first call is not a cold-disk measurement: checksum validation has already read the file. These are local debug-emulator observations, not physical-device performance or release acceptance limits. - -The inspector exposes typed normalized facts alongside per-track metadata and container evidence. It keeps extractor indices separate from physical container indices; Dolby Vision may expose both a Dolby view and an HEVC base view of one physical track. Container matrices provide normalized orientation while raw Android rotation remains visible. Reported duration, nominal rate, bitrate, encoder delay and padding remain observations, not effective-timeline, cadence, or bitrate measurements. - -Compressed samples are not read by this implementation. GOP/IDR, sample timestamps/cadence, measured rates/bitrates, A/V start offset and effective duration/edit-list facts remain unknown. Unproven configurations, conflicting evidence, complex geometry and absent color stay unknown. HEVC multilayer/predicted reference-set configuration, AAC PCE/extension layouts, anamorphic container geometry and clean-aperture extensions need additional bounded metadata support before those properties can be claimed. API 23 color handling has unit/Robolectric coverage; this runner proves only the selected emulator's real platform behavior. - -The ordinary unit/build check is: - -```sh -./gradlew :library:testDebugUnitTest :library:testReleaseUnitTest \ - :library:assembleRelease :app:assembleRelease --console=plain -``` diff --git a/scripts/run-metadata-fixtures.py b/scripts/run-metadata-fixtures.py deleted file mode 100644 index 40838f56..00000000 --- a/scripts/run-metadata-fixtures.py +++ /dev/null @@ -1,145 +0,0 @@ -#!/usr/bin/env python3 -"""Run metadata-only instrumentation from an external Swift fixture catalog on a local emulator.""" -import argparse -import hashlib -import json -import os -import re -from pathlib import Path -import shutil -import subprocess -import tempfile - - -def run(command, **kwargs): - try: - return subprocess.run(command, check=True, **kwargs) - except subprocess.CalledProcessError as error: - if error.stdout: print(error.stdout) - if error.stderr: print(error.stderr) - raise - - -def sha256(path): - digest = hashlib.sha256() - with path.open("rb") as source: - for chunk in iter(lambda: source.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--catalog", type=Path, required=True) - parser.add_argument("--media", type=Path, required=True) - parser.add_argument("--serial", required=True, help="An already-running local emulator") - parser.add_argument("--evidence", type=Path, required=True, help="External evidence directory") - args = parser.parse_args() - repo = Path(__file__).resolve().parents[1] - adb = str(Path(os.environ["ANDROID_HOME"]) / "platform-tools/adb") - prefix = [adb, "-s", args.serial] - qemu = run(prefix + ["shell", "getprop", "ro.kernel.qemu"], capture_output=True, text=True).stdout.strip() - if qemu != "1": - parser.error("This runner requires a local emulator; physical-device participation needs separate authorization.") - evidence = args.evidence.resolve() - product_root = Path(run(["git", "-C", str(repo), "rev-parse", "--path-format=absolute", "--git-common-dir"], - capture_output=True, text=True).stdout.strip()).parent - if evidence.is_relative_to(product_root): - parser.error("Keep fixture binaries and evidence outside the product checkout.") - evidence.mkdir(parents=True, exist_ok=True) - catalog = json.loads(args.catalog.read_text()) - if catalog["schemaVersion"] != 1: - parser.error("Unsupported shared catalog version") - names = set() - ids = set() - for fixture in catalog["fixtures"]: - fixture_id = fixture["id"] - if not re.fullmatch(r"[a-z0-9][a-z0-9-]+", fixture_id) or fixture_id in ids: - parser.error("Fixture IDs must be unique safe identifiers") - ids.add(fixture_id) - name = fixture["canonicalFilename"] - if Path(name).name != name or name in names: - parser.error("Fixture filenames must be unique basenames") - names.add(name) - source = args.media / name - if source.stat().st_size != fixture["sizeBytes"] or sha256(source) != fixture["sha256"]: - parser.error("Fixture integrity failed: " + fixture["id"]) - print("Verified", len(names), "external fixtures", flush=True) - stage = Path(tempfile.mkdtemp(prefix="android-metadata-assets-")) - package = "com.mux.video.upload.test" - device_dir = "/sdcard/Android/data/" + package + "/files/metadata-fixtures" - try: - assets = stage / "assets/fixtures" - assets.mkdir(parents=True) - multi = stage / "android-distinct-multi-audio.mp4" - source = args.media / "standard-h264-sdr-1080p30-aac-stereo.mp4" - run(["ffmpeg", "-v", "error", "-nostdin", "-y", "-i", str(source), - "-f", "lavfi", "-i", "sine=frequency=880:sample_rate=48000:duration=3", - "-map", "0:v:0", "-map", "0:a:0", "-map", "1:a:0", "-c:v", "copy", - "-c:a", "aac", "-ac:a:0", "2", "-ac:a:1", "1", "-disposition:a:0", "0", - "-disposition:a:1", "default", "-shortest", str(multi)]) - if "android-distinct-multi-audio" in ids or "android-malformed-container" in ids: - parser.error("Android overlay fixture IDs already exist in the source catalog") - source_fixture = next(f for f in catalog["fixtures"] if f["canonicalFilename"] == source.name) - catalog["fixtures"].append({ - "id": "android-distinct-multi-audio", "canonicalFilename": multi.name, - "sizeBytes": multi.stat().st_size, "sha256": sha256(multi), - "materialization": {"kind": "derived", "sourceSha256": source_fixture["sha256"]}, - "facts": {"videoCodec": "h264", "encodedDimensions": [1920, 1080], - "displayDimensions": [1920, 1080], "dynamicRange": "sdr", - "audioLayouts": ["Stereo", "Mono"]}, - }) - malformed = stage / "android-malformed-container.mp4" - malformed.write_bytes(bytes([42]) * 64) - catalog["fixtures"].append({ - "id": "android-malformed-container", "canonicalFilename": malformed.name, - "sizeBytes": malformed.stat().st_size, "sha256": sha256(malformed), - "materialization": {"kind": "generated"}, "facts": {}, - "expectedMetadataFailure": True, - }) - (assets / "catalog.json").write_text(json.dumps(catalog, indent=2)) - (evidence / "run-catalog.json").write_text(json.dumps(catalog, indent=2)) - (evidence / "ffmpeg-version.txt").write_text(run(["ffmpeg", "-version"], capture_output=True, text=True).stdout) - env = dict(os.environ, ANDROID_SERIAL=args.serial) - with (evidence / "build.log").open("w") as log: - run([str(repo / "gradlew"), ":library:assembleDebugAndroidTest", - "-PmediaFixtureAssets=" + str(stage / "assets"), "--console=plain"], - cwd=repo, env=env, stdout=log, stderr=subprocess.STDOUT) - apk = repo / "library/build/outputs/apk/androidTest/debug/library-debug-androidTest.apk" - run(prefix + ["install", "-r", "-t", str(apk)], capture_output=True) - run(prefix + ["shell", "mkdir", "-p", device_dir]) - results = [] - for fixture in catalog["fixtures"]: - # Leave space for emulator services; never clear unrelated apps or data. - available = int(run(prefix + ["shell", "df", "-k", "/data"], - capture_output=True, text=True).stdout.splitlines()[-1].split()[3]) * 1024 - if fixture["sizeBytes"] + 128 * 1024 * 1024 > available: - results.append({"id": fixture["id"], "status": "insufficientEmulatorStorage"}) - continue - local = stage / fixture["canonicalFilename"] if fixture["id"].startswith("android-") else args.media / fixture["canonicalFilename"] - remote = device_dir + "/" + fixture["canonicalFilename"] - try: - run(prefix + ["push", str(local), remote], capture_output=True) - output = run(prefix + ["shell", "am", "instrument", "-w", "-r", - "-e", "class", "com.mux.video.upload.internal.standardization.MediaMetadataFixtureTests", - "-e", "fixtureId", fixture["id"], package + "/androidx.test.runner.AndroidJUnitRunner"], - capture_output=True, text=True).stdout - (evidence / (fixture["id"] + ".log")).write_text(output) - if "FAILURES!!!" in output or "OK (2 tests)" not in output: - raise RuntimeError("Instrumentation failed: " + fixture["id"]) - report = run(prefix + ["exec-out", "run-as", package, "cat", "files/metadata-results.json"], - capture_output=True).stdout - data = json.loads(report) - results.extend(data["results"]) - data["results"] = results - (evidence / "metadata-results.json").write_text(json.dumps(data, indent=2)) - print("Inspected", fixture["id"], flush=True) - finally: - run(prefix + ["shell", "rm", "-f", remote]) - print("Evidence:", evidence, flush=True) - finally: - shutil.rmtree(stage) - - -if __name__ == "__main__": - main() From b3a6aa114b605a447bf73e11ce9e05a0032a0ce0 Mon Sep 17 00:00:00 2001 From: Eugene Morozov Date: Fri, 2 Oct 2026 13:39:57 -0700 Subject: [PATCH 03/49] Resolve metadata review findings and inspect effective timelines --- .../internal/standardization/CodecMetadata.kt | 80 +++++---- .../standardization/IsoContainerMetadata.kt | 123 +++++++++---- .../standardization/IsoTimelineMetadata.kt | 170 ++++++++++++++++++ .../standardization/MediaMetadataInspector.kt | 133 +++++++++----- .../IsoContainerMetadataTests.kt | 38 +++- .../IsoTimelineMetadataTests.kt | 164 +++++++++++++++++ .../MediaMetadataReaderTests.kt | 99 +++++++++- 7 files changed, 696 insertions(+), 111 deletions(-) create mode 100644 library/src/main/java/com/mux/video/upload/internal/standardization/IsoTimelineMetadata.kt create mode 100644 library/src/test/java/com/mux/video/upload/internal/standardization/IsoTimelineMetadataTests.kt diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/CodecMetadata.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/CodecMetadata.kt index b2af00b1..1a8ca8b0 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/CodecMetadata.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/CodecMetadata.kt @@ -3,6 +3,7 @@ package com.mux.video.upload.internal.standardization import androidx.media3.common.util.ParsableBitArray import androidx.media3.container.NalUnitUtil import androidx.media3.container.ParsableNalUnitBitArray +import androidx.media3.extractor.AacUtil /** Color codes use Media3/Android's normalized constants, not ISO code points. */ internal data class ColorMetadata( @@ -131,13 +132,37 @@ internal object CodecMetadataReader { } val sets = bits.readUnsignedExpGolombCodedInt() if (sets !in 0..64) return false + var previousPocs = emptyList() for (i in 0 until sets) { - // Predicted reference sets need a fuller bounded syntax reader; leave them unknown for now. - if (i > 0 && bits.readBit()) return false - val negative = bits.readUnsignedExpGolombCodedInt() - val positive = bits.readUnsignedExpGolombCodedInt() - if (negative !in 0..16 || positive !in 0..16 || negative + positive > 16) return false - repeat(negative + positive) { bits.readUnsignedExpGolombCodedInt(); bits.skipBit() } + if (i > 0 && bits.readBit()) { + val sign = if (bits.readBit()) -1 else 1 + val delta = bits.readUnsignedExpGolombCodedInt() + if (delta !in 0..65535) return false + val deltaRps = sign * (delta + 1) + // In an SPS the reference is always the preceding set; delta_idx_minus1 is absent. + val predicted = (previousPocs + 0).mapNotNull { poc -> + val used = bits.readBit() || bits.readBit() + (poc + deltaRps).takeIf { used && it != 0 } + } + if (predicted.size > 16) return false + previousPocs = predicted.filter { it < 0 }.sortedDescending() + predicted.filter { it > 0 }.sorted() + } else { + val negative = bits.readUnsignedExpGolombCodedInt() + val positive = bits.readUnsignedExpGolombCodedInt() + if (negative !in 0..16 || positive !in 0..16 || negative + positive > 16) return false + val pocs = mutableListOf() + for ((count, sign) in listOf(negative to -1, positive to 1)) { + var poc = 0 + repeat(count) { + val delta = bits.readUnsignedExpGolombCodedInt() + if (delta !in 0..65535) return false + poc += sign * (delta + 1) + pocs.add(poc) + bits.skipBit() + } + } + previousPocs = pocs + } } if (bits.readBit()) { val count = bits.readUnsignedExpGolombCodedInt() @@ -157,44 +182,37 @@ internal object CodecMetadataReader { /** MediaExtractor supplies AVC/HEVC CSD in Annex B form, with three- or four-byte prefixes. */ private fun annexBNalUnits(data: ByteArray): List { - val starts = mutableListOf>() - var i = 0 - while (i + 2 < data.size) { - val prefix = when { - data[i] != 0.toByte() || data[i + 1] != 0.toByte() -> 0 - data[i + 2] == 1.toByte() -> 3 - i + 3 < data.size && data[i + 2] == 0.toByte() && data[i + 3] == 1.toByte() -> 4 - else -> 0 - } - if (prefix > 0) { starts.add(i to prefix); i += prefix } else i++ - } - return starts.mapIndexed { index, (start, prefix) -> - data.copyOfRange(start + prefix, starts.getOrNull(index + 1)?.first ?: data.size) + val units = mutableListOf() + val flags = BooleanArray(3) + var prefix = NalUnitUtil.findNalUnit(data, 0, data.size, flags) + while (prefix < data.size) { + val payload = prefix + 3 + NalUnitUtil.clearPrefixFlags(flags) + val next = NalUnitUtil.findNalUnit(data, payload, data.size, flags) + // A four-byte prefix includes a leading zero outside the three-byte start code. + val end = if (next > payload && next < data.size && data[next - 1] == 0.toByte()) next - 1 else next + if (payload < end) units.add(data.copyOfRange(payload, end)) + prefix = next } + return units } /** Channel count alone cannot prove a surround layout. PCE/extension configurations stay unknown. */ fun aacLayout(configuration: ByteArray?, channelCount: Int?): MediaFact { if (configuration == null || configuration.size !in 2..MAX_CONFIGURATION_BYTES) return MediaFact.Unknown return try { - val bits = ParsableBitArray(configuration) - val objectType = bits.readBits(5) + val objectType = ParsableBitArray(configuration).readBits(5) if (objectType !in 1..4) return MediaFact.Unknown - val frequencyIndex = bits.readBits(4) - if (frequencyIndex == 15) { - if (bits.readBits(24) <= 0) return MediaFact.Unknown - } else if (frequencyIndex > 12) return MediaFact.Unknown - val channels = bits.readBits(4) - val expectedCount = when (channels) { 1 -> 1; 2 -> 2; 3 -> 3; 4 -> 4; 5 -> 5; 6 -> 6; 7 -> 8; else -> null } - ?: return MediaFact.Unknown - if (channelCount != null && channelCount != expectedCount) return MediaFact.Unknown - MediaFact.Known(when (channels) { + val config = AacUtil.parseAudioSpecificConfig(configuration) + if (config.sampleRateHz <= 0 || (channelCount != null && config.channelCount != channelCount)) + return MediaFact.Unknown + MediaFact.Known(when (config.channelCount) { 1 -> AudioChannelLayout.Mono 2 -> AudioChannelLayout.Stereo 6 -> AudioChannelLayout.FivePointOne else -> AudioChannelLayout.Other }) - } catch (_: RuntimeException) { MediaFact.Unknown } + } catch (_: Exception) { MediaFact.Unknown } } } diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/IsoContainerMetadata.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/IsoContainerMetadata.kt index d380b20e..d47f6e00 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/IsoContainerMetadata.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/IsoContainerMetadata.kt @@ -3,6 +3,20 @@ package com.mux.video.upload.internal.standardization import java.io.File import java.io.RandomAccessFile +internal data class IsoMetadataBox(val type: String, val payload: Long, val end: Long) + +internal object IsoSampleEntries { + val avc = setOf("avc1", "avc3", "dva1", "dvav") + val hevc = setOf("hvc1", "hev1", "dvh1", "dvhe") + val dolby = setOf("dvh1", "dvhe", "dva1", "dvav") + val visual = avc + hevc + setOf("mp4v", "s263", "vp08", "vp09", "av01", "apco", "apcs", "apcn", "apch", "ap4h", "ap4x") + fun codec(entry: String): VideoCodec? = when (entry) { + in avc -> VideoCodec.H264 + in hevc -> VideoCodec.Hevc + else -> null + } +} + internal data class IsoTrackMetadata( val handler: String?, val trackId: MediaFact, @@ -10,24 +24,36 @@ internal data class IsoTrackMetadata( val hasDolbyVisionConfiguration: Boolean, val rotationDegrees: MediaFact, val hasSimpleSampleGeometry: Boolean, + val pixelAspectRatio: MediaFact = MediaFact.Unknown, + val sampleDimensions: MediaFact = MediaFact.Unknown, + val timeline: MediaFact = MediaFact.Unknown, ) { val hasSimpleOrientation: Boolean get() = rotationDegrees != MediaFact.Unknown + fun matches(kind: TrackKind): Boolean = (handler == "vide" && kind == TrackKind.Video) || + (handler == "soun" && kind == TrackKind.Audio) } -/** Seeks past sample payloads/tables; reads only bounded track and sample-description metadata. */ +/** Seeks past media payloads; reads bounded track, geometry and presentation timing metadata. */ internal object IsoContainerMetadataReader { private const val MAX_BOXES = 4096 + private const val MAX_TOP_LEVEL_BOXES = 131072 private const val MAX_ENTRIES = 16 - private data class Box(val type: String, val payload: Long, val end: Long) fun read(file: File): MediaFact> = try { RandomAccessFile(file, "r").use { input -> val reader = Reader(input) - val moov = reader.boxes(0, input.length()).singleOrNull { it.type == "moov" } - ?: return MediaFact.Unknown - val tracks = reader.boxes(moov.payload, moov.end).filter { it.type == "trak" } + var movie: IsoMetadataBox? = null + var fragmented = false + for (box in reader.boxes(0, input.length(), topLevel = true)) { + if (box.type == "moov") { require(movie == null); movie = box } + if (box.type == "moof") fragmented = true + } + val moov = movie ?: return MediaFact.Unknown + val children = reader.boxes(moov.payload, moov.end).toList() + val movieHeader = children.singleOrNull { it.type == "mvhd" } + val tracks = children.filter { it.type == "trak" } if (tracks.size !in 1..64) return MediaFact.Unknown - val metadata = tracks.map { reader.track(it) } + val metadata = tracks.map { reader.track(it, movieHeader, fragmented) } val ids = metadata.mapNotNull { it.trackId.valueOrNull } require(ids.distinct().size == ids.size) MediaFact.Known(metadata) @@ -36,9 +62,12 @@ internal object IsoContainerMetadataReader { private class Reader(val input: RandomAccessFile) { var boxCount = 0 - fun boxes(start: Long, end: Long, allowQuickTimeTerminator: Boolean = false): List { + val timelines = IsoTimelineMetadataReader(input) + + fun boxes(start: Long, end: Long, allowQuickTimeTerminator: Boolean = false, + topLevel: Boolean = false): Sequence = sequence { require(start <= end && end <= input.length()) - val result = mutableListOf() + var topCount = 0 var position = start while (position < end) { if (allowQuickTimeTerminator && end - position == 4L) { @@ -46,24 +75,25 @@ internal object IsoContainerMetadataReader { require(input.readInt() == 0) break } - require(end - position >= 8 && ++boxCount <= MAX_BOXES) + require(end - position >= 8) + require(if (topLevel) ++topCount <= MAX_TOP_LEVEL_BOXES else ++boxCount <= MAX_BOXES) input.seek(position) val size32 = input.readInt().toLong() and 0xffffffffL val type = fourCc() val header = if (size32 == 1L) 16 else 8 + require(end - position >= header) val size = when (size32) { 0L -> end - position; 1L -> input.readLong(); else -> size32 } require(size >= header && size <= end - position) - result.add(Box(type, position + header, position + size)) + yield(IsoMetadataBox(type, position + header, position + size)) position += size } - return result } - fun track(track: Box): IsoTrackMetadata { - val children = boxes(track.payload, track.end) + fun track(track: IsoMetadataBox, movieHeader: IsoMetadataBox?, fragmented: Boolean): IsoTrackMetadata { + val children = boxes(track.payload, track.end).toList() val tkhd = children.singleOrNull { it.type == "tkhd" } val mdia = children.single { it.type == "mdia" } - val media = boxes(mdia.payload, mdia.end) + val media = boxes(mdia.payload, mdia.end).toList() val handler = media.singleOrNull { it.type == "hdlr" }?.let { require(it.end - it.payload >= 12) input.seek(it.payload + 8) @@ -83,51 +113,78 @@ internal object IsoContainerMetadataReader { return IsoTrackMetadata(handler, trackId, emptyList(), false, orientation, false) val minf = media.single { it.type == "minf" } val stbl = boxes(minf.payload, minf.end).single { it.type == "stbl" } - val stsd = boxes(stbl.payload, stbl.end).single { it.type == "stsd" } + val tables = boxes(stbl.payload, stbl.end).toList() + val stsd = tables.single { it.type == "stsd" } require(stsd.end - stsd.payload >= 8) input.seek(stsd.payload + 4) val count = input.readInt() require(count in 1..MAX_ENTRIES) - val entries = boxes(stsd.payload + 8, stsd.end) + val entries = boxes(stsd.payload + 8, stsd.end).toList() require(entries.size == count) var dolby = false var simpleGeometry = true + var aspect: MediaFact = MediaFact.Unknown + var sampleSize: MediaFact = MediaFact.Unknown if (handler == "vide") { for (entry in entries) { - if (entry.type in listOf("dvhe", "dvh1", "dvav", "dva1")) dolby = true - if (entry.type in listOf("hvc1", "hev1", "avc1", "avc3", "dvhe", "dvh1", "dvav", "dva1")) { + if (entry.type in IsoSampleEntries.dolby) dolby = true + if (entry.type in IsoSampleEntries.visual) { require(entry.end - entry.payload >= 78) - val extensions = boxes(entry.payload + 78, entry.end, allowQuickTimeTerminator = true) - if (extensions.any { it.type in listOf("dvcC", "dvvC", "dvwC") }) dolby = true + input.seek(entry.payload + 24) + val width = input.readUnsignedShort() + val height = input.readUnsignedShort() + if (width > 0 && height > 0) sampleSize = MediaFact.Known(Dimensions(width, height)) + val extensions = boxes(entry.payload + 78, entry.end, allowQuickTimeTerminator = true).toList() + if (extensions.any { it.type in setOf("dvcC", "dvvC", "dvwC") }) dolby = true if (extensions.any { it.type == "clap" }) simpleGeometry = false val aspects = extensions.filter { it.type == "pasp" } if (aspects.size > 1) simpleGeometry = false - for (aspect in aspects) { - require(aspect.end - aspect.payload == 8L) - input.seek(aspect.payload) + aspect = aspects.singleOrNull()?.let { + require(it.end - it.payload == 8L) + input.seek(it.payload) val horizontal = input.readInt().toLong() and 0xffffffffL val vertical = input.readInt().toLong() and 0xffffffffL - // Retain uncertain anamorphic/clean-aperture geometry for a later richer reader. - if (horizontal == 0L || horizontal != vertical) simpleGeometry = false - } - } + if (horizontal == 0L || vertical == 0L) { simpleGeometry = false; MediaFact.Unknown } + else MediaFact.Known(horizontal.toDouble() / vertical) + } ?: MediaFact.Unknown + // A declared presentation size can establish aspect even for codecs without an SPS reader. + if (aspect == MediaFact.Unknown && aspects.isEmpty()) aspect = trackAspect(tkhd, sampleSize) + } else simpleGeometry = false } } - return IsoTrackMetadata(handler, trackId, entries.map { it.type }, dolby, orientation, simpleGeometry) + val timeline = if (fragmented || entries.size != 1) MediaFact.Unknown else try { + val edits = children.filter { it.type == "edts" } + require(edits.size <= 1) + val editChildren = edits.singleOrNull()?.let { boxes(it.payload, it.end).toList() } + timelines.read(movieHeader, media.singleOrNull { it.type == "mdhd" }, editChildren, tables) + } catch (_: Exception) { MediaFact.Unknown } + return IsoTrackMetadata(handler, trackId, entries.map { it.type }, dolby, orientation, simpleGeometry, + if (entries.size == 1 && simpleGeometry) aspect else MediaFact.Unknown, + if (entries.size == 1) sampleSize else MediaFact.Unknown, timeline) + } + + private fun trackAspect(tkhd: IsoMetadataBox?, sampleSize: MediaFact): MediaFact { + val size = sampleSize.valueOrNull ?: return MediaFact.Unknown + val header = tkhd ?: return MediaFact.Unknown + input.seek(header.payload) + val offset = when (input.readUnsignedByte()) { 0 -> 76; 1 -> 88; else -> return MediaFact.Unknown } + if (header.end - header.payload < offset + 8) return MediaFact.Unknown + input.seek(header.payload + offset) + val width = (input.readInt().toLong() and 0xffffffffL) / 65536.0 + val height = (input.readInt().toLong() and 0xffffffffL) / 65536.0 + return if (width > 0 && height > 0) positive(width / height * size.height / size.width) else MediaFact.Unknown } - private fun orientation(box: Box): MediaFact { + private fun orientation(box: IsoMetadataBox): MediaFact { input.seek(box.payload) val version = input.readUnsignedByte() val offset = when (version) { 0 -> 40; 1 -> 52; else -> return MediaFact.Unknown } require(box.end - box.payload >= offset + 36) input.seek(box.payload + offset) val matrix = IntArray(9) { input.readInt() } - // Unit rotations plus translation only. Scale, shear, reflection and perspective stay unproven. if (matrix[2] != 0 || matrix[5] != 0 || matrix[8] != 0x40000000) return MediaFact.Unknown - val axes = listOf(matrix[0], matrix[1], matrix[3], matrix[4]) - // Normalize the container transform to the shared clockwise convention. - return when (axes) { + // Same normalized transform convention as Swift. Android's reported angle is negated separately. + return when (listOf(matrix[0], matrix[1], matrix[3], matrix[4])) { listOf(65536, 0, 0, 65536) -> MediaFact.Known(0) listOf(0, 65536, -65536, 0) -> MediaFact.Known(270) listOf(-65536, 0, 0, -65536) -> MediaFact.Known(180) diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/IsoTimelineMetadata.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/IsoTimelineMetadata.kt new file mode 100644 index 00000000..eea758c5 --- /dev/null +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/IsoTimelineMetadata.kt @@ -0,0 +1,170 @@ +package com.mux.video.upload.internal.standardization + +import java.io.RandomAccessFile +import kotlin.math.max +import kotlin.math.min + +/** Presentation bounds derived from timing tables and a supported unit-rate edit, not sample payloads. */ +internal data class IsoTrackTimeline( + val startSeconds: Double, + val endSeconds: Double, + val movieDurationSeconds: Double, + val movieTickSeconds: Double, + val hasEdits: Boolean, +) + +/** Bounded run-length timing metadata. Cadence, GOP and compressed-sample evidence are separate. */ +internal class IsoTimelineMetadataReader(private val input: RandomAccessFile) { + private data class Timebase(val scale: Long, val duration: Long) + private data class Run(val count: Long, val value: Long) + private data class Edit(val duration: Long, val mediaTime: Long) + private var remainingRows = MAX_TIMING_ROWS + + fun read(movieHeader: IsoMetadataBox?, mediaHeader: IsoMetadataBox?, + editChildren: List?, tables: List): MediaFact = try { + val movie = timebase(movieHeader ?: return MediaFact.Unknown) + val media = timebase(mediaHeader ?: return MediaFact.Unknown) + val decode = runs(tables.single { it.type == "stts" }, composition = false) + val totalSamples = decode.fold(0L) { sum, run -> boundedAdd(sum, run.count) } + require(totalSamples > 0 && totalSamples == sampleCount(tables)) + val compositionBoxes = tables.filter { it.type == "ctts" } + require(compositionBoxes.size <= 1) + val composition = compositionBoxes.singleOrNull()?.let { runs(it, composition = true) } + ?: listOf(Run(totalSamples, 0)) + require(composition.fold(0L) { sum, run -> boundedAdd(sum, run.count) } == totalSamples) + val decodeDuration = decode.fold(0L) { sum, run -> boundedAdd(sum, boundedMultiply(run.count, run.value)) } + require(decodeDuration == media.duration && decodeDuration <= MAX_EXACT_TIME) + + val edits = editChildren?.let { + require(it.size == 1 && it.single().type == "elst") + edits(it.single()) + } + val mediaEdit = edits?.last() + val emptyDuration = if (edits?.size == 2) edits.first().duration.toDouble() / movie.scale else 0.0 + val lower = mediaEdit?.mediaTime?.toDouble() ?: Double.NEGATIVE_INFINITY + val upper = mediaEdit?.let { lower + it.duration.toDouble() * media.scale / movie.scale } + ?: Double.POSITIVE_INFINITY + require(mediaEdit == null || (lower >= 0 && upper > lower && upper <= MAX_EXACT_TIME)) + + var decodeIndex = 0 + var compositionIndex = 0 + var decodeRemaining = decode.first().count + var compositionRemaining = composition.first().count + var dts = 0L + var first = Double.POSITIVE_INFINITY + var end = Double.NEGATIVE_INFINITY + while (decodeIndex < decode.size && compositionIndex < composition.size) { + val count = min(decodeRemaining, compositionRemaining) + val duration = boundedMultiply(count, decode[decodeIndex].value) + val startPts = boundedAdd(dts, composition[compositionIndex].value) + val endPts = boundedAdd(startPts, duration) + require(startPts in -MAX_EXACT_TIME..MAX_EXACT_TIME && endPts in -MAX_EXACT_TIME..MAX_EXACT_TIME) + // Each intersection has one duration and composition offset; its presentation intervals are contiguous. + val clippedStart = max(startPts.toDouble(), lower) + val clippedEnd = min(endPts.toDouble(), upper) + if (clippedEnd > clippedStart) { + val shift = if (mediaEdit != null) emptyDuration - lower / media.scale else 0.0 + first = min(first, clippedStart / media.scale + shift) + end = max(end, clippedEnd / media.scale + shift) + } + dts = boundedAdd(dts, duration) + decodeRemaining -= count + compositionRemaining -= count + if (decodeRemaining == 0L && ++decodeIndex < decode.size) decodeRemaining = decode[decodeIndex].count + if (compositionRemaining == 0L && ++compositionIndex < composition.size) + compositionRemaining = composition[compositionIndex].count + } + require(first.isFinite() && end.isFinite() && end > first) + MediaFact.Known(IsoTrackTimeline(first, end, movie.duration.toDouble() / movie.scale, + 1.0 / movie.scale, edits != null)) + } catch (_: Exception) { MediaFact.Unknown } + + private fun timebase(box: IsoMetadataBox): Timebase { + require(box.end - box.payload >= 4) + input.seek(box.payload) + val version = input.readInt() + require(version == 0 || version == 0x01000000) + val offset = if (version == 0) 12 else 20 + val length = if (version == 0) 8 else 12 + require(box.end - box.payload >= offset + length) + input.seek(box.payload + offset) + val scale = uint() + val duration = if (version == 0) uint() else input.readLong() + require(scale > 0 && duration in 1..MAX_EXACT_TIME && (version != 0 || duration != 0xffffffffL)) + return Timebase(scale, duration) + } + + private fun runs(box: IsoMetadataBox, composition: Boolean): List { + require(box.end - box.payload >= 8) + input.seek(box.payload) + val version = input.readInt() + require(version == 0 || (composition && version == 0x01000000)) + val count = uint() + require(count in 1..remainingRows.toLong() && box.end - box.payload == 8 + count * 8) + remainingRows -= count.toInt() + return List(count.toInt()) { + val samples = uint() + val value = if (version == 0x01000000) input.readInt().toLong() else uint() + require(samples > 0 && (composition || value > 0)) + Run(samples, value) + } + } + + private fun sampleCount(tables: List): Long { + val box = tables.single { it.type == "stsz" || it.type == "stz2" } + require(box.end - box.payload >= 12) + input.seek(box.payload) + require(input.readInt() == 0) + val size = uint() + val count = uint() + require(count > 0) + if (box.type == "stsz") { + require(box.end - box.payload == if (size > 0) 12L else 12 + count * 4) + } else { + val fieldSize = size and 255 + require(size == fieldSize && fieldSize in setOf(4L, 8L, 16L)) + require(box.end - box.payload == 12 + (count * fieldSize + 7) / 8) + } + return count + } + + private fun edits(box: IsoMetadataBox): List { + require(box.end - box.payload >= 8) + input.seek(box.payload) + val version = input.readInt() + require(version == 0 || version == 0x01000000) + val count = uint() + val rowBytes = if (version == 0) 12 else 20 + require(count in 1..2 && box.end - box.payload == 8 + count * rowBytes) + val edits = List(count.toInt()) { + val duration = if (version == 0) uint() else input.readLong() + val mediaTime = if (version == 0) input.readInt().toLong() else input.readLong() + require(duration in 1..MAX_EXACT_TIME && mediaTime in -1..MAX_EXACT_TIME) + require(input.readInt() == 65536) // Unit media rate only; dwell/repeated/rate-changing edits stay unknown. + Edit(duration, mediaTime) + } + require(edits.last().mediaTime >= 0 && (edits.size == 1 || edits.first().mediaTime == -1L)) + return edits + } + + // Explicit bounds also retain exact integer-to-double conversion on API 23. + private fun boundedAdd(a: Long, b: Long): Long { + require(a in -MAX_EXACT_TIME..MAX_EXACT_TIME && b in -MAX_EXACT_TIME..MAX_EXACT_TIME) + val sum = a + b + require(sum in -MAX_EXACT_TIME..MAX_EXACT_TIME) + return sum + } + + private fun boundedMultiply(a: Long, b: Long): Long { + require(a in 0..MAX_EXACT_TIME && b in 0..MAX_EXACT_TIME) + require(b == 0L || a <= MAX_EXACT_TIME / b) + return a * b + } + + private fun uint(): Long = input.readInt().toLong() and 0xffffffffL + + companion object { + private const val MAX_TIMING_ROWS = 65536 + private const val MAX_EXACT_TIME = 1L shl 53 + } +} diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/MediaMetadataInspector.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/MediaMetadataInspector.kt index a7081d7b..20ff2ba9 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/MediaMetadataInspector.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/MediaMetadataInspector.kt @@ -8,6 +8,7 @@ import java.io.File import java.io.IOException import java.nio.ByteBuffer import kotlin.math.roundToInt +import kotlin.math.abs internal enum class ContainerKind { IsoBaseMedia, Matroska, Other } internal enum class TrackKind { Video, Audio, Other, Unknown } @@ -70,26 +71,25 @@ internal class MediaMetadataInspector { return try { val container = container(file) extractor.setDataSource(file.absolutePath) - if (extractor.trackCount !in 1..MAX_TRACKS) - return MetadataInspectionResult.Failure(MetadataFailure.LimitExceeded) + trackCountFailure(extractor.trackCount)?.let { return MetadataInspectionResult.Failure(it) } val iso = if (container == MediaFact.Known(ContainerKind.IsoBaseMedia)) IsoContainerMetadataReader.read(file) else MediaFact.Unknown val tracks = (0 until extractor.trackCount).map { index -> val format = extractor.getTrackFormat(index) - val orientation = if (format.string(MediaFormat.KEY_MIME)?.startsWith("video/") == true) - iso.valueOrNull?.filter { it.handler == "vide" }?.singleOrNull()?.rotationDegrees else null - val track = MediaTrackMetadataReader.read(index, format, containerRotation = orientation) val containers = iso.valueOrNull + val orientation = if (format.string(MediaFormat.KEY_MIME)?.startsWith("video/") == true) + containers?.filter { it.handler == "vide" }?.singleOrNull() else null + val size = orientation?.sampleDimensions?.valueOrNull + val aspect = if (size != null && size.width == format.int(MediaFormat.KEY_WIDTH) && + size.height == format.int(MediaFormat.KEY_HEIGHT) && orientation.hasSimpleSampleGeometry) + orientation.pixelAspectRatio else MediaFact.Unknown + val track = MediaTrackMetadataReader.read(index, format, + containerRotation = orientation?.rotationDegrees, containerAspect = aspect) val id = track.sourceTrackId.valueOrNull val match = if (id != null) containers?.indexOfFirst { it.trackId.valueOrNull == id } else null - val ordered = containers?.getOrNull(index)?.handler - val sameKind = (ordered == "vide" && track.kind == TrackKind.Video) || - (ordered == "soun" && track.kind == TrackKind.Audio) val containerIndex = when { - match != null && match >= 0 && - ((containers!![match].handler == "vide" && track.kind == TrackKind.Video) || - (containers[match].handler == "soun" && track.kind == TrackKind.Audio)) -> MediaFact.Known(match) - id == null && containers?.size == extractor.trackCount && sameKind -> MediaFact.Known(index) + match != null && match >= 0 && containers!![match].matches(track.kind) -> MediaFact.Known(match) + id == null && containers?.size == extractor.trackCount && containers[index].matches(track.kind) -> MediaFact.Known(index) else -> MediaFact.Unknown } track.copy(containerIndex = containerIndex) @@ -106,11 +106,11 @@ internal class MediaMetadataInspector { } } - private fun container(file: File): MediaFact { + internal fun container(file: File): MediaFact { val header = ByteArray(12) val count = file.inputStream().use { it.read(header) } return when { - count >= 8 && String(header, 4, 4, Charsets.US_ASCII) in listOf("ftyp", "moov", "mdat", "free", "wide") -> + count >= 8 && String(header, 4, 4, Charsets.US_ASCII) in setOf("ftyp", "moov", "mdat", "free", "wide", "skip", "uuid", "pnot") -> MediaFact.Known(ContainerKind.IsoBaseMedia) count >= 4 && header.take(4) == listOf(0x1a.toByte(), 0x45.toByte(), 0xdf.toByte(), 0xa3.toByte()) -> MediaFact.Known(ContainerKind.Matroska) @@ -119,7 +119,14 @@ internal class MediaMetadataInspector { } } - companion object { private const val MAX_TRACKS = 64 } + companion object { + private const val MAX_TRACKS = 64 + internal fun trackCountFailure(count: Int): MetadataFailure? = when { + count <= 0 -> MetadataFailure.Malformed + count > MAX_TRACKS -> MetadataFailure.LimitExceeded + else -> null + } + } } /** Reconcile physical container tracks with extractor views, including Dolby Vision base-layer views. */ @@ -128,10 +135,14 @@ internal object MediaContainerMetadataReader { tracks: List): MediaFacts { val dolby = iso.valueOrNull?.any { it.hasDolbyVisionConfiguration } == true || tracks.any { it.video?.dynamicRange == MediaFact.Known(DynamicRange.DolbyVision) } - if (container != MediaFact.Known(ContainerKind.IsoBaseMedia)) return MediaTrackMetadataReader.facts(tracks) - val containers = iso.valueOrNull - ?: return MediaFacts(dynamicRange = if (dolby) MediaFact.Known(DynamicRange.DolbyVision) else MediaFact.Unknown) - if (containers.any { it.handler == null }) return MediaFacts() + fun fallback(): MediaFacts { + val facts = MediaTrackMetadataReader.facts(tracks, + audioOrderKnown = container != MediaFact.Known(ContainerKind.IsoBaseMedia)) + return if (dolby) facts.copy(dynamicRange = MediaFact.Known(DynamicRange.DolbyVision)) else facts + } + if (container != MediaFact.Known(ContainerKind.IsoBaseMedia)) return fallback() + val containers = iso.valueOrNull ?: return fallback() + if (containers.any { it.handler == null }) return fallback().copy(videoTrackCount = MediaFact.Unknown, audioTracks = MediaFact.Unknown) val videoIndices = containers.indices.filter { containers[it].handler == "vide" } val audioIndices = containers.indices.filter { containers[it].handler == "soun" } val audioViews = tracks.filter { it.kind == TrackKind.Audio }.groupBy { it.containerIndex.valueOrNull } @@ -151,28 +162,45 @@ internal object MediaContainerMetadataReader { if (views.any { it.containerIndex.valueOrNull != index }) return base val selected = if (views.size == 1) views.single() else { if (!dolby || views.size != 2 || views.count { it.mimeType.valueOrNull == "video/dolby-vision" } != 1) return base - val codec = when (description.sampleEntries.single()) { - "hvc1", "hev1", "dvh1", "dvhe" -> VideoCodec.Hevc - "avc1", "avc3", "dva1", "dvav" -> VideoCodec.H264 - else -> return base - } + val codec = IsoSampleEntries.codec(description.sampleEntries.single()) ?: return base views.singleOrNull { it.video?.codec == MediaFact.Known(codec) } ?: return base } val video = selected.video ?: return base - return base.copy( - videoCodec = video.codec, videoProfile = video.profile, encodedDimensions = video.encodedDimensions, + val facts = video.toFacts(base).copy( displayDimensions = if (description.hasSimpleOrientation && description.hasSimpleSampleGeometry) video.displayDimensions else MediaFact.Unknown, rotationDegrees = if (description.hasSimpleOrientation) video.rotationDegrees else MediaFact.Unknown, - pixelFormat = video.configuration.pixelFormat, dynamicRange = if (dolby) MediaFact.Known(DynamicRange.DolbyVision) else video.dynamicRange, ) + return timelineFacts(facts, containers, index, audioIndices.firstOrNull(), audioViews) + } + + private fun timelineFacts(base: MediaFacts, containers: List, videoIndex: Int, + audioIndex: Int?, audioViews: Map>): MediaFacts { + val media = containers.filter { it.handler == "vide" || it.handler == "soun" } + val timelines = media.map { it.timeline.valueOrNull } + val duration = if (timelines.all { it != null }) { + val known = timelines.filterNotNull() + val declaration = known.map { it.movieDurationSeconds }.distinct().singleOrNull() + val end = known.maxOf { it.endSeconds } + if (declaration != null && abs(declaration - end) <= known.maxOf { it.movieTickSeconds } + 1e-6) + MediaFact.Known(declaration) else MediaFact.Unknown + } else MediaFact.Unknown + val video = containers[videoIndex].timeline.valueOrNull + val audio = audioIndex?.let { containers[it].timeline.valueOrNull } + val audioMetadata = audioIndex?.let { audioViews[it]?.singleOrNull()?.audio } + // An edit already accounts for codec priming. Do not apply extractor delay/padding twice. + val unaccountedPriming = audio != null && !audio.hasEdits && audioMetadata != null && + ((audioMetadata.encoderDelaySamples.valueOrNull ?: 0) > 0 || (audioMetadata.encoderPaddingSamples.valueOrNull ?: 0) > 0) + val offset = if (video != null && audio != null && !unaccountedPriming) + MediaFact.Known(audio.startSeconds - video.startSeconds) else MediaFact.Unknown + return base.copy(durationSeconds = duration, audioVideoStartOffsetSeconds = offset) } } internal object MediaTrackMetadataReader { fun read(index: Int, format: MediaFormat, apiLevel: Int = Build.VERSION.SDK_INT, - containerRotation: MediaFact? = null): TrackMetadata { + containerRotation: MediaFact? = null, containerAspect: MediaFact = MediaFact.Unknown): TrackMetadata { val mime = format.string(MediaFormat.KEY_MIME) val kind = when { mime == null -> TrackKind.Unknown @@ -185,28 +213,30 @@ internal object MediaTrackMetadataReader { positive(format.long(MediaFormat.KEY_DURATION)?.toDouble()?.div(1_000_000)), positive(format.number(MediaFormat.KEY_FRAME_RATE)), positiveCode(format.int(MediaFormat.KEY_BIT_RATE)), - video = if (kind == TrackKind.Video) video(format, mime, apiLevel, containerRotation) else null, + video = if (kind == TrackKind.Video) video(format, mime, apiLevel, containerRotation, containerAspect) else null, audio = if (kind == TrackKind.Audio) audio(format, mime) else null, ) } - fun facts(tracks: List): MediaFacts { - if (tracks.any { it.kind == TrackKind.Unknown }) return MediaFacts() + fun facts(tracks: List, audioOrderKnown: Boolean = true): MediaFacts { + val dolby = tracks.any { it.video?.dynamicRange == MediaFact.Known(DynamicRange.DolbyVision) } + if (tracks.any { it.kind == TrackKind.Unknown }) return MediaFacts( + dynamicRange = if (dolby) MediaFact.Known(DynamicRange.DolbyVision) else MediaFact.Unknown) val videos = tracks.filter { it.kind == TrackKind.Video } val audio = tracks.mapNotNull { it.audio }.map { AudioTrack(it.format) } - val base = MediaFacts(videoTrackCount = MediaFact.Known(videos.size), audioTracks = MediaFact.Known(audio)) - val video = videos.singleOrNull()?.video ?: return base - return base.copy( - videoCodec = video.codec, videoProfile = video.profile, - encodedDimensions = video.encodedDimensions, displayDimensions = video.displayDimensions, - rotationDegrees = video.rotationDegrees, pixelFormat = video.configuration.pixelFormat, - dynamicRange = video.dynamicRange, - // Duration, nominal rate and declared bitrate require later sample/timeline proof. + // Dolby/base-layer views may alias one physical track; unproven identities cannot prove two tracks. + val ids = videos.map { it.sourceTrackId.valueOrNull } + val ambiguousViews = dolby && videos.size > 1 && (ids.any { it == null } || ids.distinct().size != ids.size) + val base = MediaFacts( + videoTrackCount = if (ambiguousViews) MediaFact.Unknown else MediaFact.Known(videos.size), + audioTracks = if (audioOrderKnown || audio.size <= 1) MediaFact.Known(audio) else MediaFact.Unknown, + dynamicRange = if (dolby) MediaFact.Known(DynamicRange.DolbyVision) else MediaFact.Unknown, ) + return videos.singleOrNull()?.video?.toFacts(base) ?: base } private fun video(format: MediaFormat, mime: String?, apiLevel: Int, - containerRotation: MediaFact?): VideoTrackMetadata { + containerRotation: MediaFact?, containerAspect: MediaFact): VideoTrackMetadata { val codec = when (mime) { "video/avc" -> MediaFact.Known(VideoCodec.H264) "video/hevc" -> MediaFact.Known(VideoCodec.Hevc) @@ -219,7 +249,10 @@ internal object MediaTrackMetadataReader { val platformRotation = format.int(MediaFormat.KEY_ROTATION)?.let { if (it % 90 == 0) MediaFact.Known((it % 360 + 360) % 360) else MediaFact.Unknown } ?: MediaFact.Unknown - val rotation = containerRotation ?: platformRotation + // Match the shared transform convention while retaining Android's raw angle as an observation. + val normalizedPlatform = platformRotation.valueOrNull?.let { MediaFact.Known((360 - it) % 360) } ?: MediaFact.Unknown + val rotation = if (format.containsKey(MediaFormat.KEY_ROTATION) && platformRotation == MediaFact.Unknown) + MediaFact.Unknown else reconcile(containerRotation ?: MediaFact.Unknown, normalizedPlatform) val platformColor = if (apiLevel >= 24) ColorMetadata( positiveCode(format.int(MediaFormat.KEY_COLOR_STANDARD)), positiveCode(format.int(MediaFormat.KEY_COLOR_TRANSFER)), @@ -230,7 +263,7 @@ internal object MediaTrackMetadataReader { else dynamicRange(transfer) return VideoTrackMetadata(codec, reconcile(platformProfile(codec.valueOrNull, format.int(MediaFormat.KEY_PROFILE)), config.profile), - encoded, display(format, encoded, rotation, config.pixelAspectRatio), rotation, platformRotation, + encoded, display(format, encoded, rotation, reconcileAspect(config.pixelAspectRatio, containerAspect)), rotation, platformRotation, config, platformColor, range) } @@ -263,7 +296,7 @@ internal object MediaTrackMetadataReader { val platformAspect = if (sarWidth != null && sarHeight != null && sarWidth > 0 && sarHeight > 0) MediaFact.Known(sarWidth.toDouble() / sarHeight) else MediaFact.Unknown if ((sarWidth != null || sarHeight != null) && platformAspect == MediaFact.Unknown) return MediaFact.Unknown - val aspect = reconcile(platformAspect, headerAspect).valueOrNull ?: return MediaFact.Unknown + val aspect = reconcileAspect(platformAspect, headerAspect).valueOrNull ?: return MediaFact.Unknown val width = visible.width * aspect if (!width.isFinite() || width < 1 || width > Int.MAX_VALUE) return MediaFact.Unknown val display = Dimensions(width.roundToInt(), visible.height) @@ -314,3 +347,17 @@ private fun MediaFormat.bytes(key: String): ByteArray? { private fun nonNegative(value: Int?): MediaFact = if (value != null && value >= 0) MediaFact.Known(value) else MediaFact.Unknown + +private fun VideoTrackMetadata.toFacts(base: MediaFacts): MediaFacts = base.copy( + videoCodec = codec, videoProfile = profile, encodedDimensions = encodedDimensions, + displayDimensions = displayDimensions, rotationDegrees = rotationDegrees, + pixelFormat = configuration.pixelFormat, dynamicRange = dynamicRange, +) + +private fun reconcileAspect(a: MediaFact, b: MediaFact): MediaFact { + val first = a.valueOrNull + val second = b.valueOrNull + // SPS ratios are floats; container ratios are exact integer ratios converted to doubles. + return if (first != null && second != null && abs(first - second) <= 1e-6 * maxOf(first, second)) a + else reconcile(a, b) +} diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/IsoContainerMetadataTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/IsoContainerMetadataTests.kt index 98d9d306..a0d585a3 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/IsoContainerMetadataTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/IsoContainerMetadataTests.kt @@ -51,11 +51,11 @@ class IsoContainerMetadataTests { assertTrue(read(box("moov", track(matrix = matrix))).valueOrNull!![0].hasSimpleOrientation) } - @Test fun cleanApertureAndAnamorphicContainerGeometryStayUnproven() { + @Test fun cleanApertureStaysUnknownAndExplicitPixelAspectIsRetained() { assertFalse(read(box("moov", track(extension = box("clap", ByteArray(32))))) .valueOrNull!![0].hasSimpleSampleGeometry) - assertFalse(read(box("moov", track(extension = box("pasp", int(2) + int(1))))) - .valueOrNull!![0].hasSimpleSampleGeometry) + assertEquals(MediaFact.Known(2.0), read(box("moov", track(extension = box("pasp", int(2) + int(1))))) + .valueOrNull!![0].pixelAspectRatio) assertTrue(read(box("moov", track(extension = box("pasp", int(1) + int(1))))) .valueOrNull!![0].hasSimpleSampleGeometry) } @@ -81,4 +81,36 @@ class IsoContainerMetadataTests { assertEquals(MediaFact.Unknown, read(box("moov", byteArrayOf()))) assertEquals(MediaFact.Unknown, read(box("moov", track()) + box("moov", track()))) } + + @Test fun everyUnitMatrixHasTheSharedNormalizedDegreeValue() { + for ((axes, degrees) in listOf( + listOf(65536, 0, 0, 65536) to 0, + listOf(0, 65536, -65536, 0) to 270, + listOf(-65536, 0, 0, -65536) to 180, + listOf(0, -65536, 65536, 0) to 90)) { + val matrix = intArrayOf(axes[0], axes[1], 0, axes[2], axes[3], 0, 1920 * 65536, 0, 0x40000000) + assertEquals(MediaFact.Known(degrees), read(box("moov", track(matrix = matrix))).valueOrNull!![0].rotationDegrees) + } + } + + @Test fun fragmentHeadersDoNotConsumeMovieMetadataBudget() { + val fragment = box("moof", byteArrayOf()) + box("mdat", byteArrayOf()) + val many = ByteArray(7200 * fragment.size) + repeat(7200) { fragment.copyInto(many, it * fragment.size) } + val movie = box("moov", track()) + assertNotEquals(MediaFact.Unknown, read(movie + many)) + assertNotEquals(MediaFact.Unknown, read(many + movie)) + assertEquals(MediaFact.Unknown, read(movie + many + movie)) + assertEquals(MediaFact.Unknown, read(movie + byteArrayOf(1))) + } + + @Test fun quickTimeLeadingBoxesAreRecognizedAsIso() { + for (type in listOf("skip", "uuid", "pnot")) { + val file = File.createTempFile("iso-sniff", ".mov") + try { + file.writeBytes(box(type, ByteArray(16)) + box("moov", track())) + assertEquals(MediaFact.Known(ContainerKind.IsoBaseMedia), MediaMetadataInspector().container(file)) + } finally { file.delete() } + } + } } diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/IsoTimelineMetadataTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/IsoTimelineMetadataTests.kt new file mode 100644 index 00000000..9a450497 --- /dev/null +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/IsoTimelineMetadataTests.kt @@ -0,0 +1,164 @@ +package com.mux.video.upload.internal.standardization + +import android.media.MediaFormat +import com.mux.exoplayeradapter.AbsRobolectricTest +import org.junit.Assert.* +import org.junit.Test +import org.robolectric.annotation.Config +import java.io.File +import java.nio.ByteBuffer + +@Config(sdk = [23]) +class IsoTimelineMetadataTests : AbsRobolectricTest() { + private fun int(value: Int) = ByteBuffer.allocate(4).putInt(value).array() + private fun long(value: Long) = ByteBuffer.allocate(8).putLong(value).array() + private fun box(type: String, payload: ByteArray) = int(payload.size + 8) + type.toByteArray() + payload + private fun header(type: String, scale: Int, duration: Long, version: Int = 0): ByteArray = box(type, + int(version shl 24) + ByteArray(if (version == 0) 8 else 16) + int(scale) + + (if (version == 0) int(duration.toInt()) else long(duration))) + private fun runs(type: String, rows: List>, version: Int = 0) = box(type, + int(version shl 24) + int(rows.size) + rows.fold(byteArrayOf()) { bytes, row -> bytes + int(row.first) + int(row.second) }) + private fun edit(rows: List>, version: Int = 0) = box("edts", box("elst", + int(version shl 24) + int(rows.size) + rows.fold(byteArrayOf()) { bytes, (duration, time, rate) -> + bytes + (if (version == 0) int(duration.toInt()) + int(time.toInt()) else long(duration) + long(time)) + int(rate) + })) + + private fun track(handler: String = "vide", id: Int = 1, scale: Int = 1000, duration: Long = 3000, + timing: ByteArray = runs("stts", listOf(3 to 1000)), samples: Int = 3, + composition: ByteArray = byteArrayOf(), edits: ByteArray = byteArrayOf(), version: Int = 0, + entry: String = if (handler == "vide") "vp09" else "mp4a", aspect: ByteArray = byteArrayOf()): ByteArray { + val tkhd = ByteArray(84) + int(id).copyInto(tkhd, 12) + val matrix = intArrayOf(65536, 0, 0, 0, 65536, 0, 0, 0, 0x40000000) + matrix.forEachIndexed { i, n -> int(n).copyInto(tkhd, 40 + i * 4) } + int(1920 * 65536).copyInto(tkhd, 76); int(1080 * 65536).copyInto(tkhd, 80) + val description = ByteArray(if (handler == "vide") 78 else 28) + if (handler == "vide") { + ByteBuffer.wrap(description).putShort(24, 1920.toShort()).putShort(26, 1080.toShort()) + } + val stsd = box("stsd", int(0) + int(1) + box(entry, description + aspect)) + val stsz = box("stsz", int(0) + int(1) + int(samples)) + return box("trak", box("tkhd", tkhd) + edits + box("mdia", header("mdhd", scale, duration, version) + + box("hdlr", ByteArray(8) + handler.toByteArray()) + box("minf", box("stbl", stsd + timing + composition + stsz)))) + } + + private fun read(vararg tracks: ByteArray, duration: Long = 3000, scale: Int = 1000, + version: Int = 0, suffix: ByteArray = byteArrayOf()): List { + val file = File.createTempFile("iso-timeline", ".mp4") + return try { + file.writeBytes(box("moov", header("mvhd", scale, duration, version) + tracks.fold(byteArrayOf(), ByteArray::plus)) + suffix) + IsoContainerMetadataReader.read(file).valueOrNull!! + } finally { file.delete() } + } + + private fun facts(iso: List, priming: Int = 0): MediaFacts { + val views = iso.mapIndexed { index, track -> + val format = if (track.handler == "vide") MediaFormat.createVideoFormat("video/x-vnd.on2.vp9", 1920, 1080) + .apply { setInteger(MediaFormat.KEY_ROTATION, 0); setInteger("color-transfer", 3) } + else MediaFormat.createAudioFormat("audio/mp4a-latm", 48000, 2).apply { + setByteBuffer("csd-0", ByteBuffer.wrap(byteArrayOf(0x11, 0x90.toByte()))) + setInteger(MediaFormat.KEY_ENCODER_DELAY, priming) + } + MediaTrackMetadataReader.read(index, format, 24, track.rotationDegrees, track.pixelAspectRatio) + .copy(containerIndex = MediaFact.Known(index)) + } + return MediaContainerMetadataReader.facts(MediaFact.Known(ContainerKind.IsoBaseMedia), MediaFact.Known(iso), views) + } + + @Test fun establishesUneditedDurationAndAvStartWithoutSamplePayloads() { + val iso = read(track(), track("soun", 2)) + val f = facts(iso) + assertEquals(MediaFact.Known(3.0), f.durationSeconds) + assertEquals(MediaFact.Known(0.0), f.audioVideoStartOffsetSeconds) + assertEquals(MediaFact.Unknown, f.timestamps) + assertEquals(MediaFact.Unknown, f.cadence) + assertEquals(MediaFact.Unknown, f.frameRate) + assertEquals(MediaFact.Unknown, f.editList) + } + + @Test fun supportsVersionOneHeadersAndSignedCompositionOffsets() { + val video = track(version = 1, composition = runs("ctts", listOf(1 to 100, 1 to -100, 1 to 0), 1)) + val audio = track("soun", 2, version = 1) + val f = facts(read(video, audio, version = 1)) + assertEquals(MediaFact.Known(3.0), f.durationSeconds) + assertEquals(-0.1, f.audioVideoStartOffsetSeconds.valueOrNull!!, 1e-9) + } + + @Test fun timingRunBoundariesAreMergedWithoutExpandingSamples() { + val t = track(timing = runs("stts", listOf(1 to 1000, 2 to 1000)), + composition = runs("ctts", listOf(2 to 0, 1 to 0))) + assertEquals(MediaFact.Known(3.0), facts(read(t)).durationSeconds) + } + + @Test fun unitRateTrimMapsCompositionOffsetsIntoPresentationTime() { + val edits = edit(listOf(Triple(2000L, 1000L, 65536))) + val iso = read(track(edits = edits), track("soun", 2, edits = edits), duration = 2000) + val f = facts(iso, priming = 1024) + assertEquals(MediaFact.Known(2.0), f.durationSeconds) + assertEquals(MediaFact.Known(0.0), f.audioVideoStartOffsetSeconds) + } + + @Test fun leadingEmptyEditCreatesKnownAudioDelay() { + val delayed = edit(listOf(Triple(100L, -1L, 65536), Triple(2900L, 0L, 65536)), version = 1) + val f = facts(read(track(), track("soun", 2, edits = delayed))) + assertEquals(MediaFact.Known(3.0), f.durationSeconds) + assertEquals(0.1, f.audioVideoStartOffsetSeconds.valueOrNull!!, 1e-9) + } + + @Test fun editAndMediaTimescalesAreConvertedIndependently() { + val media = track(scale = 48000, duration = 144000, timing = runs("stts", listOf(3 to 48000)), + edits = edit(listOf(Triple(2000L, 48000L, 65536)))) + assertEquals(MediaFact.Known(2.0), facts(read(media, duration = 2000)).durationSeconds) + } + + @Test fun unsupportedEditsDoNotDiscardCodecAndGeometry() { + for (edits in listOf( + edit(listOf(Triple(3000L, 0L, 0))), + edit(listOf(Triple(1500L, 0L, 65536), Triple(1500L, 1000L, 65536))), + edit(listOf(Triple(3000L, -1L, 65536))))) { + val f = facts(read(track(edits = edits))) + assertEquals(MediaFact.Known(VideoCodec.Other), f.videoCodec) + assertEquals(MediaFact.Known(Dimensions(1920, 1080)), f.displayDimensions) + assertEquals(MediaFact.Unknown, f.durationSeconds) + } + } + + @Test fun malformedOrConflictingTablesKeepOnlyTimelineUnknown() { + for (t in listOf( + track(samples = 4), track(duration = 2000), + track(composition = runs("ctts", listOf(2 to 0))), + track(timing = runs("stts", listOf(3 to 0))), + track(timing = box("stts", int(0) + int(65537))), + track(timing = box("stts", int(0) + int(1) + int(3))), + track(composition = runs("ctts", listOf(3 to 0), 2)), + track(duration = 0), + track(timing = runs("stts", listOf(-1 to -1)), samples = -1))) { + val f = facts(read(t)) + assertEquals(MediaFact.Known(VideoCodec.Other), f.videoCodec) + assertEquals(MediaFact.Unknown, f.durationSeconds) + } + } + + @Test fun movieDurationConflictAndFragmentedTimelinesStayUnknown() { + assertEquals(MediaFact.Unknown, facts(read(track(), duration = 2000)).durationSeconds) + val iso = read(track(), suffix = box("moof", byteArrayOf()) + box("mdat", byteArrayOf())) + assertEquals(MediaFact.Unknown, iso.single().timeline) + assertEquals(MediaFact.Known(VideoCodec.Other), facts(iso).videoCodec) + } + + @Test fun primingWithoutAnEditCannotProveAudioStartOffset() { + val f = facts(read(track(), track("soun", 2)), priming = 1024) + assertEquals(MediaFact.Unknown, f.audioVideoStartOffsetSeconds) + assertEquals(MediaFact.Known(3.0), f.durationSeconds) + } + + @Test fun otherVisualCodecsUseDeclaredGeometryWithoutCodecHeaderGuessing() { + for (entry in listOf("vp09", "mp4v", "apch")) { + val iso = read(track(entry = entry)) + assertEquals(MediaFact.Known(1.0), iso.single().pixelAspectRatio) + assertEquals(MediaFact.Known(Dimensions(1920, 1080)), facts(iso).displayDimensions) + } + val anamorphic = read(track(aspect = box("pasp", int(2) + int(1)))) + assertEquals(MediaFact.Known(Dimensions(3840, 1080)), facts(anamorphic).displayDimensions) + } +} diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/MediaMetadataReaderTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/MediaMetadataReaderTests.kt index de84bd3f..77b92be3 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/MediaMetadataReaderTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/MediaMetadataReaderTests.kt @@ -92,7 +92,7 @@ class MediaMetadataReaderTests : AbsRobolectricTest() { @Test fun normalizesSignedRotationAndSwapsPortraitAxes() { val format = video().apply { setInteger(MediaFormat.KEY_ROTATION, -90) } val f = facts(format) - assertEquals(MediaFact.Known(270), f.rotationDegrees) + assertEquals(MediaFact.Known(90), f.rotationDegrees) assertEquals(MediaFact.Known(Dimensions(1080, 1920)), f.displayDimensions) } @@ -242,4 +242,101 @@ class MediaMetadataReaderTests : AbsRobolectricTest() { MediaTrackMetadataReader.read(1, MediaFormat()))) assertEquals(MediaFacts(), f) } + + @Test fun normalizesAndroidQuarterTurnsBeforeReconcilingContainerRotation() { + for ((raw, normalized) in listOf(0 to 0, 90 to 270, 180 to 180, 270 to 90)) { + val format = video().apply { setInteger(MediaFormat.KEY_ROTATION, raw) } + val track = MediaTrackMetadataReader.read(0, format, 23, MediaFact.Known(normalized)) + assertEquals(MediaFact.Known(raw), track.video!!.reportedPlatformRotationDegrees) + assertEquals(MediaFact.Known(normalized), track.video.rotationDegrees) + val conflict = MediaTrackMetadataReader.read(0, format, 23, MediaFact.Known((normalized + 180) % 360)) + assertEquals(MediaFact.Unknown, conflict.video!!.rotationDegrees) + assertEquals(MediaFact.Unknown, conflict.video.displayDimensions) + } + } + + @Test fun absentPlatformRotationCanUseContainerButInvalidRotationCannot() { + val format = MediaFormat.createVideoFormat("video/avc", 1920, 1080) + assertEquals(MediaFact.Known(270), MediaTrackMetadataReader.read(0, format, 23, MediaFact.Known(270)).video!!.rotationDegrees) + format.setInteger(MediaFormat.KEY_ROTATION, 45) + assertEquals(MediaFact.Unknown, MediaTrackMetadataReader.read(0, format, 23, MediaFact.Known(270)).video!!.rotationDegrees) + } + + @Test fun failedIsoReadRetainsResolutionButDoesNotGuessMultipleAudioOrder() { + val tracks = listOf(MediaTrackMetadataReader.read(0, video()), + MediaTrackMetadataReader.read(1, audio(2, "1190")), MediaTrackMetadataReader.read(2, audio(1, "1188"))) + val f = MediaContainerMetadataReader.facts(MediaFact.Known(ContainerKind.IsoBaseMedia), MediaFact.Unknown, tracks) + assertEquals(MediaFact.Known(VideoCodec.H264), f.videoCodec) + assertEquals(MediaFact.Known(Dimensions(1920, 1080)), f.displayDimensions) + assertEquals(MediaFact.Unknown, f.audioTracks) + assertEquals(MediaFact.Unknown, f.durationSeconds) + } + + @Test fun dolbyEvidenceSurvivesNonIsoMissingIsoUnknownHandlerAndUnknownMime() { + val dolby = MediaTrackMetadataReader.read(0, video("video/dolby-vision", null)) + val base = MediaTrackMetadataReader.read(1, video("video/hevc", hevcHlg)) + for (container in listOf(ContainerKind.Matroska, ContainerKind.Other, ContainerKind.IsoBaseMedia)) { + val f = MediaContainerMetadataReader.facts(MediaFact.Known(container), MediaFact.Unknown, listOf(dolby, base)) + assertEquals(MediaFact.Known(DynamicRange.DolbyVision), f.dynamicRange) + assertEquals(MediaFact.Unknown, f.videoTrackCount) + assertEquals(StandardInputAction.Fallback(FallbackReason.UnsupportedHdr(DynamicRange.DolbyVision)), StandardInputPlanner().plan(f).action) + } + val unknownHandler = MediaFact.Known(listOf(isoTrack("vide", "dvh1", 1, true).copy(handler = null))) + assertEquals(MediaFact.Known(DynamicRange.DolbyVision), MediaContainerMetadataReader.facts( + MediaFact.Known(ContainerKind.IsoBaseMedia), unknownHandler, listOf(dolby)).dynamicRange) + assertEquals(MediaFact.Known(DynamicRange.DolbyVision), MediaTrackMetadataReader.facts( + listOf(dolby, MediaTrackMetadataReader.read(1, MediaFormat()))).dynamicRange) + } + + @Test fun distinctDolbyTrackIdentitiesStillProveUnsafeMultipleTracks() { + val tracks = listOf(MediaTrackMetadataReader.read(0, video("video/dolby-vision", null)).copy(sourceTrackId = MediaFact.Known(1)), + MediaTrackMetadataReader.read(1, video("video/hevc", hevcHlg)).copy(sourceTrackId = MediaFact.Known(2))) + assertEquals(MediaFact.Known(2), MediaTrackMetadataReader.facts(tracks).videoTrackCount) + } + + @Test fun otherCodecUsesProvenAspectWithoutAssumingSquarePixels() { + val format = video("video/x-vnd.on2.vp9", null).apply { setInteger("color-transfer", 3) } + assertEquals(MediaFact.Unknown, facts(format, 24).displayDimensions) + val track = MediaTrackMetadataReader.read(0, format, 24, containerAspect = MediaFact.Known(1.0)) + val f = MediaTrackMetadataReader.facts(listOf(track)).copy(frameRate = MediaFact.Known(30.0)) + assertEquals(MediaFact.Known(Dimensions(1920, 1080)), f.displayDimensions) + assertTrue(StandardInputPlanner().plan(f, capabilities = fullCapabilities()).action is StandardInputAction.Convert) + format.setInteger("sar-width", 2); format.setInteger("sar-height", 1) + assertEquals(MediaFact.Unknown, MediaTrackMetadataReader.read(0, format, 24, + containerAspect = MediaFact.Known(1.0)).video!!.displayDimensions) + } + + @Test fun predictedHevcReferenceSetsRetainMain10HlgEvidence() { + val nal = hex("420101022000000300900000030000030078a003c0801107cad965654a4c1af7780b509120904000000300400000078200") + val f = facts(video("video/hevc", byteArrayOf(0, 0, 0, 1) + nal)) + assertEquals(MediaFact.Known(VideoProfile.HevcMain10), f.videoProfile) + assertEquals(MediaFact.Known(PixelFormat(10, ChromaSubsampling.Yuv420)), f.pixelFormat) + assertEquals(MediaFact.Known(DynamicRange.Hlg), f.dynamicRange) + assertTrue(StandardInputPlanner().plan(f).action is StandardInputAction.UploadOriginal) + } + + @Test fun supportsMixedAnnexBPrefixesAndKeepsAacExtensionsUnknown() { + val mixed = avc + byteArrayOf(0, 0, 1, 0x68, 1) + assertEquals(CodecMetadataReader.video(VideoCodec.H264, listOf(avc)), CodecMetadataReader.video(VideoCodec.H264, listOf(mixed))) + for (config in listOf("2b9208", "eb9208", "1180", "11f0")) + assertEquals(MediaFact.Unknown, CodecMetadataReader.aacLayout(hex(config), 2)) + } + + @Test fun emptyAndExcessiveTrackCountsHaveDifferentFailures() { + assertEquals(MetadataFailure.Malformed, MediaMetadataInspector.trackCountFailure(0)) + assertEquals(MetadataFailure.LimitExceeded, MediaMetadataInspector.trackCountFailure(65)) + assertNull(MediaMetadataInspector.trackCountFailure(1)) + assertNull(MediaMetadataInspector.trackCountFailure(64)) + } + + @Test fun predictedHevcChainsUseTheDerivedReferenceCountAfterZeroPocIsRemoved() { + val csd = hex("00000001420101022000000300900000030000030078a003c0801107cad965654a4c08bdec780b509120904000000300400000078200") + assertEquals(MediaFact.Known(DynamicRange.Hlg), facts(video("video/hevc", csd)).dynamicRange) + assertEquals(MediaFact.Known(PixelFormat(10, ChromaSubsampling.Yuv420)), facts(video("video/hevc", csd)).pixelFormat) + } + + @Test fun predictedHevcSetsCannotGrowPastTheReferenceBound() { + val csd = hex("00000001420101022000000300900000030000030078a003c0801107cad965654a4c1847ffffffff047fffef016a12241208000003000800000300f040") + assertEquals(CodecMetadata(), CodecMetadataReader.video(VideoCodec.Hevc, listOf(csd))) + } } From 5b174d8f793e7f0d8df86262d91d70b725b34ae8 Mon Sep 17 00:00:00 2001 From: Eugene Morozov Date: Fri, 2 Oct 2026 14:08:50 -0700 Subject: [PATCH 04/49] Reuse Media3 timeline parsing and resolve metadata review findings --- .../standardization/IsoContainerMetadata.kt | 12 +- .../standardization/IsoTimelineMetadata.kt | 219 ++++++++---------- .../internal/standardization/MediaFacts.kt | 5 + .../standardization/MediaMetadataInspector.kt | 3 +- .../IsoContainerMetadataTests.kt | 33 ++- .../IsoTimelineMetadataTests.kt | 48 +++- .../MediaMetadataReaderTests.kt | 22 +- 7 files changed, 205 insertions(+), 137 deletions(-) diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/IsoContainerMetadata.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/IsoContainerMetadata.kt index d47f6e00..982b9a56 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/IsoContainerMetadata.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/IsoContainerMetadata.kt @@ -2,6 +2,7 @@ package com.mux.video.upload.internal.standardization import java.io.File import java.io.RandomAccessFile +import java.nio.ByteBuffer internal data class IsoMetadataBox(val type: String, val payload: Long, val end: Long) @@ -63,6 +64,7 @@ internal object IsoContainerMetadataReader { private class Reader(val input: RandomAccessFile) { var boxCount = 0 val timelines = IsoTimelineMetadataReader(input) + val boxHeader = ByteArray(16) fun boxes(start: Long, end: Long, allowQuickTimeTerminator: Boolean = false, topLevel: Boolean = false): Sequence = sequence { @@ -78,11 +80,13 @@ internal object IsoContainerMetadataReader { require(end - position >= 8) require(if (topLevel) ++topCount <= MAX_TOP_LEVEL_BOXES else ++boxCount <= MAX_BOXES) input.seek(position) - val size32 = input.readInt().toLong() and 0xffffffffL - val type = fourCc() + input.readFully(boxHeader, 0, minOf(16L, end - position).toInt()) + val bytes = ByteBuffer.wrap(boxHeader) + val size32 = bytes.int.toLong() and 0xffffffffL + val type = String(boxHeader, 4, 4, Charsets.US_ASCII) val header = if (size32 == 1L) 16 else 8 require(end - position >= header) - val size = when (size32) { 0L -> end - position; 1L -> input.readLong(); else -> size32 } + val size = when (size32) { 0L -> end - position; 1L -> bytes.getLong(8); else -> size32 } require(size >= header && size <= end - position) yield(IsoMetadataBox(type, position + header, position + size)) position += size @@ -148,7 +152,7 @@ internal object IsoContainerMetadataReader { else MediaFact.Known(horizontal.toDouble() / vertical) } ?: MediaFact.Unknown // A declared presentation size can establish aspect even for codecs without an SPS reader. - if (aspect == MediaFact.Unknown && aspects.isEmpty()) aspect = trackAspect(tkhd, sampleSize) + if (aspects.isEmpty() && IsoSampleEntries.codec(entry.type) == null) aspect = trackAspect(tkhd, sampleSize) } else simpleGeometry = false } } diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/IsoTimelineMetadata.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/IsoTimelineMetadata.kt index eea758c5..42bcede7 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/IsoTimelineMetadata.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/IsoTimelineMetadata.kt @@ -1,10 +1,16 @@ package com.mux.video.upload.internal.standardization +import androidx.media3.common.C +import androidx.media3.common.Format +import androidx.media3.common.util.ParsableByteArray +import androidx.media3.container.Mp4Box +import androidx.media3.extractor.GaplessInfoHolder +import androidx.media3.extractor.mp4.BoxParser +import androidx.media3.extractor.mp4.Track +import com.google.common.primitives.ImmutableLongArray import java.io.RandomAccessFile -import kotlin.math.max -import kotlin.math.min +import java.nio.ByteBuffer -/** Presentation bounds derived from timing tables and a supported unit-rate edit, not sample payloads. */ internal data class IsoTrackTimeline( val startSeconds: Double, val endSeconds: Double, @@ -13,158 +19,135 @@ internal data class IsoTrackTimeline( val hasEdits: Boolean, ) -/** Bounded run-length timing metadata. Cadence, GOP and compressed-sample evidence are separate. */ +/** Bounds metadata allocations, then delegates composition timestamps and edits to Media3. */ internal class IsoTimelineMetadataReader(private val input: RandomAccessFile) { private data class Timebase(val scale: Long, val duration: Long) - private data class Run(val count: Long, val value: Long) private data class Edit(val duration: Long, val mediaTime: Long) - private var remainingRows = MAX_TIMING_ROWS fun read(movieHeader: IsoMetadataBox?, mediaHeader: IsoMetadataBox?, editChildren: List?, tables: List): MediaFact = try { val movie = timebase(movieHeader ?: return MediaFact.Unknown) val media = timebase(mediaHeader ?: return MediaFact.Unknown) - val decode = runs(tables.single { it.type == "stts" }, composition = false) - val totalSamples = decode.fold(0L) { sum, run -> boundedAdd(sum, run.count) } - require(totalSamples > 0 && totalSamples == sampleCount(tables)) - val compositionBoxes = tables.filter { it.type == "ctts" } - require(compositionBoxes.size <= 1) - val composition = compositionBoxes.singleOrNull()?.let { runs(it, composition = true) } - ?: listOf(Run(totalSamples, 0)) - require(composition.fold(0L) { sum, run -> boundedAdd(sum, run.count) } == totalSamples) - val decodeDuration = decode.fold(0L) { sum, run -> boundedAdd(sum, boundedMultiply(run.count, run.value)) } - require(decodeDuration == media.duration && decodeDuration <= MAX_EXACT_TIME) - + val selected = tables.filter { it.type in TABLE_TYPES } + require(selected.map { it.type }.distinct().size == selected.size) + require(selected.sumOf { it.end - it.payload + 8 } <= MAX_TABLE_BYTES) + val data = selected.associate { it.type to bytes(it) } + val samples = sampleCount(data) + require(samples in 1..MAX_SAMPLES) + require(validateRuns(data.getValue("stts"), samples, composition = false) == media.duration) + data["ctts"]?.let { validateRuns(it, samples, composition = true) } val edits = editChildren?.let { require(it.size == 1 && it.single().type == "elst") edits(it.single()) } - val mediaEdit = edits?.last() - val emptyDuration = if (edits?.size == 2) edits.first().duration.toDouble() / movie.scale else 0.0 - val lower = mediaEdit?.mediaTime?.toDouble() ?: Double.NEGATIVE_INFINITY - val upper = mediaEdit?.let { lower + it.duration.toDouble() * media.scale / movie.scale } - ?: Double.POSITIVE_INFINITY - require(mediaEdit == null || (lower >= 0 && upper > lower && upper <= MAX_EXACT_TIME)) - - var decodeIndex = 0 - var compositionIndex = 0 - var decodeRemaining = decode.first().count - var compositionRemaining = composition.first().count - var dts = 0L - var first = Double.POSITIVE_INFINITY - var end = Double.NEGATIVE_INFINITY - while (decodeIndex < decode.size && compositionIndex < composition.size) { - val count = min(decodeRemaining, compositionRemaining) - val duration = boundedMultiply(count, decode[decodeIndex].value) - val startPts = boundedAdd(dts, composition[compositionIndex].value) - val endPts = boundedAdd(startPts, duration) - require(startPts in -MAX_EXACT_TIME..MAX_EXACT_TIME && endPts in -MAX_EXACT_TIME..MAX_EXACT_TIME) - // Each intersection has one duration and composition offset; its presentation intervals are contiguous. - val clippedStart = max(startPts.toDouble(), lower) - val clippedEnd = min(endPts.toDouble(), upper) - if (clippedEnd > clippedStart) { - val shift = if (mediaEdit != null) emptyDuration - lower / media.scale else 0.0 - first = min(first, clippedStart / media.scale + shift) - end = max(end, clippedEnd / media.scale + shift) - } - dts = boundedAdd(dts, duration) - decodeRemaining -= count - compositionRemaining -= count - if (decodeRemaining == 0L && ++decodeIndex < decode.size) decodeRemaining = decode[decodeIndex].count - if (compositionRemaining == 0L && ++compositionIndex < composition.size) - compositionRemaining = composition[compositionIndex].count + val stbl = Mp4Box.ContainerBox(Mp4Box.TYPE_stbl, 0) + data.forEach { (type, bytes) -> + val code = ByteBuffer.wrap(type.toByteArray(Charsets.US_ASCII)).int + stbl.add(Mp4Box.LeafBox(code, ParsableByteArray(bytes))) } - require(first.isFinite() && end.isFinite() && end > first) - MediaFact.Known(IsoTrackTimeline(first, end, movie.duration.toDouble() / movie.scale, + // Request generic timing, so Media3 applies edits to timestamps rather than translating + // short audio trims into codec-specific gapless delay/padding. No codec parsing is needed. + val track = Track.Builder().setType(C.TRACK_TYPE_UNKNOWN).setTimescale(media.scale) + .setMovieTimescale(movie.scale).setFormat(Format.Builder().setMaxNumReorderSamples(samples).build()) + .setEditListDurations(edits?.let { ImmutableLongArray.copyOf(it.map(Edit::duration)) }) + .setEditListMediaTimes(edits?.let { ImmutableLongArray.copyOf(it.map(Edit::mediaTime)) }).build() + val parsed = BoxParser.parseStbl(track, stbl, GaplessInfoHolder(), false) + require(parsed.timestampsUs.isNotEmpty() && parsed.durationUs > 0) + if (edits == null) require(parsed.sampleCount == samples) + val emptyDuration = if (edits?.size == 2) edits.first().duration.toDouble() / movie.scale else 0.0 + val first = parsed.timestampsUs.min().toDouble() / 1_000_000 + // Edit preroll is needed for decoding but is outside the effective presentation interval. + val start = if (edits == null) first else maxOf(first, emptyDuration) + val end = parsed.durationUs.toDouble() / 1_000_000 + require(start.isFinite() && end > start) + MediaFact.Known(IsoTrackTimeline(start, end, movie.duration.toDouble() / movie.scale, 1.0 / movie.scale, edits != null)) } catch (_: Exception) { MediaFact.Unknown } - private fun timebase(box: IsoMetadataBox): Timebase { - require(box.end - box.payload >= 4) + /** A single buffered read per box; Media3 expects the ordinary eight-byte box header. */ + private fun bytes(box: IsoMetadataBox): ByteArray { + val length = box.end - box.payload + require(length in 0..MAX_TABLE_BYTES) + val result = ByteArray(length.toInt() + 8) + ByteBuffer.wrap(result).putInt(result.size).put(box.type.toByteArray(Charsets.US_ASCII)) input.seek(box.payload) - val version = input.readInt() + input.readFully(result, 8, length.toInt()) + return result + } + + private fun timebase(box: IsoMetadataBox): Timebase { + val data = ByteBuffer.wrap(bytes(box)) + val version = data.getInt(8) require(version == 0 || version == 0x01000000) - val offset = if (version == 0) 12 else 20 - val length = if (version == 0) 8 else 12 - require(box.end - box.payload >= offset + length) - input.seek(box.payload + offset) - val scale = uint() - val duration = if (version == 0) uint() else input.readLong() + data.position(if (version == 0) 20 else 28) + val scale = data.int.toLong() and 0xffffffffL + val duration = if (version == 0) data.int.toLong() and 0xffffffffL else data.long require(scale > 0 && duration in 1..MAX_EXACT_TIME && (version != 0 || duration != 0xffffffffL)) return Timebase(scale, duration) } - private fun runs(box: IsoMetadataBox, composition: Boolean): List { - require(box.end - box.payload >= 8) - input.seek(box.payload) - val version = input.readInt() + /** Reject inconsistent or allocation-driving counts before the reusable parser sees them. */ + private fun validateRuns(bytes: ByteArray, samples: Int, composition: Boolean): Long { + val data = ByteBuffer.wrap(bytes) + data.position(8) + val version = data.int require(version == 0 || (composition && version == 0x01000000)) - val count = uint() - require(count in 1..remainingRows.toLong() && box.end - box.payload == 8 + count * 8) - remainingRows -= count.toInt() - return List(count.toInt()) { - val samples = uint() - val value = if (version == 0x01000000) input.readInt().toLong() else uint() - require(samples > 0 && (composition || value > 0)) - Run(samples, value) + val rows = data.int + require(rows in 1..samples && data.remaining().toLong() == rows.toLong() * 8) + var count = 0L + var duration = 0L + repeat(rows) { + val run = data.int + val value = data.int // Media3 also accepts signed composition offsets in version zero. + require(run > 0 && (composition || value > 0)) + count += run + require(count <= samples) + if (!composition) duration += run.toLong() * value } + require(count == samples.toLong() && duration <= MAX_EXACT_TIME) + return duration } - private fun sampleCount(tables: List): Long { - val box = tables.single { it.type == "stsz" || it.type == "stz2" } - require(box.end - box.payload >= 12) - input.seek(box.payload) - require(input.readInt() == 0) - val size = uint() - val count = uint() - require(count > 0) - if (box.type == "stsz") { - require(box.end - box.payload == if (size > 0) 12L else 12 + count * 4) - } else { - val fieldSize = size and 255 - require(size == fieldSize && fieldSize in setOf(4L, 8L, 16L)) - require(box.end - box.payload == 12 + (count * fieldSize + 7) / 8) + private fun sampleCount(data: Map): Int { + require(data.keys.count { it == "stsz" || it == "stz2" } == 1) + val compact = "stz2" in data + val bytes = ByteBuffer.wrap(data.getValue(if (compact) "stz2" else "stsz")) + bytes.position(8) + require(bytes.int == 0) + val size = bytes.int.toLong() and 0xffffffffL + val count = bytes.int + require(count in 1..MAX_SAMPLES) + val remaining = if (!compact) { if (size > 0) 0L else count.toLong() * 4 } else { + require(size in setOf(4L, 8L, 16L)) + (count * size + 7) / 8 } + require(bytes.remaining().toLong() == remaining) return count } private fun edits(box: IsoMetadataBox): List { - require(box.end - box.payload >= 8) - input.seek(box.payload) - val version = input.readInt() + val data = ByteBuffer.wrap(bytes(box)) + data.position(8) + val version = data.int require(version == 0 || version == 0x01000000) - val count = uint() - val rowBytes = if (version == 0) 12 else 20 - require(count in 1..2 && box.end - box.payload == 8 + count * rowBytes) - val edits = List(count.toInt()) { - val duration = if (version == 0) uint() else input.readLong() - val mediaTime = if (version == 0) input.readInt().toLong() else input.readLong() - require(duration in 1..MAX_EXACT_TIME && mediaTime in -1..MAX_EXACT_TIME) - require(input.readInt() == 65536) // Unit media rate only; dwell/repeated/rate-changing edits stay unknown. - Edit(duration, mediaTime) + val count = data.int + require(count in 1..2 && data.remaining() == count * if (version == 0) 12 else 20) + val edits = List(count) { + val duration = if (version == 0) data.int.toLong() and 0xffffffffL else data.long + val time = if (version == 0) data.int.toLong() else data.long + require(duration in 1..MAX_EXACT_TIME && time in -1..MAX_EXACT_TIME && data.int == 65536) + Edit(duration, time) } - require(edits.last().mediaTime >= 0 && (edits.size == 1 || edits.first().mediaTime == -1L)) + require(edits.last().mediaTime >= 0 && (count == 1 || edits.first().mediaTime == -1L)) return edits } - // Explicit bounds also retain exact integer-to-double conversion on API 23. - private fun boundedAdd(a: Long, b: Long): Long { - require(a in -MAX_EXACT_TIME..MAX_EXACT_TIME && b in -MAX_EXACT_TIME..MAX_EXACT_TIME) - val sum = a + b - require(sum in -MAX_EXACT_TIME..MAX_EXACT_TIME) - return sum - } - - private fun boundedMultiply(a: Long, b: Long): Long { - require(a in 0..MAX_EXACT_TIME && b in 0..MAX_EXACT_TIME) - require(b == 0L || a <= MAX_EXACT_TIME / b) - return a * b - } - - private fun uint(): Long = input.readInt().toLong() and 0xffffffffL - companion object { - private const val MAX_TIMING_ROWS = 65536 + // Per-track limits allow dense VFR/B-frame tables without a shared row budget. + // Media3 materializes per-sample metadata; cap that allocation independently of file size. + private const val MAX_SAMPLES = 500_000 + private const val MAX_TABLE_BYTES = 8L * 1024 * 1024 private const val MAX_EXACT_TIME = 1L shl 53 + private val TABLE_TYPES = setOf("stts", "ctts", "stsz", "stz2", "stsc", "stco", "co64", "stss") } } diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/MediaFacts.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/MediaFacts.kt index 00b187ef..fa32e5b7 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/MediaFacts.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/MediaFacts.kt @@ -64,6 +64,11 @@ internal data class MediaFacts( val videoProfile: MediaFact = MediaFact.Unknown, val encodedDimensions: MediaFact = MediaFact.Unknown, val displayDimensions: MediaFact = MediaFact.Unknown, + /** + * Shared Swift transform convention, opposite to Android/Media3 for quarter turns. + * tkhd [0, 1, -1, 0] / Android KEY_ROTATION = 90 becomes 270. + * Convert a known value back to Android with (360 - degrees) % 360. + */ val rotationDegrees: MediaFact = MediaFact.Unknown, val videoTrackCount: MediaFact = MediaFact.Unknown, /** Measured nominal rate for policy; it does not imply constant presentation intervals. */ diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/MediaMetadataInspector.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/MediaMetadataInspector.kt index 20ff2ba9..b8448e4d 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/MediaMetadataInspector.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/MediaMetadataInspector.kt @@ -250,7 +250,8 @@ internal object MediaTrackMetadataReader { if (it % 90 == 0) MediaFact.Known((it % 360 + 360) % 360) else MediaFact.Unknown } ?: MediaFact.Unknown // Match the shared transform convention while retaining Android's raw angle as an observation. - val normalizedPlatform = platformRotation.valueOrNull?.let { MediaFact.Known((360 - it) % 360) } ?: MediaFact.Unknown + val normalizedPlatform = platformRotation.valueOrNull?.let { MediaFact.Known((360 - it) % 360) } + ?: if (!format.containsKey(MediaFormat.KEY_ROTATION) && containerRotation == null) MediaFact.Known(0) else MediaFact.Unknown val rotation = if (format.containsKey(MediaFormat.KEY_ROTATION) && platformRotation == MediaFact.Unknown) MediaFact.Unknown else reconcile(containerRotation ?: MediaFact.Unknown, normalizedPlatform) val platformColor = if (apiLevel >= 24) ColorMetadata( diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/IsoContainerMetadataTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/IsoContainerMetadataTests.kt index a0d585a3..1266e05e 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/IsoContainerMetadataTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/IsoContainerMetadataTests.kt @@ -1,19 +1,30 @@ package com.mux.video.upload.internal.standardization +import android.media.MediaFormat +import com.mux.exoplayeradapter.AbsRobolectricTest +import org.robolectric.annotation.Config import org.junit.Assert.* import org.junit.Test import java.io.File import java.nio.ByteBuffer -class IsoContainerMetadataTests { +@Config(sdk = [23]) +class IsoContainerMetadataTests : AbsRobolectricTest() { private fun int(value: Int) = ByteBuffer.allocate(4).putInt(value).array() private fun box(type: String, payload: ByteArray) = int(payload.size + 8) + type.toByteArray() + payload private fun track(handler: String = "vide", entry: String = "hvc1", extension: ByteArray = byteArrayOf(), - matrix: IntArray = intArrayOf(65536, 0, 0, 0, 65536, 0, 0, 0, 0x40000000)): ByteArray { + matrix: IntArray = intArrayOf(65536, 0, 0, 0, 65536, 0, 0, 0, 0x40000000), + swappedDisplaySize: Boolean = false): ByteArray { val tkhd = ByteArray(84) for (i in matrix.indices) int(matrix[i]).copyInto(tkhd, 40 + i * 4) val hdlr = ByteArray(8) + handler.toByteArray() - val sampleEntry = box(entry, ByteArray(if (handler == "vide") 78 else 28) + extension) + val description = ByteArray(if (handler == "vide") 78 else 28) + if (handler == "vide") { + ByteBuffer.wrap(description).putShort(24, 1920.toShort()).putShort(26, 1080.toShort()) + int((if (swappedDisplaySize) 1080 else 1920) * 65536).copyInto(tkhd, 76) + int((if (swappedDisplaySize) 1920 else 1080) * 65536).copyInto(tkhd, 80) + } + val sampleEntry = box(entry, description + extension) val stsd = box("stsd", ByteArray(4) + int(1) + sampleEntry) return box("trak", box("tkhd", tkhd) + box("mdia", box("hdlr", hdlr) + box("minf", box("stbl", stsd)))) @@ -60,6 +71,22 @@ class IsoContainerMetadataTests { .valueOrNull!![0].hasSimpleSampleGeometry) } + @Test fun inferredTrackAspectCannotContradictAvcSps() { + val iso = read(box("moov", track(entry = "avc1", swappedDisplaySize = true))).valueOrNull!!.single() + assertEquals(MediaFact.Unknown, iso.pixelAspectRatio) + val sps = "000000016742c028da01e0089f97016a02020280000003008000001e078c1950" + .chunked(2).map { it.toInt(16).toByte() }.toByteArray() + val format = MediaFormat.createVideoFormat("video/avc", 1920, 1080).apply { + setByteBuffer("csd-0", ByteBuffer.wrap(sps)) + } + val metadata = MediaTrackMetadataReader.read(0, format, 23, iso.rotationDegrees, iso.pixelAspectRatio) + assertEquals(MediaFact.Known(Dimensions(1920, 1080)), metadata.video!!.displayDimensions) + val explicit = read(box("moov", track(entry = "avc1", swappedDisplaySize = true, + extension = box("pasp", int(2) + int(1))))).valueOrNull!!.single() + assertEquals(MediaFact.Unknown, MediaTrackMetadataReader.read(0, format, 23, + explicit.rotationDegrees, explicit.pixelAspectRatio).video!!.displayDimensions) + } + @Test fun malformedSizesAndBoxLimitsStayUnknown() { for (data in listOf(byteArrayOf(), int(4096) + "moov".toByteArray(), int(4) + "moov".toByteArray(), int(1) + "moov".toByteArray() + ByteArray(8) { -1 })) { diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/IsoTimelineMetadataTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/IsoTimelineMetadataTests.kt index 9a450497..fd11b2d8 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/IsoTimelineMetadataTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/IsoTimelineMetadataTests.kt @@ -16,8 +16,11 @@ class IsoTimelineMetadataTests : AbsRobolectricTest() { private fun header(type: String, scale: Int, duration: Long, version: Int = 0): ByteArray = box(type, int(version shl 24) + ByteArray(if (version == 0) 8 else 16) + int(scale) + (if (version == 0) int(duration.toInt()) else long(duration))) - private fun runs(type: String, rows: List>, version: Int = 0) = box(type, - int(version shl 24) + int(rows.size) + rows.fold(byteArrayOf()) { bytes, row -> bytes + int(row.first) + int(row.second) }) + private fun runs(type: String, rows: List>, version: Int = 0): ByteArray { + val data = ByteBuffer.allocate(8 + rows.size * 8).putInt(version shl 24).putInt(rows.size) + rows.forEach { data.putInt(it.first).putInt(it.second) } + return box(type, data.array()) + } private fun edit(rows: List>, version: Int = 0) = box("edts", box("elst", int(version shl 24) + int(rows.size) + rows.fold(byteArrayOf()) { bytes, (duration, time, rate) -> bytes + (if (version == 0) int(duration.toInt()) + int(time.toInt()) else long(duration) + long(time)) + int(rate) @@ -38,8 +41,10 @@ class IsoTimelineMetadataTests : AbsRobolectricTest() { } val stsd = box("stsd", int(0) + int(1) + box(entry, description + aspect)) val stsz = box("stsz", int(0) + int(1) + int(samples)) + val chunks = box("stco", int(0) + int(1) + int(0)) + + box("stsc", int(0) + int(1) + int(1) + int(samples) + int(1)) return box("trak", box("tkhd", tkhd) + edits + box("mdia", header("mdhd", scale, duration, version) + - box("hdlr", ByteArray(8) + handler.toByteArray()) + box("minf", box("stbl", stsd + timing + composition + stsz)))) + box("hdlr", ByteArray(8) + handler.toByteArray()) + box("minf", box("stbl", stsd + timing + composition + stsz + chunks)))) } private fun read(vararg tracks: ByteArray, duration: Long = 3000, scale: Int = 1000, @@ -84,12 +89,47 @@ class IsoTimelineMetadataTests : AbsRobolectricTest() { assertEquals(-0.1, f.audioVideoStartOffsetSeconds.valueOrNull!!, 1e-9) } - @Test fun timingRunBoundariesAreMergedWithoutExpandingSamples() { + @Test fun media3HandlesDifferentTimingRunBoundaries() { val t = track(timing = runs("stts", listOf(1 to 1000, 2 to 1000)), composition = runs("ctts", listOf(2 to 0, 1 to 0))) assertEquals(MediaFact.Known(3.0), facts(read(t)).durationSeconds) } + @Test fun denseVfrAndCompositionTablesHaveIndependentPerTrackBounds() { + val count = 72000 + val timing = runs("stts", List(count) { 1 to if (it % 2 == 0) 33 else 34 }) + val composition = runs("ctts", List(count) { 1 to 0 }) + val duration = count / 2 * 67L + val iso = read(track(timing = timing, composition = composition, samples = count, duration = duration), + track("soun", 2, timing = timing, samples = count, duration = duration), duration = duration) + assertEquals(MediaFact.Known(duration / 1000.0), facts(iso).durationSeconds) + assertEquals(MediaFact.Known(0.0), facts(iso).audioVideoStartOffsetSeconds) + } + + @Test fun versionZeroNegativeCompositionOffsetsMatchMedia3() { + val t = track(composition = runs("ctts", listOf(1 to -1000, 2 to 0))) + val f = facts(read(t, track("soun", 2))) + assertEquals(MediaFact.Known(3.0), f.durationSeconds) + assertEquals(MediaFact.Known(1.0), f.audioVideoStartOffsetSeconds) + } + + @Test fun editedTimelineIncludesLateDecodedEarlyPresentationFrames() { + val video = track(samples = 5, duration = 5000, timing = runs("stts", listOf(5 to 1000)), + composition = runs("ctts", listOf(1 to 1000, 1 to 3000, 1 to 1000, 1 to -3000, 1 to -2000), 1), + edits = edit(listOf(Triple(2000L, 0L, 65536)))) + val audio = track("soun", 2, samples = 2, duration = 2000, timing = runs("stts", listOf(2 to 1000))) + val f = facts(read(video, audio, duration = 2000)) + assertEquals(MediaFact.Known(2.0), f.durationSeconds) + assertEquals(MediaFact.Known(0.0), f.audioVideoStartOffsetSeconds) + } + + @Test fun allocationLimitDoesNotDiscardOtherMetadata() { + val t = track(samples = 500001, duration = 500001000L, timing = runs("stts", listOf(500001 to 1000))) + val f = facts(read(t, duration = 500001000L)) + assertEquals(MediaFact.Unknown, f.durationSeconds) + assertEquals(MediaFact.Known(Dimensions(1920, 1080)), f.displayDimensions) + } + @Test fun unitRateTrimMapsCompositionOffsetsIntoPresentationTime() { val edits = edit(listOf(Triple(2000L, 1000L, 65536))) val iso = read(track(edits = edits), track("soun", 2, edits = edits), duration = 2000) diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/MediaMetadataReaderTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/MediaMetadataReaderTests.kt index 77b92be3..1373e93d 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/MediaMetadataReaderTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/MediaMetadataReaderTests.kt @@ -16,9 +16,9 @@ class MediaMetadataReaderTests : AbsRobolectricTest() { private val hevcHlg = hex("00000001420101022000000300900000030000030078a003c0801107cad965654a4c2f016a12241208000003000800000300f040") private val hevcPq = hex("00000001420101022000000300900000030000030078a003c0801107cad965654a4c2f016a12201208000003000800000300f040") - private fun video(mime: String = "video/avc", csd: ByteArray? = avc): MediaFormat = + private fun video(mime: String = "video/avc", csd: ByteArray? = avc, rotation: Int? = 0): MediaFormat = MediaFormat.createVideoFormat(mime, 1920, 1080).apply { - setInteger(MediaFormat.KEY_ROTATION, 0) + rotation?.let { setInteger(MediaFormat.KEY_ROTATION, it) } csd?.let { setByteBuffer("csd-0", ByteBuffer.wrap(it)) } } @@ -96,10 +96,11 @@ class MediaMetadataReaderTests : AbsRobolectricTest() { assertEquals(MediaFact.Known(Dimensions(1080, 1920)), f.displayDimensions) } - @Test fun missingOrNonQuarterRotationDoesNotGuessDisplay() { - val missing = MediaFormat.createVideoFormat("video/avc", 1920, 1080) - assertEquals(MediaFact.Unknown, facts(missing).rotationDegrees) - assertEquals(MediaFact.Unknown, facts(missing).displayDimensions) + @Test fun absentRotationMeansZeroButNonQuarterRotationStaysUnknown() { + val missing = video(rotation = null) + assertEquals(MediaFact.Known(0), facts(missing).rotationDegrees) + assertEquals(MediaFact.Known(Dimensions(1920, 1080)), facts(missing).displayDimensions) + assertEquals(MediaFact.Unknown, MediaTrackMetadataReader.read(0, missing).video!!.reportedPlatformRotationDegrees) val format = video().apply { setInteger(MediaFormat.KEY_ROTATION, 45) } assertEquals(MediaFact.Unknown, facts(format).displayDimensions) } @@ -263,7 +264,7 @@ class MediaMetadataReaderTests : AbsRobolectricTest() { } @Test fun failedIsoReadRetainsResolutionButDoesNotGuessMultipleAudioOrder() { - val tracks = listOf(MediaTrackMetadataReader.read(0, video()), + val tracks = listOf(MediaTrackMetadataReader.read(0, video(rotation = null)), MediaTrackMetadataReader.read(1, audio(2, "1190")), MediaTrackMetadataReader.read(2, audio(1, "1188"))) val f = MediaContainerMetadataReader.facts(MediaFact.Known(ContainerKind.IsoBaseMedia), MediaFact.Unknown, tracks) assertEquals(MediaFact.Known(VideoCodec.H264), f.videoCodec) @@ -272,6 +273,13 @@ class MediaMetadataReaderTests : AbsRobolectricTest() { assertEquals(MediaFact.Unknown, f.durationSeconds) } + @Test fun matroskaWithoutRotationRetainsKnownDisplayDimensions() { + val track = MediaTrackMetadataReader.read(0, video(rotation = null)) + val f = MediaContainerMetadataReader.facts(MediaFact.Known(ContainerKind.Matroska), MediaFact.Unknown, listOf(track)) + assertEquals(MediaFact.Known(0), f.rotationDegrees) + assertEquals(MediaFact.Known(Dimensions(1920, 1080)), f.displayDimensions) + } + @Test fun dolbyEvidenceSurvivesNonIsoMissingIsoUnknownHandlerAndUnknownMime() { val dolby = MediaTrackMetadataReader.read(0, video("video/dolby-vision", null)) val base = MediaTrackMetadataReader.read(1, video("video/hevc", hevcHlg)) From 2b78e850fbe1f3ba4a48f5c7eb068268416c8d8d Mon Sep 17 00:00:00 2001 From: Eugene Morozov Date: Fri, 2 Oct 2026 14:31:37 -0700 Subject: [PATCH 05/49] Keep ambiguous timeline and ISO rotation evidence unknown --- .../standardization/IsoContainerMetadata.kt | 3 +- .../standardization/IsoTimelineMetadata.kt | 13 ++++-- .../standardization/MediaMetadataInspector.kt | 4 +- .../IsoTimelineMetadataTests.kt | 46 ++++++++++++++++--- .../MediaMetadataReaderTests.kt | 18 +++++--- 5 files changed, 65 insertions(+), 19 deletions(-) diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/IsoContainerMetadata.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/IsoContainerMetadata.kt index 982b9a56..0fdcb470 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/IsoContainerMetadata.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/IsoContainerMetadata.kt @@ -59,7 +59,8 @@ internal object IsoContainerMetadataReader { require(ids.distinct().size == ids.size) MediaFact.Known(metadata) } - } catch (_: Exception) { MediaFact.Unknown } + } catch (_: LinkageError) { MediaFact.Unknown } + catch (_: Exception) { MediaFact.Unknown } private class Reader(val input: RandomAccessFile) { var boxCount = 0 diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/IsoTimelineMetadata.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/IsoTimelineMetadata.kt index 42bcede7..9748a158 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/IsoTimelineMetadata.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/IsoTimelineMetadata.kt @@ -35,11 +35,13 @@ internal class IsoTimelineMetadataReader(private val input: RandomAccessFile) { val samples = sampleCount(data) require(samples in 1..MAX_SAMPLES) require(validateRuns(data.getValue("stts"), samples, composition = false) == media.duration) - data["ctts"]?.let { validateRuns(it, samples, composition = true) } val edits = editChildren?.let { require(it.size == 1 && it.single().type == "elst") edits(it.single()) } + // Without an edit, Media3's duration uses the last decoded composition offset, not the + // maximum presentation end. Only a uniform nonnegative offset establishes that end here. + data["ctts"]?.let { validateRuns(it, samples, composition = true, unedited = edits == null) } val stbl = Mp4Box.ContainerBox(Mp4Box.TYPE_stbl, 0) data.forEach { (type, bytes) -> val code = ByteBuffer.wrap(type.toByteArray(Charsets.US_ASCII)).int @@ -59,10 +61,11 @@ internal class IsoTimelineMetadataReader(private val input: RandomAccessFile) { // Edit preroll is needed for decoding but is outside the effective presentation interval. val start = if (edits == null) first else maxOf(first, emptyDuration) val end = parsed.durationUs.toDouble() / 1_000_000 - require(start.isFinite() && end > start) + require(start.isFinite() && start >= 0 && end > start) MediaFact.Known(IsoTrackTimeline(start, end, movie.duration.toDouble() / movie.scale, 1.0 / movie.scale, edits != null)) - } catch (_: Exception) { MediaFact.Unknown } + } catch (_: LinkageError) { MediaFact.Unknown } + catch (_: Exception) { MediaFact.Unknown } /** A single buffered read per box; Media3 expects the ordinary eight-byte box header. */ private fun bytes(box: IsoMetadataBox): ByteArray { @@ -87,19 +90,21 @@ internal class IsoTimelineMetadataReader(private val input: RandomAccessFile) { } /** Reject inconsistent or allocation-driving counts before the reusable parser sees them. */ - private fun validateRuns(bytes: ByteArray, samples: Int, composition: Boolean): Long { + private fun validateRuns(bytes: ByteArray, samples: Int, composition: Boolean, unedited: Boolean = false): Long { val data = ByteBuffer.wrap(bytes) data.position(8) val version = data.int require(version == 0 || (composition && version == 0x01000000)) val rows = data.int require(rows in 1..samples && data.remaining().toLong() == rows.toLong() * 8) + val firstValue = data.getInt(20) var count = 0L var duration = 0L repeat(rows) { val run = data.int val value = data.int // Media3 also accepts signed composition offsets in version zero. require(run > 0 && (composition || value > 0)) + if (composition && unedited) require(value >= 0 && value == firstValue) count += run require(count <= samples) if (!composition) duration += run.toLong() * value diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/MediaMetadataInspector.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/MediaMetadataInspector.kt index b8448e4d..27f26fce 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/MediaMetadataInspector.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/MediaMetadataInspector.kt @@ -84,7 +84,9 @@ internal class MediaMetadataInspector { size.height == format.int(MediaFormat.KEY_HEIGHT) && orientation.hasSimpleSampleGeometry) orientation.pixelAspectRatio else MediaFact.Unknown val track = MediaTrackMetadataReader.read(index, format, - containerRotation = orientation?.rotationDegrees, containerAspect = aspect) + containerRotation = orientation?.rotationDegrees + ?: if (container == MediaFact.Known(ContainerKind.IsoBaseMedia)) MediaFact.Unknown else null, + containerAspect = aspect) val id = track.sourceTrackId.valueOrNull val match = if (id != null) containers?.indexOfFirst { it.trackId.valueOrNull == id } else null val containerIndex = when { diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/IsoTimelineMetadataTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/IsoTimelineMetadataTests.kt index fd11b2d8..8fc7a4d4 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/IsoTimelineMetadataTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/IsoTimelineMetadataTests.kt @@ -1,6 +1,10 @@ package com.mux.video.upload.internal.standardization import android.media.MediaFormat +import androidx.media3.extractor.mp4.BoxParser +import io.mockk.every +import io.mockk.mockkStatic +import io.mockk.unmockkStatic import com.mux.exoplayeradapter.AbsRobolectricTest import org.junit.Assert.* import org.junit.Test @@ -82,7 +86,8 @@ class IsoTimelineMetadataTests : AbsRobolectricTest() { } @Test fun supportsVersionOneHeadersAndSignedCompositionOffsets() { - val video = track(version = 1, composition = runs("ctts", listOf(1 to 100, 1 to -100, 1 to 0), 1)) + val video = track(version = 1, composition = runs("ctts", listOf(1 to 100, 1 to -100, 1 to 0), 1), + edits = edit(listOf(Triple(3000L, 0L, 65536)))) val audio = track("soun", 2, version = 1) val f = facts(read(video, audio, version = 1)) assertEquals(MediaFact.Known(3.0), f.durationSeconds) @@ -93,6 +98,10 @@ class IsoTimelineMetadataTests : AbsRobolectricTest() { val t = track(timing = runs("stts", listOf(1 to 1000, 2 to 1000)), composition = runs("ctts", listOf(2 to 0, 1 to 0))) assertEquals(MediaFact.Known(3.0), facts(read(t)).durationSeconds) + val shifted = track(composition = runs("ctts", listOf(3 to 100))) + val f = facts(read(shifted, track("soun", 2), duration = 3100)) + assertEquals(MediaFact.Known(3.1), f.durationSeconds) + assertEquals(-0.1, f.audioVideoStartOffsetSeconds.valueOrNull!!, 1e-9) } @Test fun denseVfrAndCompositionTablesHaveIndependentPerTrackBounds() { @@ -106,11 +115,36 @@ class IsoTimelineMetadataTests : AbsRobolectricTest() { assertEquals(MediaFact.Known(0.0), facts(iso).audioVideoStartOffsetSeconds) } - @Test fun versionZeroNegativeCompositionOffsetsMatchMedia3() { - val t = track(composition = runs("ctts", listOf(1 to -1000, 2 to 0))) - val f = facts(read(t, track("soun", 2))) - assertEquals(MediaFact.Known(3.0), f.durationSeconds) - assertEquals(MediaFact.Known(1.0), f.audioVideoStartOffsetSeconds) + @Test fun uneditedNegativeCompositionStartDoesNotClaimAnEffectiveOffset() { + for (offsets in listOf(listOf(1 to -1000, 2 to 0), listOf(3 to -1000))) { + val t = track(composition = runs("ctts", offsets)) + val f = facts(read(t, track("soun", 2))) + assertEquals(MediaFact.Unknown, f.durationSeconds) + assertEquals(MediaFact.Unknown, f.audioVideoStartOffsetSeconds) + assertEquals(MediaFact.Known(Dimensions(1920, 1080)), f.displayDimensions) + } + } + + @Test fun uneditedReorderingCannotClaimDurationFromTheLastDecodedFrame() { + for (lastOffset in listOf(-1000, 0)) { + val t = track(composition = runs("ctts", listOf(1 to 0, 1 to 2000, 1 to lastOffset), 1)) + for (duration in listOf(2000L, 3000L, 4000L)) { + val iso = read(t, duration = duration) + assertEquals(MediaFact.Unknown, iso.single().timeline) + assertEquals(MediaFact.Unknown, facts(iso).durationSeconds) + } + } + } + + @Test fun timelineLinkageFailureRetainsIndependentMetadata() { + mockkStatic(BoxParser::class) + try { + every { BoxParser.parseStbl(any(), any(), any(), any()) } throws NoSuchMethodError("parseStbl") + val f = facts(read(track())) + assertEquals(MediaFact.Unknown, f.durationSeconds) + assertEquals(MediaFact.Known(VideoCodec.Other), f.videoCodec) + assertEquals(MediaFact.Known(Dimensions(1920, 1080)), f.displayDimensions) + } finally { unmockkStatic(BoxParser::class) } } @Test fun editedTimelineIncludesLateDecodedEarlyPresentationFrames() { diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/MediaMetadataReaderTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/MediaMetadataReaderTests.kt index 1373e93d..8469acab 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/MediaMetadataReaderTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/MediaMetadataReaderTests.kt @@ -264,13 +264,17 @@ class MediaMetadataReaderTests : AbsRobolectricTest() { } @Test fun failedIsoReadRetainsResolutionButDoesNotGuessMultipleAudioOrder() { - val tracks = listOf(MediaTrackMetadataReader.read(0, video(rotation = null)), - MediaTrackMetadataReader.read(1, audio(2, "1190")), MediaTrackMetadataReader.read(2, audio(1, "1188"))) - val f = MediaContainerMetadataReader.facts(MediaFact.Known(ContainerKind.IsoBaseMedia), MediaFact.Unknown, tracks) - assertEquals(MediaFact.Known(VideoCodec.H264), f.videoCodec) - assertEquals(MediaFact.Known(Dimensions(1920, 1080)), f.displayDimensions) - assertEquals(MediaFact.Unknown, f.audioTracks) - assertEquals(MediaFact.Unknown, f.durationSeconds) + for (rotation in listOf(0, null)) { + val tracks = listOf(MediaTrackMetadataReader.read(0, video(rotation = rotation), containerRotation = MediaFact.Unknown), + MediaTrackMetadataReader.read(1, audio(2, "1190")), MediaTrackMetadataReader.read(2, audio(1, "1188"))) + val f = MediaContainerMetadataReader.facts(MediaFact.Known(ContainerKind.IsoBaseMedia), MediaFact.Unknown, tracks) + assertEquals(MediaFact.Known(VideoCodec.H264), f.videoCodec) + assertEquals(MediaFact.Known(Dimensions(1920, 1080)), f.encodedDimensions) + assertEquals(if (rotation == null) MediaFact.Unknown else MediaFact.Known(0), f.rotationDegrees) + assertEquals(if (rotation == null) MediaFact.Unknown else MediaFact.Known(Dimensions(1920, 1080)), f.displayDimensions) + assertEquals(MediaFact.Unknown, f.audioTracks) + assertEquals(MediaFact.Unknown, f.durationSeconds) + } } @Test fun matroskaWithoutRotationRetainsKnownDisplayDimensions() { From 75b52b9f495dd9fee37418a2e56212232b339e7a Mon Sep 17 00:00:00 2001 From: Eugene Morozov Date: Fri, 2 Oct 2026 14:47:50 -0700 Subject: [PATCH 06/49] Simplify metadata inspection and defer effective timeline validation --- .../internal/standardization/CodecMetadata.kt | 34 +-- .../standardization/IsoContainerMetadata.kt | 71 ++---- .../standardization/IsoTimelineMetadata.kt | 158 ------------ .../standardization/MediaMetadataInspector.kt | 67 +---- .../IsoContainerMetadataTests.kt | 34 +-- .../IsoTimelineMetadataTests.kt | 238 ------------------ .../MediaMetadataReaderTests.kt | 38 ++- 7 files changed, 62 insertions(+), 578 deletions(-) delete mode 100644 library/src/main/java/com/mux/video/upload/internal/standardization/IsoTimelineMetadata.kt delete mode 100644 library/src/test/java/com/mux/video/upload/internal/standardization/IsoTimelineMetadataTests.kt diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/CodecMetadata.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/CodecMetadata.kt index 1a8ca8b0..c0e8480a 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/CodecMetadata.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/CodecMetadata.kt @@ -132,36 +132,24 @@ internal object CodecMetadataReader { } val sets = bits.readUnsignedExpGolombCodedInt() if (sets !in 0..64) return false - var previousPocs = emptyList() + var previousCount = 0 for (i in 0 until sets) { if (i > 0 && bits.readBit()) { - val sign = if (bits.readBit()) -1 else 1 - val delta = bits.readUnsignedExpGolombCodedInt() - if (delta !in 0..65535) return false - val deltaRps = sign * (delta + 1) - // In an SPS the reference is always the preceding set; delta_idx_minus1 is absent. - val predicted = (previousPocs + 0).mapNotNull { poc -> - val used = bits.readBit() || bits.readBit() - (poc + deltaRps).takeIf { used && it != 0 } - } - if (predicted.size > 16) return false - previousPocs = predicted.filter { it < 0 }.sortedDescending() + predicted.filter { it > 0 }.sorted() + // Media3 derives predicted sets. Only a final predicted set can be checked without + // duplicating those calculations; longer predicted chains remain unknown. + if (i != sets - 1) return false + bits.skipBit() + bits.readUnsignedExpGolombCodedInt() + repeat(previousCount + 1) { if (!bits.readBit()) bits.skipBit() } } else { val negative = bits.readUnsignedExpGolombCodedInt() val positive = bits.readUnsignedExpGolombCodedInt() if (negative !in 0..16 || positive !in 0..16 || negative + positive > 16) return false - val pocs = mutableListOf() - for ((count, sign) in listOf(negative to -1, positive to 1)) { - var poc = 0 - repeat(count) { - val delta = bits.readUnsignedExpGolombCodedInt() - if (delta !in 0..65535) return false - poc += sign * (delta + 1) - pocs.add(poc) - bits.skipBit() - } + previousCount = negative + positive + repeat(previousCount) { + bits.readUnsignedExpGolombCodedInt() + bits.skipBit() } - previousPocs = pocs } } if (bits.readBit()) { diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/IsoContainerMetadata.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/IsoContainerMetadata.kt index 0fdcb470..dd78ff02 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/IsoContainerMetadata.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/IsoContainerMetadata.kt @@ -10,12 +10,7 @@ internal object IsoSampleEntries { val avc = setOf("avc1", "avc3", "dva1", "dvav") val hevc = setOf("hvc1", "hev1", "dvh1", "dvhe") val dolby = setOf("dvh1", "dvhe", "dva1", "dvav") - val visual = avc + hevc + setOf("mp4v", "s263", "vp08", "vp09", "av01", "apco", "apcs", "apcn", "apch", "ap4h", "ap4x") - fun codec(entry: String): VideoCodec? = when (entry) { - in avc -> VideoCodec.H264 - in hevc -> VideoCodec.Hevc - else -> null - } + val visual = avc + hevc } internal data class IsoTrackMetadata( @@ -25,16 +20,13 @@ internal data class IsoTrackMetadata( val hasDolbyVisionConfiguration: Boolean, val rotationDegrees: MediaFact, val hasSimpleSampleGeometry: Boolean, - val pixelAspectRatio: MediaFact = MediaFact.Unknown, - val sampleDimensions: MediaFact = MediaFact.Unknown, - val timeline: MediaFact = MediaFact.Unknown, ) { val hasSimpleOrientation: Boolean get() = rotationDegrees != MediaFact.Unknown fun matches(kind: TrackKind): Boolean = (handler == "vide" && kind == TrackKind.Video) || (handler == "soun" && kind == TrackKind.Audio) } -/** Seeks past media payloads; reads bounded track, geometry and presentation timing metadata. */ +/** Seeks past media payloads; reads track identity, simple orientation and Dolby Vision signaling. */ internal object IsoContainerMetadataReader { private const val MAX_BOXES = 4096 private const val MAX_TOP_LEVEL_BOXES = 131072 @@ -44,27 +36,22 @@ internal object IsoContainerMetadataReader { RandomAccessFile(file, "r").use { input -> val reader = Reader(input) var movie: IsoMetadataBox? = null - var fragmented = false for (box in reader.boxes(0, input.length(), topLevel = true)) { if (box.type == "moov") { require(movie == null); movie = box } - if (box.type == "moof") fragmented = true } val moov = movie ?: return MediaFact.Unknown val children = reader.boxes(moov.payload, moov.end).toList() - val movieHeader = children.singleOrNull { it.type == "mvhd" } val tracks = children.filter { it.type == "trak" } if (tracks.size !in 1..64) return MediaFact.Unknown - val metadata = tracks.map { reader.track(it, movieHeader, fragmented) } + val metadata = tracks.map { reader.track(it) } val ids = metadata.mapNotNull { it.trackId.valueOrNull } require(ids.distinct().size == ids.size) MediaFact.Known(metadata) } - } catch (_: LinkageError) { MediaFact.Unknown } - catch (_: Exception) { MediaFact.Unknown } + } catch (_: Exception) { MediaFact.Unknown } private class Reader(val input: RandomAccessFile) { var boxCount = 0 - val timelines = IsoTimelineMetadataReader(input) val boxHeader = ByteArray(16) fun boxes(start: Long, end: Long, allowQuickTimeTerminator: Boolean = false, @@ -94,7 +81,7 @@ internal object IsoContainerMetadataReader { } } - fun track(track: IsoMetadataBox, movieHeader: IsoMetadataBox?, fragmented: Boolean): IsoTrackMetadata { + fun track(track: IsoMetadataBox): IsoTrackMetadata { val children = boxes(track.payload, track.end).toList() val tkhd = children.singleOrNull { it.type == "tkhd" } val mdia = children.single { it.type == "mdia" } @@ -127,57 +114,29 @@ internal object IsoContainerMetadataReader { val entries = boxes(stsd.payload + 8, stsd.end).toList() require(entries.size == count) var dolby = false - var simpleGeometry = true - var aspect: MediaFact = MediaFact.Unknown - var sampleSize: MediaFact = MediaFact.Unknown + var simpleGeometry = entries.size == 1 if (handler == "vide") { for (entry in entries) { if (entry.type in IsoSampleEntries.dolby) dolby = true if (entry.type in IsoSampleEntries.visual) { require(entry.end - entry.payload >= 78) - input.seek(entry.payload + 24) - val width = input.readUnsignedShort() - val height = input.readUnsignedShort() - if (width > 0 && height > 0) sampleSize = MediaFact.Known(Dimensions(width, height)) val extensions = boxes(entry.payload + 78, entry.end, allowQuickTimeTerminator = true).toList() if (extensions.any { it.type in setOf("dvcC", "dvvC", "dvwC") }) dolby = true if (extensions.any { it.type == "clap" }) simpleGeometry = false val aspects = extensions.filter { it.type == "pasp" } if (aspects.size > 1) simpleGeometry = false - aspect = aspects.singleOrNull()?.let { - require(it.end - it.payload == 8L) - input.seek(it.payload) - val horizontal = input.readInt().toLong() and 0xffffffffL - val vertical = input.readInt().toLong() and 0xffffffffL - if (horizontal == 0L || vertical == 0L) { simpleGeometry = false; MediaFact.Unknown } - else MediaFact.Known(horizontal.toDouble() / vertical) - } ?: MediaFact.Unknown - // A declared presentation size can establish aspect even for codecs without an SPS reader. - if (aspects.isEmpty() && IsoSampleEntries.codec(entry.type) == null) aspect = trackAspect(tkhd, sampleSize) + for (aspect in aspects) { + require(aspect.end - aspect.payload == 8L) + input.seek(aspect.payload) + val horizontal = input.readInt() + val vertical = input.readInt() + // Only square pixels are supported; unusual geometry keeps original-file fallback. + if (horizontal == 0 || horizontal != vertical) simpleGeometry = false + } } else simpleGeometry = false } } - val timeline = if (fragmented || entries.size != 1) MediaFact.Unknown else try { - val edits = children.filter { it.type == "edts" } - require(edits.size <= 1) - val editChildren = edits.singleOrNull()?.let { boxes(it.payload, it.end).toList() } - timelines.read(movieHeader, media.singleOrNull { it.type == "mdhd" }, editChildren, tables) - } catch (_: Exception) { MediaFact.Unknown } - return IsoTrackMetadata(handler, trackId, entries.map { it.type }, dolby, orientation, simpleGeometry, - if (entries.size == 1 && simpleGeometry) aspect else MediaFact.Unknown, - if (entries.size == 1) sampleSize else MediaFact.Unknown, timeline) - } - - private fun trackAspect(tkhd: IsoMetadataBox?, sampleSize: MediaFact): MediaFact { - val size = sampleSize.valueOrNull ?: return MediaFact.Unknown - val header = tkhd ?: return MediaFact.Unknown - input.seek(header.payload) - val offset = when (input.readUnsignedByte()) { 0 -> 76; 1 -> 88; else -> return MediaFact.Unknown } - if (header.end - header.payload < offset + 8) return MediaFact.Unknown - input.seek(header.payload + offset) - val width = (input.readInt().toLong() and 0xffffffffL) / 65536.0 - val height = (input.readInt().toLong() and 0xffffffffL) / 65536.0 - return if (width > 0 && height > 0) positive(width / height * size.height / size.width) else MediaFact.Unknown + return IsoTrackMetadata(handler, trackId, entries.map { it.type }, dolby, orientation, simpleGeometry) } private fun orientation(box: IsoMetadataBox): MediaFact { diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/IsoTimelineMetadata.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/IsoTimelineMetadata.kt deleted file mode 100644 index 9748a158..00000000 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/IsoTimelineMetadata.kt +++ /dev/null @@ -1,158 +0,0 @@ -package com.mux.video.upload.internal.standardization - -import androidx.media3.common.C -import androidx.media3.common.Format -import androidx.media3.common.util.ParsableByteArray -import androidx.media3.container.Mp4Box -import androidx.media3.extractor.GaplessInfoHolder -import androidx.media3.extractor.mp4.BoxParser -import androidx.media3.extractor.mp4.Track -import com.google.common.primitives.ImmutableLongArray -import java.io.RandomAccessFile -import java.nio.ByteBuffer - -internal data class IsoTrackTimeline( - val startSeconds: Double, - val endSeconds: Double, - val movieDurationSeconds: Double, - val movieTickSeconds: Double, - val hasEdits: Boolean, -) - -/** Bounds metadata allocations, then delegates composition timestamps and edits to Media3. */ -internal class IsoTimelineMetadataReader(private val input: RandomAccessFile) { - private data class Timebase(val scale: Long, val duration: Long) - private data class Edit(val duration: Long, val mediaTime: Long) - - fun read(movieHeader: IsoMetadataBox?, mediaHeader: IsoMetadataBox?, - editChildren: List?, tables: List): MediaFact = try { - val movie = timebase(movieHeader ?: return MediaFact.Unknown) - val media = timebase(mediaHeader ?: return MediaFact.Unknown) - val selected = tables.filter { it.type in TABLE_TYPES } - require(selected.map { it.type }.distinct().size == selected.size) - require(selected.sumOf { it.end - it.payload + 8 } <= MAX_TABLE_BYTES) - val data = selected.associate { it.type to bytes(it) } - val samples = sampleCount(data) - require(samples in 1..MAX_SAMPLES) - require(validateRuns(data.getValue("stts"), samples, composition = false) == media.duration) - val edits = editChildren?.let { - require(it.size == 1 && it.single().type == "elst") - edits(it.single()) - } - // Without an edit, Media3's duration uses the last decoded composition offset, not the - // maximum presentation end. Only a uniform nonnegative offset establishes that end here. - data["ctts"]?.let { validateRuns(it, samples, composition = true, unedited = edits == null) } - val stbl = Mp4Box.ContainerBox(Mp4Box.TYPE_stbl, 0) - data.forEach { (type, bytes) -> - val code = ByteBuffer.wrap(type.toByteArray(Charsets.US_ASCII)).int - stbl.add(Mp4Box.LeafBox(code, ParsableByteArray(bytes))) - } - // Request generic timing, so Media3 applies edits to timestamps rather than translating - // short audio trims into codec-specific gapless delay/padding. No codec parsing is needed. - val track = Track.Builder().setType(C.TRACK_TYPE_UNKNOWN).setTimescale(media.scale) - .setMovieTimescale(movie.scale).setFormat(Format.Builder().setMaxNumReorderSamples(samples).build()) - .setEditListDurations(edits?.let { ImmutableLongArray.copyOf(it.map(Edit::duration)) }) - .setEditListMediaTimes(edits?.let { ImmutableLongArray.copyOf(it.map(Edit::mediaTime)) }).build() - val parsed = BoxParser.parseStbl(track, stbl, GaplessInfoHolder(), false) - require(parsed.timestampsUs.isNotEmpty() && parsed.durationUs > 0) - if (edits == null) require(parsed.sampleCount == samples) - val emptyDuration = if (edits?.size == 2) edits.first().duration.toDouble() / movie.scale else 0.0 - val first = parsed.timestampsUs.min().toDouble() / 1_000_000 - // Edit preroll is needed for decoding but is outside the effective presentation interval. - val start = if (edits == null) first else maxOf(first, emptyDuration) - val end = parsed.durationUs.toDouble() / 1_000_000 - require(start.isFinite() && start >= 0 && end > start) - MediaFact.Known(IsoTrackTimeline(start, end, movie.duration.toDouble() / movie.scale, - 1.0 / movie.scale, edits != null)) - } catch (_: LinkageError) { MediaFact.Unknown } - catch (_: Exception) { MediaFact.Unknown } - - /** A single buffered read per box; Media3 expects the ordinary eight-byte box header. */ - private fun bytes(box: IsoMetadataBox): ByteArray { - val length = box.end - box.payload - require(length in 0..MAX_TABLE_BYTES) - val result = ByteArray(length.toInt() + 8) - ByteBuffer.wrap(result).putInt(result.size).put(box.type.toByteArray(Charsets.US_ASCII)) - input.seek(box.payload) - input.readFully(result, 8, length.toInt()) - return result - } - - private fun timebase(box: IsoMetadataBox): Timebase { - val data = ByteBuffer.wrap(bytes(box)) - val version = data.getInt(8) - require(version == 0 || version == 0x01000000) - data.position(if (version == 0) 20 else 28) - val scale = data.int.toLong() and 0xffffffffL - val duration = if (version == 0) data.int.toLong() and 0xffffffffL else data.long - require(scale > 0 && duration in 1..MAX_EXACT_TIME && (version != 0 || duration != 0xffffffffL)) - return Timebase(scale, duration) - } - - /** Reject inconsistent or allocation-driving counts before the reusable parser sees them. */ - private fun validateRuns(bytes: ByteArray, samples: Int, composition: Boolean, unedited: Boolean = false): Long { - val data = ByteBuffer.wrap(bytes) - data.position(8) - val version = data.int - require(version == 0 || (composition && version == 0x01000000)) - val rows = data.int - require(rows in 1..samples && data.remaining().toLong() == rows.toLong() * 8) - val firstValue = data.getInt(20) - var count = 0L - var duration = 0L - repeat(rows) { - val run = data.int - val value = data.int // Media3 also accepts signed composition offsets in version zero. - require(run > 0 && (composition || value > 0)) - if (composition && unedited) require(value >= 0 && value == firstValue) - count += run - require(count <= samples) - if (!composition) duration += run.toLong() * value - } - require(count == samples.toLong() && duration <= MAX_EXACT_TIME) - return duration - } - - private fun sampleCount(data: Map): Int { - require(data.keys.count { it == "stsz" || it == "stz2" } == 1) - val compact = "stz2" in data - val bytes = ByteBuffer.wrap(data.getValue(if (compact) "stz2" else "stsz")) - bytes.position(8) - require(bytes.int == 0) - val size = bytes.int.toLong() and 0xffffffffL - val count = bytes.int - require(count in 1..MAX_SAMPLES) - val remaining = if (!compact) { if (size > 0) 0L else count.toLong() * 4 } else { - require(size in setOf(4L, 8L, 16L)) - (count * size + 7) / 8 - } - require(bytes.remaining().toLong() == remaining) - return count - } - - private fun edits(box: IsoMetadataBox): List { - val data = ByteBuffer.wrap(bytes(box)) - data.position(8) - val version = data.int - require(version == 0 || version == 0x01000000) - val count = data.int - require(count in 1..2 && data.remaining() == count * if (version == 0) 12 else 20) - val edits = List(count) { - val duration = if (version == 0) data.int.toLong() and 0xffffffffL else data.long - val time = if (version == 0) data.int.toLong() else data.long - require(duration in 1..MAX_EXACT_TIME && time in -1..MAX_EXACT_TIME && data.int == 65536) - Edit(duration, time) - } - require(edits.last().mediaTime >= 0 && (count == 1 || edits.first().mediaTime == -1L)) - return edits - } - - companion object { - // Per-track limits allow dense VFR/B-frame tables without a shared row budget. - // Media3 materializes per-sample metadata; cap that allocation independently of file size. - private const val MAX_SAMPLES = 500_000 - private const val MAX_TABLE_BYTES = 8L * 1024 * 1024 - private const val MAX_EXACT_TIME = 1L shl 53 - private val TABLE_TYPES = setOf("stts", "ctts", "stsz", "stz2", "stsc", "stco", "co64", "stss") - } -} diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/MediaMetadataInspector.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/MediaMetadataInspector.kt index 27f26fce..dcd0bb6b 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/MediaMetadataInspector.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/MediaMetadataInspector.kt @@ -7,8 +7,6 @@ import android.os.Build import java.io.File import java.io.IOException import java.nio.ByteBuffer -import kotlin.math.roundToInt -import kotlin.math.abs internal enum class ContainerKind { IsoBaseMedia, Matroska, Other } internal enum class TrackKind { Video, Audio, Other, Unknown } @@ -53,6 +51,7 @@ internal data class TrackMetadata( val audio: AudioTrackMetadata? = null, ) +/** Effective duration and A/V offset remain unknown until timeline validation is implemented. */ internal data class MediaMetadataInspection( val container: MediaFact, val tracks: List, @@ -79,14 +78,9 @@ internal class MediaMetadataInspector { val containers = iso.valueOrNull val orientation = if (format.string(MediaFormat.KEY_MIME)?.startsWith("video/") == true) containers?.filter { it.handler == "vide" }?.singleOrNull() else null - val size = orientation?.sampleDimensions?.valueOrNull - val aspect = if (size != null && size.width == format.int(MediaFormat.KEY_WIDTH) && - size.height == format.int(MediaFormat.KEY_HEIGHT) && orientation.hasSimpleSampleGeometry) - orientation.pixelAspectRatio else MediaFact.Unknown val track = MediaTrackMetadataReader.read(index, format, containerRotation = orientation?.rotationDegrees - ?: if (container == MediaFact.Known(ContainerKind.IsoBaseMedia)) MediaFact.Unknown else null, - containerAspect = aspect) + ?: if (container == MediaFact.Known(ContainerKind.IsoBaseMedia)) MediaFact.Unknown else null) val id = track.sourceTrackId.valueOrNull val match = if (id != null) containers?.indexOfFirst { it.trackId.valueOrNull == id } else null val containerIndex = when { @@ -131,7 +125,7 @@ internal class MediaMetadataInspector { } } -/** Reconcile physical container tracks with extractor views, including Dolby Vision base-layer views. */ +/** Container identities establish physical track count and audio order; ambiguous views stay unknown. */ internal object MediaContainerMetadataReader { fun facts(container: MediaFact, iso: MediaFact>, tracks: List): MediaFacts { @@ -162,47 +156,20 @@ internal object MediaContainerMetadataReader { if (description.sampleEntries.size != 1) return base val views = tracks.filter { it.kind == TrackKind.Video } if (views.any { it.containerIndex.valueOrNull != index }) return base - val selected = if (views.size == 1) views.single() else { - if (!dolby || views.size != 2 || views.count { it.mimeType.valueOrNull == "video/dolby-vision" } != 1) return base - val codec = IsoSampleEntries.codec(description.sampleEntries.single()) ?: return base - views.singleOrNull { it.video?.codec == MediaFact.Known(codec) } ?: return base - } + val selected = views.singleOrNull() ?: return base val video = selected.video ?: return base - val facts = video.toFacts(base).copy( + return video.toFacts(base).copy( displayDimensions = if (description.hasSimpleOrientation && description.hasSimpleSampleGeometry) video.displayDimensions else MediaFact.Unknown, rotationDegrees = if (description.hasSimpleOrientation) video.rotationDegrees else MediaFact.Unknown, dynamicRange = if (dolby) MediaFact.Known(DynamicRange.DolbyVision) else video.dynamicRange, ) - return timelineFacts(facts, containers, index, audioIndices.firstOrNull(), audioViews) - } - - private fun timelineFacts(base: MediaFacts, containers: List, videoIndex: Int, - audioIndex: Int?, audioViews: Map>): MediaFacts { - val media = containers.filter { it.handler == "vide" || it.handler == "soun" } - val timelines = media.map { it.timeline.valueOrNull } - val duration = if (timelines.all { it != null }) { - val known = timelines.filterNotNull() - val declaration = known.map { it.movieDurationSeconds }.distinct().singleOrNull() - val end = known.maxOf { it.endSeconds } - if (declaration != null && abs(declaration - end) <= known.maxOf { it.movieTickSeconds } + 1e-6) - MediaFact.Known(declaration) else MediaFact.Unknown - } else MediaFact.Unknown - val video = containers[videoIndex].timeline.valueOrNull - val audio = audioIndex?.let { containers[it].timeline.valueOrNull } - val audioMetadata = audioIndex?.let { audioViews[it]?.singleOrNull()?.audio } - // An edit already accounts for codec priming. Do not apply extractor delay/padding twice. - val unaccountedPriming = audio != null && !audio.hasEdits && audioMetadata != null && - ((audioMetadata.encoderDelaySamples.valueOrNull ?: 0) > 0 || (audioMetadata.encoderPaddingSamples.valueOrNull ?: 0) > 0) - val offset = if (video != null && audio != null && !unaccountedPriming) - MediaFact.Known(audio.startSeconds - video.startSeconds) else MediaFact.Unknown - return base.copy(durationSeconds = duration, audioVideoStartOffsetSeconds = offset) } } internal object MediaTrackMetadataReader { fun read(index: Int, format: MediaFormat, apiLevel: Int = Build.VERSION.SDK_INT, - containerRotation: MediaFact? = null, containerAspect: MediaFact = MediaFact.Unknown): TrackMetadata { + containerRotation: MediaFact? = null): TrackMetadata { val mime = format.string(MediaFormat.KEY_MIME) val kind = when { mime == null -> TrackKind.Unknown @@ -215,7 +182,7 @@ internal object MediaTrackMetadataReader { positive(format.long(MediaFormat.KEY_DURATION)?.toDouble()?.div(1_000_000)), positive(format.number(MediaFormat.KEY_FRAME_RATE)), positiveCode(format.int(MediaFormat.KEY_BIT_RATE)), - video = if (kind == TrackKind.Video) video(format, mime, apiLevel, containerRotation, containerAspect) else null, + video = if (kind == TrackKind.Video) video(format, mime, apiLevel, containerRotation) else null, audio = if (kind == TrackKind.Audio) audio(format, mime) else null, ) } @@ -238,7 +205,7 @@ internal object MediaTrackMetadataReader { } private fun video(format: MediaFormat, mime: String?, apiLevel: Int, - containerRotation: MediaFact?, containerAspect: MediaFact): VideoTrackMetadata { + containerRotation: MediaFact?): VideoTrackMetadata { val codec = when (mime) { "video/avc" -> MediaFact.Known(VideoCodec.H264) "video/hevc" -> MediaFact.Known(VideoCodec.Hevc) @@ -266,7 +233,7 @@ internal object MediaTrackMetadataReader { else dynamicRange(transfer) return VideoTrackMetadata(codec, reconcile(platformProfile(codec.valueOrNull, format.int(MediaFormat.KEY_PROFILE)), config.profile), - encoded, display(format, encoded, rotation, reconcileAspect(config.pixelAspectRatio, containerAspect)), rotation, platformRotation, + encoded, display(format, encoded, rotation, config.pixelAspectRatio), rotation, platformRotation, config, platformColor, range) } @@ -299,11 +266,9 @@ internal object MediaTrackMetadataReader { val platformAspect = if (sarWidth != null && sarHeight != null && sarWidth > 0 && sarHeight > 0) MediaFact.Known(sarWidth.toDouble() / sarHeight) else MediaFact.Unknown if ((sarWidth != null || sarHeight != null) && platformAspect == MediaFact.Unknown) return MediaFact.Unknown - val aspect = reconcileAspect(platformAspect, headerAspect).valueOrNull ?: return MediaFact.Unknown - val width = visible.width * aspect - if (!width.isFinite() || width < 1 || width > Int.MAX_VALUE) return MediaFact.Unknown - val display = Dimensions(width.roundToInt(), visible.height) - return MediaFact.Known(if (degrees == 90 || degrees == 270) Dimensions(display.height, display.width) else display) + // Phone recordings use square pixels. Do not infer display geometry for unusual aspect ratios. + if (reconcile(platformAspect, headerAspect) != MediaFact.Known(1.0)) return MediaFact.Unknown + return MediaFact.Known(if (degrees == 90 || degrees == 270) Dimensions(visible.height, visible.width) else visible) } private fun dimensions(width: Int?, height: Int?): MediaFact = @@ -356,11 +321,3 @@ private fun VideoTrackMetadata.toFacts(base: MediaFacts): MediaFacts = base.copy displayDimensions = displayDimensions, rotationDegrees = rotationDegrees, pixelFormat = configuration.pixelFormat, dynamicRange = dynamicRange, ) - -private fun reconcileAspect(a: MediaFact, b: MediaFact): MediaFact { - val first = a.valueOrNull - val second = b.valueOrNull - // SPS ratios are floats; container ratios are exact integer ratios converted to doubles. - return if (first != null && second != null && abs(first - second) <= 1e-6 * maxOf(first, second)) a - else reconcile(a, b) -} diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/IsoContainerMetadataTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/IsoContainerMetadataTests.kt index 1266e05e..f231398f 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/IsoContainerMetadataTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/IsoContainerMetadataTests.kt @@ -1,6 +1,5 @@ package com.mux.video.upload.internal.standardization -import android.media.MediaFormat import com.mux.exoplayeradapter.AbsRobolectricTest import org.robolectric.annotation.Config import org.junit.Assert.* @@ -13,16 +12,15 @@ class IsoContainerMetadataTests : AbsRobolectricTest() { private fun int(value: Int) = ByteBuffer.allocate(4).putInt(value).array() private fun box(type: String, payload: ByteArray) = int(payload.size + 8) + type.toByteArray() + payload private fun track(handler: String = "vide", entry: String = "hvc1", extension: ByteArray = byteArrayOf(), - matrix: IntArray = intArrayOf(65536, 0, 0, 0, 65536, 0, 0, 0, 0x40000000), - swappedDisplaySize: Boolean = false): ByteArray { + matrix: IntArray = intArrayOf(65536, 0, 0, 0, 65536, 0, 0, 0, 0x40000000)): ByteArray { val tkhd = ByteArray(84) for (i in matrix.indices) int(matrix[i]).copyInto(tkhd, 40 + i * 4) val hdlr = ByteArray(8) + handler.toByteArray() val description = ByteArray(if (handler == "vide") 78 else 28) if (handler == "vide") { ByteBuffer.wrap(description).putShort(24, 1920.toShort()).putShort(26, 1080.toShort()) - int((if (swappedDisplaySize) 1080 else 1920) * 65536).copyInto(tkhd, 76) - int((if (swappedDisplaySize) 1920 else 1080) * 65536).copyInto(tkhd, 80) + int(1920 * 65536).copyInto(tkhd, 76) + int(1080 * 65536).copyInto(tkhd, 80) } val sampleEntry = box(entry, description + extension) val stsd = box("stsd", ByteArray(4) + int(1) + sampleEntry) @@ -62,31 +60,15 @@ class IsoContainerMetadataTests : AbsRobolectricTest() { assertTrue(read(box("moov", track(matrix = matrix))).valueOrNull!![0].hasSimpleOrientation) } - @Test fun cleanApertureStaysUnknownAndExplicitPixelAspectIsRetained() { - assertFalse(read(box("moov", track(extension = box("clap", ByteArray(32))))) - .valueOrNull!![0].hasSimpleSampleGeometry) - assertEquals(MediaFact.Known(2.0), read(box("moov", track(extension = box("pasp", int(2) + int(1))))) - .valueOrNull!![0].pixelAspectRatio) + @Test fun unsupportedGeometryStaysUnknownAndSquarePixelsAreAccepted() { + for (extension in listOf(box("clap", ByteArray(32)), box("pasp", int(2) + int(1)), + box("pasp", int(1) + int(1)) + box("pasp", int(1) + int(1)))) { + assertFalse(read(box("moov", track(extension = extension))).valueOrNull!![0].hasSimpleSampleGeometry) + } assertTrue(read(box("moov", track(extension = box("pasp", int(1) + int(1))))) .valueOrNull!![0].hasSimpleSampleGeometry) } - @Test fun inferredTrackAspectCannotContradictAvcSps() { - val iso = read(box("moov", track(entry = "avc1", swappedDisplaySize = true))).valueOrNull!!.single() - assertEquals(MediaFact.Unknown, iso.pixelAspectRatio) - val sps = "000000016742c028da01e0089f97016a02020280000003008000001e078c1950" - .chunked(2).map { it.toInt(16).toByte() }.toByteArray() - val format = MediaFormat.createVideoFormat("video/avc", 1920, 1080).apply { - setByteBuffer("csd-0", ByteBuffer.wrap(sps)) - } - val metadata = MediaTrackMetadataReader.read(0, format, 23, iso.rotationDegrees, iso.pixelAspectRatio) - assertEquals(MediaFact.Known(Dimensions(1920, 1080)), metadata.video!!.displayDimensions) - val explicit = read(box("moov", track(entry = "avc1", swappedDisplaySize = true, - extension = box("pasp", int(2) + int(1))))).valueOrNull!!.single() - assertEquals(MediaFact.Unknown, MediaTrackMetadataReader.read(0, format, 23, - explicit.rotationDegrees, explicit.pixelAspectRatio).video!!.displayDimensions) - } - @Test fun malformedSizesAndBoxLimitsStayUnknown() { for (data in listOf(byteArrayOf(), int(4096) + "moov".toByteArray(), int(4) + "moov".toByteArray(), int(1) + "moov".toByteArray() + ByteArray(8) { -1 })) { diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/IsoTimelineMetadataTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/IsoTimelineMetadataTests.kt deleted file mode 100644 index 8fc7a4d4..00000000 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/IsoTimelineMetadataTests.kt +++ /dev/null @@ -1,238 +0,0 @@ -package com.mux.video.upload.internal.standardization - -import android.media.MediaFormat -import androidx.media3.extractor.mp4.BoxParser -import io.mockk.every -import io.mockk.mockkStatic -import io.mockk.unmockkStatic -import com.mux.exoplayeradapter.AbsRobolectricTest -import org.junit.Assert.* -import org.junit.Test -import org.robolectric.annotation.Config -import java.io.File -import java.nio.ByteBuffer - -@Config(sdk = [23]) -class IsoTimelineMetadataTests : AbsRobolectricTest() { - private fun int(value: Int) = ByteBuffer.allocate(4).putInt(value).array() - private fun long(value: Long) = ByteBuffer.allocate(8).putLong(value).array() - private fun box(type: String, payload: ByteArray) = int(payload.size + 8) + type.toByteArray() + payload - private fun header(type: String, scale: Int, duration: Long, version: Int = 0): ByteArray = box(type, - int(version shl 24) + ByteArray(if (version == 0) 8 else 16) + int(scale) + - (if (version == 0) int(duration.toInt()) else long(duration))) - private fun runs(type: String, rows: List>, version: Int = 0): ByteArray { - val data = ByteBuffer.allocate(8 + rows.size * 8).putInt(version shl 24).putInt(rows.size) - rows.forEach { data.putInt(it.first).putInt(it.second) } - return box(type, data.array()) - } - private fun edit(rows: List>, version: Int = 0) = box("edts", box("elst", - int(version shl 24) + int(rows.size) + rows.fold(byteArrayOf()) { bytes, (duration, time, rate) -> - bytes + (if (version == 0) int(duration.toInt()) + int(time.toInt()) else long(duration) + long(time)) + int(rate) - })) - - private fun track(handler: String = "vide", id: Int = 1, scale: Int = 1000, duration: Long = 3000, - timing: ByteArray = runs("stts", listOf(3 to 1000)), samples: Int = 3, - composition: ByteArray = byteArrayOf(), edits: ByteArray = byteArrayOf(), version: Int = 0, - entry: String = if (handler == "vide") "vp09" else "mp4a", aspect: ByteArray = byteArrayOf()): ByteArray { - val tkhd = ByteArray(84) - int(id).copyInto(tkhd, 12) - val matrix = intArrayOf(65536, 0, 0, 0, 65536, 0, 0, 0, 0x40000000) - matrix.forEachIndexed { i, n -> int(n).copyInto(tkhd, 40 + i * 4) } - int(1920 * 65536).copyInto(tkhd, 76); int(1080 * 65536).copyInto(tkhd, 80) - val description = ByteArray(if (handler == "vide") 78 else 28) - if (handler == "vide") { - ByteBuffer.wrap(description).putShort(24, 1920.toShort()).putShort(26, 1080.toShort()) - } - val stsd = box("stsd", int(0) + int(1) + box(entry, description + aspect)) - val stsz = box("stsz", int(0) + int(1) + int(samples)) - val chunks = box("stco", int(0) + int(1) + int(0)) + - box("stsc", int(0) + int(1) + int(1) + int(samples) + int(1)) - return box("trak", box("tkhd", tkhd) + edits + box("mdia", header("mdhd", scale, duration, version) + - box("hdlr", ByteArray(8) + handler.toByteArray()) + box("minf", box("stbl", stsd + timing + composition + stsz + chunks)))) - } - - private fun read(vararg tracks: ByteArray, duration: Long = 3000, scale: Int = 1000, - version: Int = 0, suffix: ByteArray = byteArrayOf()): List { - val file = File.createTempFile("iso-timeline", ".mp4") - return try { - file.writeBytes(box("moov", header("mvhd", scale, duration, version) + tracks.fold(byteArrayOf(), ByteArray::plus)) + suffix) - IsoContainerMetadataReader.read(file).valueOrNull!! - } finally { file.delete() } - } - - private fun facts(iso: List, priming: Int = 0): MediaFacts { - val views = iso.mapIndexed { index, track -> - val format = if (track.handler == "vide") MediaFormat.createVideoFormat("video/x-vnd.on2.vp9", 1920, 1080) - .apply { setInteger(MediaFormat.KEY_ROTATION, 0); setInteger("color-transfer", 3) } - else MediaFormat.createAudioFormat("audio/mp4a-latm", 48000, 2).apply { - setByteBuffer("csd-0", ByteBuffer.wrap(byteArrayOf(0x11, 0x90.toByte()))) - setInteger(MediaFormat.KEY_ENCODER_DELAY, priming) - } - MediaTrackMetadataReader.read(index, format, 24, track.rotationDegrees, track.pixelAspectRatio) - .copy(containerIndex = MediaFact.Known(index)) - } - return MediaContainerMetadataReader.facts(MediaFact.Known(ContainerKind.IsoBaseMedia), MediaFact.Known(iso), views) - } - - @Test fun establishesUneditedDurationAndAvStartWithoutSamplePayloads() { - val iso = read(track(), track("soun", 2)) - val f = facts(iso) - assertEquals(MediaFact.Known(3.0), f.durationSeconds) - assertEquals(MediaFact.Known(0.0), f.audioVideoStartOffsetSeconds) - assertEquals(MediaFact.Unknown, f.timestamps) - assertEquals(MediaFact.Unknown, f.cadence) - assertEquals(MediaFact.Unknown, f.frameRate) - assertEquals(MediaFact.Unknown, f.editList) - } - - @Test fun supportsVersionOneHeadersAndSignedCompositionOffsets() { - val video = track(version = 1, composition = runs("ctts", listOf(1 to 100, 1 to -100, 1 to 0), 1), - edits = edit(listOf(Triple(3000L, 0L, 65536)))) - val audio = track("soun", 2, version = 1) - val f = facts(read(video, audio, version = 1)) - assertEquals(MediaFact.Known(3.0), f.durationSeconds) - assertEquals(-0.1, f.audioVideoStartOffsetSeconds.valueOrNull!!, 1e-9) - } - - @Test fun media3HandlesDifferentTimingRunBoundaries() { - val t = track(timing = runs("stts", listOf(1 to 1000, 2 to 1000)), - composition = runs("ctts", listOf(2 to 0, 1 to 0))) - assertEquals(MediaFact.Known(3.0), facts(read(t)).durationSeconds) - val shifted = track(composition = runs("ctts", listOf(3 to 100))) - val f = facts(read(shifted, track("soun", 2), duration = 3100)) - assertEquals(MediaFact.Known(3.1), f.durationSeconds) - assertEquals(-0.1, f.audioVideoStartOffsetSeconds.valueOrNull!!, 1e-9) - } - - @Test fun denseVfrAndCompositionTablesHaveIndependentPerTrackBounds() { - val count = 72000 - val timing = runs("stts", List(count) { 1 to if (it % 2 == 0) 33 else 34 }) - val composition = runs("ctts", List(count) { 1 to 0 }) - val duration = count / 2 * 67L - val iso = read(track(timing = timing, composition = composition, samples = count, duration = duration), - track("soun", 2, timing = timing, samples = count, duration = duration), duration = duration) - assertEquals(MediaFact.Known(duration / 1000.0), facts(iso).durationSeconds) - assertEquals(MediaFact.Known(0.0), facts(iso).audioVideoStartOffsetSeconds) - } - - @Test fun uneditedNegativeCompositionStartDoesNotClaimAnEffectiveOffset() { - for (offsets in listOf(listOf(1 to -1000, 2 to 0), listOf(3 to -1000))) { - val t = track(composition = runs("ctts", offsets)) - val f = facts(read(t, track("soun", 2))) - assertEquals(MediaFact.Unknown, f.durationSeconds) - assertEquals(MediaFact.Unknown, f.audioVideoStartOffsetSeconds) - assertEquals(MediaFact.Known(Dimensions(1920, 1080)), f.displayDimensions) - } - } - - @Test fun uneditedReorderingCannotClaimDurationFromTheLastDecodedFrame() { - for (lastOffset in listOf(-1000, 0)) { - val t = track(composition = runs("ctts", listOf(1 to 0, 1 to 2000, 1 to lastOffset), 1)) - for (duration in listOf(2000L, 3000L, 4000L)) { - val iso = read(t, duration = duration) - assertEquals(MediaFact.Unknown, iso.single().timeline) - assertEquals(MediaFact.Unknown, facts(iso).durationSeconds) - } - } - } - - @Test fun timelineLinkageFailureRetainsIndependentMetadata() { - mockkStatic(BoxParser::class) - try { - every { BoxParser.parseStbl(any(), any(), any(), any()) } throws NoSuchMethodError("parseStbl") - val f = facts(read(track())) - assertEquals(MediaFact.Unknown, f.durationSeconds) - assertEquals(MediaFact.Known(VideoCodec.Other), f.videoCodec) - assertEquals(MediaFact.Known(Dimensions(1920, 1080)), f.displayDimensions) - } finally { unmockkStatic(BoxParser::class) } - } - - @Test fun editedTimelineIncludesLateDecodedEarlyPresentationFrames() { - val video = track(samples = 5, duration = 5000, timing = runs("stts", listOf(5 to 1000)), - composition = runs("ctts", listOf(1 to 1000, 1 to 3000, 1 to 1000, 1 to -3000, 1 to -2000), 1), - edits = edit(listOf(Triple(2000L, 0L, 65536)))) - val audio = track("soun", 2, samples = 2, duration = 2000, timing = runs("stts", listOf(2 to 1000))) - val f = facts(read(video, audio, duration = 2000)) - assertEquals(MediaFact.Known(2.0), f.durationSeconds) - assertEquals(MediaFact.Known(0.0), f.audioVideoStartOffsetSeconds) - } - - @Test fun allocationLimitDoesNotDiscardOtherMetadata() { - val t = track(samples = 500001, duration = 500001000L, timing = runs("stts", listOf(500001 to 1000))) - val f = facts(read(t, duration = 500001000L)) - assertEquals(MediaFact.Unknown, f.durationSeconds) - assertEquals(MediaFact.Known(Dimensions(1920, 1080)), f.displayDimensions) - } - - @Test fun unitRateTrimMapsCompositionOffsetsIntoPresentationTime() { - val edits = edit(listOf(Triple(2000L, 1000L, 65536))) - val iso = read(track(edits = edits), track("soun", 2, edits = edits), duration = 2000) - val f = facts(iso, priming = 1024) - assertEquals(MediaFact.Known(2.0), f.durationSeconds) - assertEquals(MediaFact.Known(0.0), f.audioVideoStartOffsetSeconds) - } - - @Test fun leadingEmptyEditCreatesKnownAudioDelay() { - val delayed = edit(listOf(Triple(100L, -1L, 65536), Triple(2900L, 0L, 65536)), version = 1) - val f = facts(read(track(), track("soun", 2, edits = delayed))) - assertEquals(MediaFact.Known(3.0), f.durationSeconds) - assertEquals(0.1, f.audioVideoStartOffsetSeconds.valueOrNull!!, 1e-9) - } - - @Test fun editAndMediaTimescalesAreConvertedIndependently() { - val media = track(scale = 48000, duration = 144000, timing = runs("stts", listOf(3 to 48000)), - edits = edit(listOf(Triple(2000L, 48000L, 65536)))) - assertEquals(MediaFact.Known(2.0), facts(read(media, duration = 2000)).durationSeconds) - } - - @Test fun unsupportedEditsDoNotDiscardCodecAndGeometry() { - for (edits in listOf( - edit(listOf(Triple(3000L, 0L, 0))), - edit(listOf(Triple(1500L, 0L, 65536), Triple(1500L, 1000L, 65536))), - edit(listOf(Triple(3000L, -1L, 65536))))) { - val f = facts(read(track(edits = edits))) - assertEquals(MediaFact.Known(VideoCodec.Other), f.videoCodec) - assertEquals(MediaFact.Known(Dimensions(1920, 1080)), f.displayDimensions) - assertEquals(MediaFact.Unknown, f.durationSeconds) - } - } - - @Test fun malformedOrConflictingTablesKeepOnlyTimelineUnknown() { - for (t in listOf( - track(samples = 4), track(duration = 2000), - track(composition = runs("ctts", listOf(2 to 0))), - track(timing = runs("stts", listOf(3 to 0))), - track(timing = box("stts", int(0) + int(65537))), - track(timing = box("stts", int(0) + int(1) + int(3))), - track(composition = runs("ctts", listOf(3 to 0), 2)), - track(duration = 0), - track(timing = runs("stts", listOf(-1 to -1)), samples = -1))) { - val f = facts(read(t)) - assertEquals(MediaFact.Known(VideoCodec.Other), f.videoCodec) - assertEquals(MediaFact.Unknown, f.durationSeconds) - } - } - - @Test fun movieDurationConflictAndFragmentedTimelinesStayUnknown() { - assertEquals(MediaFact.Unknown, facts(read(track(), duration = 2000)).durationSeconds) - val iso = read(track(), suffix = box("moof", byteArrayOf()) + box("mdat", byteArrayOf())) - assertEquals(MediaFact.Unknown, iso.single().timeline) - assertEquals(MediaFact.Known(VideoCodec.Other), facts(iso).videoCodec) - } - - @Test fun primingWithoutAnEditCannotProveAudioStartOffset() { - val f = facts(read(track(), track("soun", 2)), priming = 1024) - assertEquals(MediaFact.Unknown, f.audioVideoStartOffsetSeconds) - assertEquals(MediaFact.Known(3.0), f.durationSeconds) - } - - @Test fun otherVisualCodecsUseDeclaredGeometryWithoutCodecHeaderGuessing() { - for (entry in listOf("vp09", "mp4v", "apch")) { - val iso = read(track(entry = entry)) - assertEquals(MediaFact.Known(1.0), iso.single().pixelAspectRatio) - assertEquals(MediaFact.Known(Dimensions(1920, 1080)), facts(iso).displayDimensions) - } - val anamorphic = read(track(aspect = box("pasp", int(2) + int(1)))) - assertEquals(MediaFact.Known(Dimensions(3840, 1080)), facts(anamorphic).displayDimensions) - } -} diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/MediaMetadataReaderTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/MediaMetadataReaderTests.kt index 8469acab..c1d6b08f 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/MediaMetadataReaderTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/MediaMetadataReaderTests.kt @@ -105,14 +105,17 @@ class MediaMetadataReaderTests : AbsRobolectricTest() { assertEquals(MediaFact.Unknown, facts(format).displayDimensions) } - @Test fun appliesCropAndPixelAspectRatio() { + @Test fun appliesOrdinaryCropAndRejectsNonSquarePixels() { val format = video(csd = null).apply { setInteger("crop-left", 10); setInteger("crop-right", 1909) setInteger("crop-top", 0); setInteger("crop-bottom", 1079) - setInteger("sar-width", 2); setInteger("sar-height", 1) + setInteger("sar-width", 1); setInteger("sar-height", 1) setInteger(MediaFormat.KEY_ROTATION, 90) } - assertEquals(MediaFact.Known(Dimensions(1080, 3800)), facts(format).displayDimensions) + assertEquals(MediaFact.Known(Dimensions(1080, 1900)), facts(format).displayDimensions) + format.setInteger("sar-width", 2) + assertEquals(MediaFact.Unknown, facts(format).displayDimensions) + format.setInteger("sar-width", 1) format.setInteger("crop-right", 1920) assertEquals(MediaFact.Unknown, facts(format).displayDimensions) } @@ -212,7 +215,7 @@ class MediaMetadataReaderTests : AbsRobolectricTest() { private fun isoTrack(handler: String, entry: String, id: Int, dolby: Boolean = false) = IsoTrackMetadata(handler, MediaFact.Known(id), listOf(entry), dolby, MediaFact.Known(0), true) - @Test fun reconcilesDolbyViewsAsOnePhysicalTrackWithoutLosingBaseGeometry() { + @Test fun dolbyAliasesKeepPhysicalTrackCountAndOriginalFallbackWithoutSelectingAView() { val iso = MediaFact.Known(listOf(isoTrack("vide", "hvc1", 1, true), isoTrack("soun", "mp4a", 2))) val views = listOf( MediaTrackMetadataReader.read(0, video("video/dolby-vision", null)).copy(containerIndex = MediaFact.Known(0)), @@ -220,8 +223,10 @@ class MediaMetadataReaderTests : AbsRobolectricTest() { MediaTrackMetadataReader.read(2, audio(2, "1190")).copy(containerIndex = MediaFact.Known(1))) val f = MediaContainerMetadataReader.facts(MediaFact.Known(ContainerKind.IsoBaseMedia), iso, views) assertEquals(MediaFact.Known(1), f.videoTrackCount) - assertEquals(MediaFact.Known(VideoCodec.Hevc), f.videoCodec) - assertEquals(MediaFact.Known(Dimensions(1920, 1080)), f.displayDimensions) + assertEquals(MediaFact.Unknown, f.videoCodec) + assertEquals(MediaFact.Unknown, f.displayDimensions) + assertEquals(StandardInputAction.Fallback(FallbackReason.UnsupportedHdr(DynamicRange.DolbyVision)), + StandardInputPlanner().plan(f).action) assertEquals(MediaFact.Known(DynamicRange.DolbyVision), f.dynamicRange) assertEquals(MediaFact.Known(listOf(AudioTrack(MediaFact.Known(AudioFormat.Aac(AudioChannelLayout.Stereo))))), f.audioTracks) } @@ -306,16 +311,11 @@ class MediaMetadataReaderTests : AbsRobolectricTest() { assertEquals(MediaFact.Known(2), MediaTrackMetadataReader.facts(tracks).videoTrackCount) } - @Test fun otherCodecUsesProvenAspectWithoutAssumingSquarePixels() { + @Test fun otherCodecWithoutSquarePixelEvidenceKeepsOriginalFallback() { val format = video("video/x-vnd.on2.vp9", null).apply { setInteger("color-transfer", 3) } - assertEquals(MediaFact.Unknown, facts(format, 24).displayDimensions) - val track = MediaTrackMetadataReader.read(0, format, 24, containerAspect = MediaFact.Known(1.0)) - val f = MediaTrackMetadataReader.facts(listOf(track)).copy(frameRate = MediaFact.Known(30.0)) - assertEquals(MediaFact.Known(Dimensions(1920, 1080)), f.displayDimensions) - assertTrue(StandardInputPlanner().plan(f, capabilities = fullCapabilities()).action is StandardInputAction.Convert) - format.setInteger("sar-width", 2); format.setInteger("sar-height", 1) - assertEquals(MediaFact.Unknown, MediaTrackMetadataReader.read(0, format, 24, - containerAspect = MediaFact.Known(1.0)).video!!.displayDimensions) + val f = facts(format, 24).copy(frameRate = MediaFact.Known(30.0)) + assertEquals(MediaFact.Unknown, f.displayDimensions) + assertTrue(StandardInputPlanner().plan(f, capabilities = fullCapabilities()).action is StandardInputAction.Fallback) } @Test fun predictedHevcReferenceSetsRetainMain10HlgEvidence() { @@ -341,14 +341,8 @@ class MediaMetadataReaderTests : AbsRobolectricTest() { assertNull(MediaMetadataInspector.trackCountFailure(64)) } - @Test fun predictedHevcChainsUseTheDerivedReferenceCountAfterZeroPocIsRemoved() { + @Test fun complexPredictedHevcChainsStayUnknown() { val csd = hex("00000001420101022000000300900000030000030078a003c0801107cad965654a4c08bdec780b509120904000000300400000078200") - assertEquals(MediaFact.Known(DynamicRange.Hlg), facts(video("video/hevc", csd)).dynamicRange) - assertEquals(MediaFact.Known(PixelFormat(10, ChromaSubsampling.Yuv420)), facts(video("video/hevc", csd)).pixelFormat) - } - - @Test fun predictedHevcSetsCannotGrowPastTheReferenceBound() { - val csd = hex("00000001420101022000000300900000030000030078a003c0801107cad965654a4c1847ffffffff047fffef016a12241208000003000800000300f040") assertEquals(CodecMetadata(), CodecMetadataReader.video(VideoCodec.Hevc, listOf(csd))) } } From de0c618e643a81ad8cfbadf30ca98e06a26a1b79 Mon Sep 17 00:00:00 2001 From: Eugene Morozov Date: Fri, 2 Oct 2026 15:42:12 -0700 Subject: [PATCH 07/49] Keep codec facts unknown when Media3 helpers are incompatible --- .../internal/standardization/CodecMetadata.kt | 17 +++++++------- .../MediaMetadataReaderTests.kt | 23 +++++++++++++++++++ 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/CodecMetadata.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/CodecMetadata.kt index c0e8480a..fd7538bb 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/CodecMetadata.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/CodecMetadata.kt @@ -25,20 +25,18 @@ internal object CodecMetadataReader { fun video(codec: VideoCodec?, configurations: List): CodecMetadata { if (configurations.sumOf { it.size.toLong() } > MAX_CONFIGURATION_BYTES) return CodecMetadata() - val results = configurations.flatMap(::annexBNalUnits).mapNotNull { nal -> - try { + return try { + val results = configurations.flatMap(::annexBNalUnits).mapNotNull { nal -> when { codec == VideoCodec.H264 && nal.isNotEmpty() && nal[0].toInt() and 31 == 7 -> avc(nal) codec == VideoCodec.Hevc && nal.size >= 2 && (nal[0].toInt() and 126) shr 1 == 33 -> hevc(nal) else -> null } - } catch (_: RuntimeException) { - // Truncated/unsupported initialization data never becomes compliant evidence. - CodecMetadata() } - } - // Several SPS descriptions may signal changing formats; do not choose an arbitrary one. - return results.distinct().singleOrNull() ?: CodecMetadata() + // Several SPS descriptions may signal changing formats; do not choose an arbitrary one. + results.distinct().singleOrNull() ?: CodecMetadata() + } catch (_: LinkageError) { CodecMetadata() } + catch (_: RuntimeException) { CodecMetadata() } } private fun avc(nal: ByteArray): CodecMetadata { @@ -200,7 +198,8 @@ internal object CodecMetadataReader { 6 -> AudioChannelLayout.FivePointOne else -> AudioChannelLayout.Other }) - } catch (_: Exception) { MediaFact.Unknown } + } catch (_: LinkageError) { MediaFact.Unknown } + catch (_: Exception) { MediaFact.Unknown } } } diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/MediaMetadataReaderTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/MediaMetadataReaderTests.kt index c1d6b08f..08ecb85c 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/MediaMetadataReaderTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/MediaMetadataReaderTests.kt @@ -1,6 +1,11 @@ package com.mux.video.upload.internal.standardization import android.media.MediaFormat +import androidx.media3.container.NalUnitUtil +import androidx.media3.extractor.AacUtil +import io.mockk.every +import io.mockk.mockkStatic +import io.mockk.unmockkStatic import com.mux.exoplayeradapter.AbsRobolectricTest import org.junit.Assert.* import org.junit.Test @@ -25,6 +30,24 @@ class MediaMetadataReaderTests : AbsRobolectricTest() { private fun facts(format: MediaFormat, api: Int = 23) = MediaTrackMetadataReader.facts(listOf(MediaTrackMetadataReader.read(0, format, api))) + @Test fun incompatibleMedia3CodecHelpersLeaveFactsUnknown() { + mockkStatic(NalUnitUtil::class) + mockkStatic(AacUtil::class) + try { + every { NalUnitUtil.parseSpsNalUnit(any(), any(), any()) } throws NoSuchMethodError() + every { NalUnitUtil.parseH265SpsNalUnit(any(), any(), any(), any()) } throws NoSuchMethodError() + every { AacUtil.parseAudioSpecificConfig(any()) } throws NoSuchMethodError() + assertEquals(CodecMetadata(), CodecMetadataReader.video(VideoCodec.H264, listOf(avc))) + assertEquals(CodecMetadata(), CodecMetadataReader.video(VideoCodec.Hevc, listOf(hevcHlg))) + assertEquals(MediaFact.Unknown, CodecMetadataReader.aacLayout(hex("1190"), 2)) + every { NalUnitUtil.findNalUnit(any(), any(), any(), any()) } throws NoSuchMethodError() + assertEquals(CodecMetadata(), CodecMetadataReader.video(VideoCodec.Hevc, listOf(hevcHlg))) + } finally { + unmockkStatic(NalUnitUtil::class) + unmockkStatic(AacUtil::class) + } + } + @Test fun readsAvcConfigurationAndGeometryOnApi23() { val f = facts(video()) assertEquals(MediaFact.Known(VideoProfile.H264Baseline), f.videoProfile) From 67b9c88431c153b781cde01400719d28bbaa4765 Mon Sep 17 00:00:00 2001 From: Eugene Morozov Date: Fri, 16 Oct 2026 11:21:30 -0700 Subject: [PATCH 08/49] Use platform metadata without duplicate codec or container parsing --- .../internal/standardization/CodecMetadata.kt | 218 ---------- .../standardization/IsoContainerMetadata.kt | 166 -------- .../standardization/MediaMetadataInspector.kt | 165 +++---- .../IsoContainerMetadataTests.kt | 125 ------ .../MediaMetadataReaderTests.kt | 403 +++++------------- 5 files changed, 160 insertions(+), 917 deletions(-) delete mode 100644 library/src/main/java/com/mux/video/upload/internal/standardization/CodecMetadata.kt delete mode 100644 library/src/main/java/com/mux/video/upload/internal/standardization/IsoContainerMetadata.kt delete mode 100644 library/src/test/java/com/mux/video/upload/internal/standardization/IsoContainerMetadataTests.kt diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/CodecMetadata.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/CodecMetadata.kt deleted file mode 100644 index fd7538bb..00000000 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/CodecMetadata.kt +++ /dev/null @@ -1,218 +0,0 @@ -package com.mux.video.upload.internal.standardization - -import androidx.media3.common.util.ParsableBitArray -import androidx.media3.container.NalUnitUtil -import androidx.media3.container.ParsableNalUnitBitArray -import androidx.media3.extractor.AacUtil - -/** Color codes use Media3/Android's normalized constants, not ISO code points. */ -internal data class ColorMetadata( - val standard: MediaFact = MediaFact.Unknown, - val transfer: MediaFact = MediaFact.Unknown, - val range: MediaFact = MediaFact.Unknown, -) - -internal data class CodecMetadata( - val profile: MediaFact = MediaFact.Unknown, - val pixelFormat: MediaFact = MediaFact.Unknown, - val color: ColorMetadata = ColorMetadata(), - val pixelAspectRatio: MediaFact = MediaFact.Unknown, -) - -/** Only codec initialization data is read here. No compressed media samples are inspected. */ -internal object CodecMetadataReader { - const val MAX_CONFIGURATION_BYTES = 64 * 1024 - - fun video(codec: VideoCodec?, configurations: List): CodecMetadata { - if (configurations.sumOf { it.size.toLong() } > MAX_CONFIGURATION_BYTES) return CodecMetadata() - return try { - val results = configurations.flatMap(::annexBNalUnits).mapNotNull { nal -> - when { - codec == VideoCodec.H264 && nal.isNotEmpty() && nal[0].toInt() and 31 == 7 -> avc(nal) - codec == VideoCodec.Hevc && nal.size >= 2 && (nal[0].toInt() and 126) shr 1 == 33 -> hevc(nal) - else -> null - } - } - // Several SPS descriptions may signal changing formats; do not choose an arbitrary one. - results.distinct().singleOrNull() ?: CodecMetadata() - } catch (_: LinkageError) { CodecMetadata() } - catch (_: RuntimeException) { CodecMetadata() } - } - - private fun avc(nal: ByteArray): CodecMetadata { - val sps = NalUnitUtil.parseSpsNalUnit(nal, 0, nal.size) - val bits = ParsableNalUnitBitArray(nal, 1, nal.size) - val profile = bits.readBits(8) - bits.skipBits(16) - bits.readUnsignedExpGolombCodedInt() // seq_parameter_set_id - val chroma = when (profile) { - 66, 77, 88 -> 1 - 100, 110, 122, 244, 44, 83, 86, 118, 128, 138 -> bits.readUnsignedExpGolombCodedInt() - else -> return CodecMetadata() - } - return CodecMetadata( - profile = MediaFact.Known(when (profile) { - 66 -> VideoProfile.H264Baseline - 77 -> VideoProfile.H264Main - 100 -> VideoProfile.H264High - else -> VideoProfile.Other - }), - pixelFormat = pixels(sps.bitDepthLumaMinus8, sps.bitDepthChromaMinus8, chroma), - color = color(sps.colorSpace, sps.colorTransfer, sps.colorRange), - pixelAspectRatio = positive(sps.pixelWidthHeightRatio.toDouble()), - ) - } - - private fun hevc(nal: ByteArray): CodecMetadata { - // Single-layer SPS only; multilayer configuration needs separate evidence. - if ((nal[0].toInt() and 1) != 0 || (nal[1].toInt() and 248) != 0) return CodecMetadata() - if (!boundedHevcSyntax(nal)) return CodecMetadata() - val sps = NalUnitUtil.parseH265SpsNalUnit(nal, 0, nal.size, null) - val profile = sps.profileTierLevel ?: return CodecMetadata() - return CodecMetadata( - profile = MediaFact.Known(when (if (profile.generalProfileSpace == 0) profile.generalProfileIdc else -1) { - 1 -> VideoProfile.HevcMain - 2 -> VideoProfile.HevcMain10 - else -> VideoProfile.Other - }), - pixelFormat = pixels(sps.bitDepthLumaMinus8, sps.bitDepthChromaMinus8, sps.chromaFormatIdc), - color = color(sps.colorSpace, sps.colorTransfer, sps.colorRange), - pixelAspectRatio = positive(sps.pixelWidthHeightRatio.toDouble()), - ) - } - - /** Bound allocation-driving SPS fields before calling the shared parser. */ - private fun boundedHevcSyntax(nal: ByteArray): Boolean { - val bits = ParsableNalUnitBitArray(nal, 2, nal.size) - bits.skipBits(4) - val layers = bits.readBits(3) - if (layers > 6) return false - bits.skipBit() - bits.skipBits(96) // general profile/tier/level - val profilePresent = BooleanArray(layers) - val levelPresent = BooleanArray(layers) - for (i in 0 until layers) { - profilePresent[i] = bits.readBit() - levelPresent[i] = bits.readBit() - } - if (layers > 0) bits.skipBits((8 - layers) * 2) - for (i in 0 until layers) { - if (profilePresent[i]) bits.skipBits(88) - if (levelPresent[i]) bits.skipBits(8) - } - if (bits.readUnsignedExpGolombCodedInt() !in 0..15) return false - val chroma = bits.readUnsignedExpGolombCodedInt() - if (chroma !in 0..3) return false - if (chroma == 3) bits.skipBit() - repeat(2) { if (bits.readUnsignedExpGolombCodedInt() !in 1..65535) return false } - if (bits.readBit()) repeat(4) { bits.readUnsignedExpGolombCodedInt() } - repeat(2) { if (bits.readUnsignedExpGolombCodedInt() !in 0..8) return false } - val pocBits = bits.readUnsignedExpGolombCodedInt() - if (pocBits !in 0..12) return false - val start = if (bits.readBit()) 0 else layers - for (i in start..layers) repeat(3) { bits.readUnsignedExpGolombCodedInt() } - repeat(6) { bits.readUnsignedExpGolombCodedInt() } - if (bits.readBit() && bits.readBit()) { - for (sizeId in 0..3) { - for (matrixId in 0..5 step if (sizeId == 3) 3 else 1) { - if (!bits.readBit()) bits.readUnsignedExpGolombCodedInt() else { - if (sizeId > 1) bits.readSignedExpGolombCodedInt() - repeat(minOf(64, 1 shl (4 + 2 * sizeId))) { bits.readSignedExpGolombCodedInt() } - } - } - } - } - bits.skipBits(2) - if (bits.readBit()) { - bits.skipBits(8) - repeat(2) { bits.readUnsignedExpGolombCodedInt() } - bits.skipBit() - } - val sets = bits.readUnsignedExpGolombCodedInt() - if (sets !in 0..64) return false - var previousCount = 0 - for (i in 0 until sets) { - if (i > 0 && bits.readBit()) { - // Media3 derives predicted sets. Only a final predicted set can be checked without - // duplicating those calculations; longer predicted chains remain unknown. - if (i != sets - 1) return false - bits.skipBit() - bits.readUnsignedExpGolombCodedInt() - repeat(previousCount + 1) { if (!bits.readBit()) bits.skipBit() } - } else { - val negative = bits.readUnsignedExpGolombCodedInt() - val positive = bits.readUnsignedExpGolombCodedInt() - if (negative !in 0..16 || positive !in 0..16 || negative + positive > 16) return false - previousCount = negative + positive - repeat(previousCount) { - bits.readUnsignedExpGolombCodedInt() - bits.skipBit() - } - } - } - if (bits.readBit()) { - val count = bits.readUnsignedExpGolombCodedInt() - if (count !in 0..32) return false - repeat(count) { bits.skipBits(pocBits + 5) } - } - return true - } - - private fun pixels(luma: Int, chroma: Int, subsampling: Int): MediaFact = - if (luma != chroma || luma !in 0..8 || subsampling !in 0..3) MediaFact.Unknown - else MediaFact.Known(PixelFormat(luma + 8, - if (subsampling == 1) ChromaSubsampling.Yuv420 else ChromaSubsampling.Other)) - - private fun color(standard: Int, transfer: Int, range: Int) = ColorMetadata( - positiveCode(standard), positiveCode(transfer), positiveCode(range)) - - /** MediaExtractor supplies AVC/HEVC CSD in Annex B form, with three- or four-byte prefixes. */ - private fun annexBNalUnits(data: ByteArray): List { - val units = mutableListOf() - val flags = BooleanArray(3) - var prefix = NalUnitUtil.findNalUnit(data, 0, data.size, flags) - while (prefix < data.size) { - val payload = prefix + 3 - NalUnitUtil.clearPrefixFlags(flags) - val next = NalUnitUtil.findNalUnit(data, payload, data.size, flags) - // A four-byte prefix includes a leading zero outside the three-byte start code. - val end = if (next > payload && next < data.size && data[next - 1] == 0.toByte()) next - 1 else next - if (payload < end) units.add(data.copyOfRange(payload, end)) - prefix = next - } - return units - } - - /** Channel count alone cannot prove a surround layout. PCE/extension configurations stay unknown. */ - fun aacLayout(configuration: ByteArray?, channelCount: Int?): MediaFact { - if (configuration == null || configuration.size !in 2..MAX_CONFIGURATION_BYTES) return MediaFact.Unknown - return try { - val objectType = ParsableBitArray(configuration).readBits(5) - if (objectType !in 1..4) return MediaFact.Unknown - val config = AacUtil.parseAudioSpecificConfig(configuration) - if (config.sampleRateHz <= 0 || (channelCount != null && config.channelCount != channelCount)) - return MediaFact.Unknown - MediaFact.Known(when (config.channelCount) { - 1 -> AudioChannelLayout.Mono - 2 -> AudioChannelLayout.Stereo - 6 -> AudioChannelLayout.FivePointOne - else -> AudioChannelLayout.Other - }) - } catch (_: LinkageError) { MediaFact.Unknown } - catch (_: Exception) { MediaFact.Unknown } - } -} - -internal fun positive(value: Double?): MediaFact = - if (value != null && value.isFinite() && value > 0) MediaFact.Known(value) else MediaFact.Unknown - -internal fun positiveCode(value: Int?): MediaFact = - if (value != null && value > 0) MediaFact.Known(value) else MediaFact.Unknown - -/** Missing evidence may be filled; contradictory evidence stays unknown. */ -internal fun reconcile(a: MediaFact, b: MediaFact): MediaFact = when { - a == MediaFact.Unknown -> b - b == MediaFact.Unknown -> a - a == b -> a - else -> MediaFact.Unknown -} diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/IsoContainerMetadata.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/IsoContainerMetadata.kt deleted file mode 100644 index dd78ff02..00000000 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/IsoContainerMetadata.kt +++ /dev/null @@ -1,166 +0,0 @@ -package com.mux.video.upload.internal.standardization - -import java.io.File -import java.io.RandomAccessFile -import java.nio.ByteBuffer - -internal data class IsoMetadataBox(val type: String, val payload: Long, val end: Long) - -internal object IsoSampleEntries { - val avc = setOf("avc1", "avc3", "dva1", "dvav") - val hevc = setOf("hvc1", "hev1", "dvh1", "dvhe") - val dolby = setOf("dvh1", "dvhe", "dva1", "dvav") - val visual = avc + hevc -} - -internal data class IsoTrackMetadata( - val handler: String?, - val trackId: MediaFact, - val sampleEntries: List, - val hasDolbyVisionConfiguration: Boolean, - val rotationDegrees: MediaFact, - val hasSimpleSampleGeometry: Boolean, -) { - val hasSimpleOrientation: Boolean get() = rotationDegrees != MediaFact.Unknown - fun matches(kind: TrackKind): Boolean = (handler == "vide" && kind == TrackKind.Video) || - (handler == "soun" && kind == TrackKind.Audio) -} - -/** Seeks past media payloads; reads track identity, simple orientation and Dolby Vision signaling. */ -internal object IsoContainerMetadataReader { - private const val MAX_BOXES = 4096 - private const val MAX_TOP_LEVEL_BOXES = 131072 - private const val MAX_ENTRIES = 16 - - fun read(file: File): MediaFact> = try { - RandomAccessFile(file, "r").use { input -> - val reader = Reader(input) - var movie: IsoMetadataBox? = null - for (box in reader.boxes(0, input.length(), topLevel = true)) { - if (box.type == "moov") { require(movie == null); movie = box } - } - val moov = movie ?: return MediaFact.Unknown - val children = reader.boxes(moov.payload, moov.end).toList() - val tracks = children.filter { it.type == "trak" } - if (tracks.size !in 1..64) return MediaFact.Unknown - val metadata = tracks.map { reader.track(it) } - val ids = metadata.mapNotNull { it.trackId.valueOrNull } - require(ids.distinct().size == ids.size) - MediaFact.Known(metadata) - } - } catch (_: Exception) { MediaFact.Unknown } - - private class Reader(val input: RandomAccessFile) { - var boxCount = 0 - val boxHeader = ByteArray(16) - - fun boxes(start: Long, end: Long, allowQuickTimeTerminator: Boolean = false, - topLevel: Boolean = false): Sequence = sequence { - require(start <= end && end <= input.length()) - var topCount = 0 - var position = start - while (position < end) { - if (allowQuickTimeTerminator && end - position == 4L) { - input.seek(position) - require(input.readInt() == 0) - break - } - require(end - position >= 8) - require(if (topLevel) ++topCount <= MAX_TOP_LEVEL_BOXES else ++boxCount <= MAX_BOXES) - input.seek(position) - input.readFully(boxHeader, 0, minOf(16L, end - position).toInt()) - val bytes = ByteBuffer.wrap(boxHeader) - val size32 = bytes.int.toLong() and 0xffffffffL - val type = String(boxHeader, 4, 4, Charsets.US_ASCII) - val header = if (size32 == 1L) 16 else 8 - require(end - position >= header) - val size = when (size32) { 0L -> end - position; 1L -> bytes.getLong(8); else -> size32 } - require(size >= header && size <= end - position) - yield(IsoMetadataBox(type, position + header, position + size)) - position += size - } - } - - fun track(track: IsoMetadataBox): IsoTrackMetadata { - val children = boxes(track.payload, track.end).toList() - val tkhd = children.singleOrNull { it.type == "tkhd" } - val mdia = children.single { it.type == "mdia" } - val media = boxes(mdia.payload, mdia.end).toList() - val handler = media.singleOrNull { it.type == "hdlr" }?.let { - require(it.end - it.payload >= 12) - input.seek(it.payload + 8) - fourCc() - } - val trackId = tkhd?.let { - input.seek(it.payload) - val version = input.readUnsignedByte() - val offset = when (version) { 0 -> 12; 1 -> 20; else -> -1 } - if (offset < 0 || it.end - it.payload < offset + 4) MediaFact.Unknown else { - input.seek(it.payload + offset) - positiveCode(input.readInt()) - } - } ?: MediaFact.Unknown - val orientation = tkhd?.let(::orientation) ?: MediaFact.Unknown - if (handler !in listOf("vide", "soun")) - return IsoTrackMetadata(handler, trackId, emptyList(), false, orientation, false) - val minf = media.single { it.type == "minf" } - val stbl = boxes(minf.payload, minf.end).single { it.type == "stbl" } - val tables = boxes(stbl.payload, stbl.end).toList() - val stsd = tables.single { it.type == "stsd" } - require(stsd.end - stsd.payload >= 8) - input.seek(stsd.payload + 4) - val count = input.readInt() - require(count in 1..MAX_ENTRIES) - val entries = boxes(stsd.payload + 8, stsd.end).toList() - require(entries.size == count) - var dolby = false - var simpleGeometry = entries.size == 1 - if (handler == "vide") { - for (entry in entries) { - if (entry.type in IsoSampleEntries.dolby) dolby = true - if (entry.type in IsoSampleEntries.visual) { - require(entry.end - entry.payload >= 78) - val extensions = boxes(entry.payload + 78, entry.end, allowQuickTimeTerminator = true).toList() - if (extensions.any { it.type in setOf("dvcC", "dvvC", "dvwC") }) dolby = true - if (extensions.any { it.type == "clap" }) simpleGeometry = false - val aspects = extensions.filter { it.type == "pasp" } - if (aspects.size > 1) simpleGeometry = false - for (aspect in aspects) { - require(aspect.end - aspect.payload == 8L) - input.seek(aspect.payload) - val horizontal = input.readInt() - val vertical = input.readInt() - // Only square pixels are supported; unusual geometry keeps original-file fallback. - if (horizontal == 0 || horizontal != vertical) simpleGeometry = false - } - } else simpleGeometry = false - } - } - return IsoTrackMetadata(handler, trackId, entries.map { it.type }, dolby, orientation, simpleGeometry) - } - - private fun orientation(box: IsoMetadataBox): MediaFact { - input.seek(box.payload) - val version = input.readUnsignedByte() - val offset = when (version) { 0 -> 40; 1 -> 52; else -> return MediaFact.Unknown } - require(box.end - box.payload >= offset + 36) - input.seek(box.payload + offset) - val matrix = IntArray(9) { input.readInt() } - if (matrix[2] != 0 || matrix[5] != 0 || matrix[8] != 0x40000000) return MediaFact.Unknown - // Same normalized transform convention as Swift. Android's reported angle is negated separately. - return when (listOf(matrix[0], matrix[1], matrix[3], matrix[4])) { - listOf(65536, 0, 0, 65536) -> MediaFact.Known(0) - listOf(0, 65536, -65536, 0) -> MediaFact.Known(270) - listOf(-65536, 0, 0, -65536) -> MediaFact.Known(180) - listOf(0, -65536, 65536, 0) -> MediaFact.Known(90) - else -> MediaFact.Unknown - } - } - - private fun fourCc(): String { - val bytes = ByteArray(4) - input.readFully(bytes) - return String(bytes, Charsets.US_ASCII) - } - } -} diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/MediaMetadataInspector.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/MediaMetadataInspector.kt index dcd0bb6b..74a55a75 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/MediaMetadataInspector.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/MediaMetadataInspector.kt @@ -1,12 +1,13 @@ package com.mux.video.upload.internal.standardization +import android.media.AudioFormat as PlatformAudioFormat +import android.media.MediaCodecInfo.CodecCapabilities import android.media.MediaCodecInfo.CodecProfileLevel import android.media.MediaExtractor import android.media.MediaFormat import android.os.Build import java.io.File import java.io.IOException -import java.nio.ByteBuffer internal enum class ContainerKind { IsoBaseMedia, Matroska, Other } internal enum class TrackKind { Video, Audio, Other, Unknown } @@ -17,6 +18,12 @@ internal sealed interface MetadataInspectionResult { data class Failure(val reason: MetadataFailure) : MetadataInspectionResult } +internal data class ColorMetadata( + val standard: MediaFact = MediaFact.Unknown, + val transfer: MediaFact = MediaFact.Unknown, + val range: MediaFact = MediaFact.Unknown, +) + internal data class VideoTrackMetadata( val codec: MediaFact, val profile: MediaFact, @@ -24,7 +31,7 @@ internal data class VideoTrackMetadata( val displayDimensions: MediaFact, val rotationDegrees: MediaFact, val reportedPlatformRotationDegrees: MediaFact, - val configuration: CodecMetadata, + val pixelFormat: MediaFact, val platformColor: ColorMetadata, val dynamicRange: MediaFact, ) @@ -40,6 +47,7 @@ internal data class AudioTrackMetadata( /** Reported metadata, not sample measurements or an effective presentation timeline. */ internal data class TrackMetadata( val extractorIndex: Int, + /** Track order reported by MediaExtractor; ISO timing identity is checked separately. */ val containerIndex: MediaFact, val sourceTrackId: MediaFact, val kind: TrackKind, @@ -55,7 +63,6 @@ internal data class TrackMetadata( internal data class MediaMetadataInspection( val container: MediaFact, val tracks: List, - val isoTracks: MediaFact>, val facts: MediaFacts, val elapsedNanos: Long, ) @@ -71,28 +78,12 @@ internal class MediaMetadataInspector { val container = container(file) extractor.setDataSource(file.absolutePath) trackCountFailure(extractor.trackCount)?.let { return MetadataInspectionResult.Failure(it) } - val iso = if (container == MediaFact.Known(ContainerKind.IsoBaseMedia)) - IsoContainerMetadataReader.read(file) else MediaFact.Unknown val tracks = (0 until extractor.trackCount).map { index -> - val format = extractor.getTrackFormat(index) - val containers = iso.valueOrNull - val orientation = if (format.string(MediaFormat.KEY_MIME)?.startsWith("video/") == true) - containers?.filter { it.handler == "vide" }?.singleOrNull() else null - val track = MediaTrackMetadataReader.read(index, format, - containerRotation = orientation?.rotationDegrees - ?: if (container == MediaFact.Known(ContainerKind.IsoBaseMedia)) MediaFact.Unknown else null) - val id = track.sourceTrackId.valueOrNull - val match = if (id != null) containers?.indexOfFirst { it.trackId.valueOrNull == id } else null - val containerIndex = when { - match != null && match >= 0 && containers!![match].matches(track.kind) -> MediaFact.Known(match) - id == null && containers?.size == extractor.trackCount && containers[index].matches(track.kind) -> MediaFact.Known(index) - else -> MediaFact.Unknown - } - track.copy(containerIndex = containerIndex) + MediaTrackMetadataReader.read(index, extractor.getTrackFormat(index)) } - val containerFacts = MediaContainerMetadataReader.facts(container, iso, tracks) + val facts = MediaTrackMetadataReader.facts(tracks) MetadataInspectionResult.Success(MediaMetadataInspection( - container, tracks, iso, containerFacts, System.nanoTime() - started)) + container, tracks, facts, System.nanoTime() - started)) } catch (_: IOException) { MetadataInspectionResult.Failure(MetadataFailure.Unreadable) } catch (_: RuntimeException) { @@ -125,51 +116,8 @@ internal class MediaMetadataInspector { } } -/** Container identities establish physical track count and audio order; ambiguous views stay unknown. */ -internal object MediaContainerMetadataReader { - fun facts(container: MediaFact, iso: MediaFact>, - tracks: List): MediaFacts { - val dolby = iso.valueOrNull?.any { it.hasDolbyVisionConfiguration } == true || - tracks.any { it.video?.dynamicRange == MediaFact.Known(DynamicRange.DolbyVision) } - fun fallback(): MediaFacts { - val facts = MediaTrackMetadataReader.facts(tracks, - audioOrderKnown = container != MediaFact.Known(ContainerKind.IsoBaseMedia)) - return if (dolby) facts.copy(dynamicRange = MediaFact.Known(DynamicRange.DolbyVision)) else facts - } - if (container != MediaFact.Known(ContainerKind.IsoBaseMedia)) return fallback() - val containers = iso.valueOrNull ?: return fallback() - if (containers.any { it.handler == null }) return fallback().copy(videoTrackCount = MediaFact.Unknown, audioTracks = MediaFact.Unknown) - val videoIndices = containers.indices.filter { containers[it].handler == "vide" } - val audioIndices = containers.indices.filter { containers[it].handler == "soun" } - val audioViews = tracks.filter { it.kind == TrackKind.Audio }.groupBy { it.containerIndex.valueOrNull } - val orderedAudio = if (audioViews.keys == audioIndices.toSet() && audioViews.values.all { it.size == 1 }) { - MediaFact.Known(audioIndices.map { index -> - val format = if (containers[index].sampleEntries.size == 1) audioViews[index]!!.single().audio!!.format - else MediaFact.Unknown - AudioTrack(format) - }) - } else MediaFact.Unknown - val base = MediaFacts(videoTrackCount = MediaFact.Known(videoIndices.size), audioTracks = orderedAudio, - dynamicRange = if (dolby) MediaFact.Known(DynamicRange.DolbyVision) else MediaFact.Unknown) - val index = videoIndices.singleOrNull() ?: return base - val description = containers[index] - if (description.sampleEntries.size != 1) return base - val views = tracks.filter { it.kind == TrackKind.Video } - if (views.any { it.containerIndex.valueOrNull != index }) return base - val selected = views.singleOrNull() ?: return base - val video = selected.video ?: return base - return video.toFacts(base).copy( - displayDimensions = if (description.hasSimpleOrientation && description.hasSimpleSampleGeometry) - video.displayDimensions else MediaFact.Unknown, - rotationDegrees = if (description.hasSimpleOrientation) video.rotationDegrees else MediaFact.Unknown, - dynamicRange = if (dolby) MediaFact.Known(DynamicRange.DolbyVision) else video.dynamicRange, - ) - } -} - internal object MediaTrackMetadataReader { - fun read(index: Int, format: MediaFormat, apiLevel: Int = Build.VERSION.SDK_INT, - containerRotation: MediaFact? = null): TrackMetadata { + fun read(index: Int, format: MediaFormat, apiLevel: Int = Build.VERSION.SDK_INT): TrackMetadata { val mime = format.string(MediaFormat.KEY_MIME) val kind = when { mime == null -> TrackKind.Unknown @@ -178,16 +126,16 @@ internal object MediaTrackMetadataReader { else -> TrackKind.Other } return TrackMetadata( - index, MediaFact.Unknown, positiveCode(format.int("track-id")), kind, mime?.let { MediaFact.Known(it) } ?: MediaFact.Unknown, + index, MediaFact.Known(index), positiveCode(format.int("track-id")), kind, mime?.let { MediaFact.Known(it) } ?: MediaFact.Unknown, positive(format.long(MediaFormat.KEY_DURATION)?.toDouble()?.div(1_000_000)), positive(format.number(MediaFormat.KEY_FRAME_RATE)), positiveCode(format.int(MediaFormat.KEY_BIT_RATE)), - video = if (kind == TrackKind.Video) video(format, mime, apiLevel, containerRotation) else null, + video = if (kind == TrackKind.Video) video(format, mime, apiLevel) else null, audio = if (kind == TrackKind.Audio) audio(format, mime) else null, ) } - fun facts(tracks: List, audioOrderKnown: Boolean = true): MediaFacts { + fun facts(tracks: List): MediaFacts { val dolby = tracks.any { it.video?.dynamicRange == MediaFact.Known(DynamicRange.DolbyVision) } if (tracks.any { it.kind == TrackKind.Unknown }) return MediaFacts( dynamicRange = if (dolby) MediaFact.Known(DynamicRange.DolbyVision) else MediaFact.Unknown) @@ -198,50 +146,54 @@ internal object MediaTrackMetadataReader { val ambiguousViews = dolby && videos.size > 1 && (ids.any { it == null } || ids.distinct().size != ids.size) val base = MediaFacts( videoTrackCount = if (ambiguousViews) MediaFact.Unknown else MediaFact.Known(videos.size), - audioTracks = if (audioOrderKnown || audio.size <= 1) MediaFact.Known(audio) else MediaFact.Unknown, + audioTracks = MediaFact.Known(audio), dynamicRange = if (dolby) MediaFact.Known(DynamicRange.DolbyVision) else MediaFact.Unknown, ) return videos.singleOrNull()?.video?.toFacts(base) ?: base } - private fun video(format: MediaFormat, mime: String?, apiLevel: Int, - containerRotation: MediaFact?): VideoTrackMetadata { + private fun video(format: MediaFormat, mime: String?, apiLevel: Int): VideoTrackMetadata { val codec = when (mime) { "video/avc" -> MediaFact.Known(VideoCodec.H264) "video/hevc" -> MediaFact.Known(VideoCodec.Hevc) null -> MediaFact.Unknown else -> MediaFact.Known(VideoCodec.Other) } - val csd = (0..2).mapNotNull { format.bytes("csd-$it") } - val config = CodecMetadataReader.video(codec.valueOrNull, csd) val encoded = dimensions(format.int(MediaFormat.KEY_WIDTH), format.int(MediaFormat.KEY_HEIGHT)) val platformRotation = format.int(MediaFormat.KEY_ROTATION)?.let { if (it % 90 == 0) MediaFact.Known((it % 360 + 360) % 360) else MediaFact.Unknown } ?: MediaFact.Unknown // Match the shared transform convention while retaining Android's raw angle as an observation. - val normalizedPlatform = platformRotation.valueOrNull?.let { MediaFact.Known((360 - it) % 360) } - ?: if (!format.containsKey(MediaFormat.KEY_ROTATION) && containerRotation == null) MediaFact.Known(0) else MediaFact.Unknown - val rotation = if (format.containsKey(MediaFormat.KEY_ROTATION) && platformRotation == MediaFact.Unknown) - MediaFact.Unknown else reconcile(containerRotation ?: MediaFact.Unknown, normalizedPlatform) + val rotation = platformRotation.valueOrNull?.let { MediaFact.Known((360 - it) % 360) } + ?: if (!format.containsKey(MediaFormat.KEY_ROTATION)) MediaFact.Known(0) else MediaFact.Unknown val platformColor = if (apiLevel >= 24) ColorMetadata( positiveCode(format.int(MediaFormat.KEY_COLOR_STANDARD)), positiveCode(format.int(MediaFormat.KEY_COLOR_TRANSFER)), positiveCode(format.int(MediaFormat.KEY_COLOR_RANGE)), ) else ColorMetadata() - val transfer = reconcile(platformColor.transfer, config.color.transfer) val range = if (mime == "video/dolby-vision") MediaFact.Known(DynamicRange.DolbyVision) - else dynamicRange(transfer) + else dynamicRange(platformColor.transfer) return VideoTrackMetadata(codec, - reconcile(platformProfile(codec.valueOrNull, format.int(MediaFormat.KEY_PROFILE)), config.profile), - encoded, display(format, encoded, rotation, config.pixelAspectRatio), rotation, platformRotation, - config, platformColor, range) + platformProfile(codec.valueOrNull, if (apiLevel >= 24) format.int(MediaFormat.KEY_PROFILE) else null), + encoded, display(format, encoded, rotation), rotation, platformRotation, + pixels(format.int(MediaFormat.KEY_COLOR_FORMAT), apiLevel), platformColor, range) } private fun audio(format: MediaFormat, mime: String?): AudioTrackMetadata { val count = positiveCode(format.int(MediaFormat.KEY_CHANNEL_COUNT)) val audio = when (mime) { - "audio/mp4a-latm" -> CodecMetadataReader.aacLayout(format.bytes("csd-0"), count.valueOrNull) - .valueOrNull?.let { MediaFact.Known(AudioFormat.Aac(it)) } ?: MediaFact.Unknown + "audio/mp4a-latm" -> { + val mask = format.int(MediaFormat.KEY_CHANNEL_MASK)?.takeIf { it != 0 } + val expected = when (count.valueOrNull) { + 1 -> setOf(PlatformAudioFormat.CHANNEL_OUT_MONO) + 2 -> setOf(PlatformAudioFormat.CHANNEL_OUT_STEREO) + 6 -> setOf(PlatformAudioFormat.CHANNEL_OUT_5POINT1) + else -> emptySet() + } + val layout = if ((mask != null && mask !in expected) || (count.valueOrNull == 6 && mask == null)) + MediaFact.Unknown else aacChannelLayout(count.valueOrNull) + layout.valueOrNull?.let { MediaFact.Known(AudioFormat.Aac(it)) } ?: MediaFact.Unknown + } null -> MediaFact.Unknown else -> MediaFact.Known(AudioFormat.OtherCodec) } @@ -249,8 +201,7 @@ internal object MediaTrackMetadataReader { nonNegative(format.int(MediaFormat.KEY_ENCODER_DELAY)), nonNegative(format.int(MediaFormat.KEY_ENCODER_PADDING))) } - private fun display(format: MediaFormat, encoded: MediaFact, rotation: MediaFact, - headerAspect: MediaFact): MediaFact { + private fun display(format: MediaFormat, encoded: MediaFact, rotation: MediaFact): MediaFact { val size = encoded.valueOrNull ?: return MediaFact.Unknown val degrees = rotation.valueOrNull ?: return MediaFact.Unknown val crop = listOf("crop-left", "crop-right", "crop-top", "crop-bottom").map { format.int(it) } @@ -263,14 +214,20 @@ internal object MediaTrackMetadataReader { } val sarWidth = format.int("sar-width") val sarHeight = format.int("sar-height") - val platformAspect = if (sarWidth != null && sarHeight != null && sarWidth > 0 && sarHeight > 0) - MediaFact.Known(sarWidth.toDouble() / sarHeight) else MediaFact.Unknown - if ((sarWidth != null || sarHeight != null) && platformAspect == MediaFact.Unknown) return MediaFact.Unknown - // Phone recordings use square pixels. Do not infer display geometry for unusual aspect ratios. - if (reconcile(platformAspect, headerAspect) != MediaFact.Known(1.0)) return MediaFact.Unknown + if ((sarWidth != null || sarHeight != null) && + (sarWidth == null || sarHeight == null || sarWidth <= 0 || sarWidth != sarHeight)) return MediaFact.Unknown return MediaFact.Known(if (degrees == 90 || degrees == 270) Dimensions(visible.height, visible.width) else visible) } + private fun pixels(color: Int?, apiLevel: Int): MediaFact = when (color) { + CodecCapabilities.COLOR_FormatYUV420Planar, CodecCapabilities.COLOR_FormatYUV420PackedPlanar, + CodecCapabilities.COLOR_FormatYUV420SemiPlanar, CodecCapabilities.COLOR_FormatYUV420PackedSemiPlanar, + CodecCapabilities.COLOR_FormatYUV420Flexible -> MediaFact.Known(PixelFormat(8, ChromaSubsampling.Yuv420)) + CodecCapabilities.COLOR_FormatYUVP010 -> if (apiLevel >= 33) + MediaFact.Known(PixelFormat(10, ChromaSubsampling.Yuv420)) else MediaFact.Unknown + else -> MediaFact.Unknown + } + private fun dimensions(width: Int?, height: Int?): MediaFact = if (width != null && height != null && width > 0 && height > 0) MediaFact.Known(Dimensions(width, height)) else MediaFact.Unknown @@ -305,19 +262,25 @@ private fun MediaFormat.long(key: String): Long? = try { if (containsKey(key)) g private fun MediaFormat.string(key: String): String? = try { if (containsKey(key)) getString(key) else null } catch (_: RuntimeException) { null } private fun MediaFormat.number(key: String): Double? = try { if (containsKey(key)) getFloat(key).toDouble() else null } catch (_: RuntimeException) { int(key)?.toDouble() } -private fun MediaFormat.bytes(key: String): ByteArray? { - return try { - val buffer: ByteBuffer = getByteBuffer(key)?.duplicate() ?: return null - if (buffer.remaining() !in 1..CodecMetadataReader.MAX_CONFIGURATION_BYTES) null - else ByteArray(buffer.remaining()).also { buffer.get(it) } - } catch (_: RuntimeException) { null } -} - private fun nonNegative(value: Int?): MediaFact = if (value != null && value >= 0) MediaFact.Known(value) else MediaFact.Unknown private fun VideoTrackMetadata.toFacts(base: MediaFacts): MediaFacts = base.copy( videoCodec = codec, videoProfile = profile, encodedDimensions = encodedDimensions, displayDimensions = displayDimensions, rotationDegrees = rotationDegrees, - pixelFormat = configuration.pixelFormat, dynamicRange = dynamicRange, + pixelFormat = pixelFormat, dynamicRange = dynamicRange, ) + +internal fun positive(value: Double?): MediaFact = + if (value != null && value.isFinite() && value > 0) MediaFact.Known(value) else MediaFact.Unknown + +internal fun positiveCode(value: Int?): MediaFact = + if (value != null && value > 0) MediaFact.Known(value) else MediaFact.Unknown + +internal fun aacChannelLayout(count: Int?): MediaFact = + if (count == null || count <= 0) MediaFact.Unknown else MediaFact.Known(when (count) { + 1 -> AudioChannelLayout.Mono + 2 -> AudioChannelLayout.Stereo + 6 -> AudioChannelLayout.FivePointOne + else -> AudioChannelLayout.Other + }) diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/IsoContainerMetadataTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/IsoContainerMetadataTests.kt deleted file mode 100644 index f231398f..00000000 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/IsoContainerMetadataTests.kt +++ /dev/null @@ -1,125 +0,0 @@ -package com.mux.video.upload.internal.standardization - -import com.mux.exoplayeradapter.AbsRobolectricTest -import org.robolectric.annotation.Config -import org.junit.Assert.* -import org.junit.Test -import java.io.File -import java.nio.ByteBuffer - -@Config(sdk = [23]) -class IsoContainerMetadataTests : AbsRobolectricTest() { - private fun int(value: Int) = ByteBuffer.allocate(4).putInt(value).array() - private fun box(type: String, payload: ByteArray) = int(payload.size + 8) + type.toByteArray() + payload - private fun track(handler: String = "vide", entry: String = "hvc1", extension: ByteArray = byteArrayOf(), - matrix: IntArray = intArrayOf(65536, 0, 0, 0, 65536, 0, 0, 0, 0x40000000)): ByteArray { - val tkhd = ByteArray(84) - for (i in matrix.indices) int(matrix[i]).copyInto(tkhd, 40 + i * 4) - val hdlr = ByteArray(8) + handler.toByteArray() - val description = ByteArray(if (handler == "vide") 78 else 28) - if (handler == "vide") { - ByteBuffer.wrap(description).putShort(24, 1920.toShort()).putShort(26, 1080.toShort()) - int(1920 * 65536).copyInto(tkhd, 76) - int(1080 * 65536).copyInto(tkhd, 80) - } - val sampleEntry = box(entry, description + extension) - val stsd = box("stsd", ByteArray(4) + int(1) + sampleEntry) - return box("trak", box("tkhd", tkhd) + - box("mdia", box("hdlr", hdlr) + box("minf", box("stbl", stsd)))) - } - - private fun read(data: ByteArray): MediaFact> { - val file = File.createTempFile("iso-metadata", ".mp4") - return try { file.writeBytes(data); IsoContainerMetadataReader.read(file) } finally { file.delete() } - } - - @Test fun readsContainerOrderAndDolbyConfigurationInOrdinaryHevcEntry() { - val tracks = read(box("ftyp", "isom".toByteArray()) + box("moov", - track(extension = box("dvvC", byteArrayOf(1, 0, 0, 0))) + track("soun", "mp4a"))).valueOrNull!! - assertEquals(listOf("vide", "soun"), tracks.map { it.handler }) - assertEquals(listOf("hvc1"), tracks[0].sampleEntries) - assertTrue(tracks[0].hasDolbyVisionConfiguration) - assertTrue(tracks[0].hasSimpleOrientation) - } - - @Test fun readsDolbySampleEntryEvenWithoutConfigurationChild() { - assertTrue(read(box("moov", track(entry = "dvh1"))).valueOrNull!![0].hasDolbyVisionConfiguration) - } - - @Test fun doesNotClaimSimpleGeometryForReflectionShearOrScale() { - for (matrix in listOf( - intArrayOf(-65536, 0, 0, 0, 65536, 0, 0, 0, 0x40000000), - intArrayOf(65536, 100, 0, 0, 65536, 0, 0, 0, 0x40000000), - intArrayOf(131072, 0, 0, 0, 65536, 0, 0, 0, 0x40000000))) { - assertFalse(read(box("moov", track(matrix = matrix))).valueOrNull!![0].hasSimpleOrientation) - } - } - - @Test fun recognizesQuarterRotationWithTranslation() { - val matrix = intArrayOf(0, 65536, 0, -65536, 0, 0, 1920 * 65536, 0, 0x40000000) - assertTrue(read(box("moov", track(matrix = matrix))).valueOrNull!![0].hasSimpleOrientation) - } - - @Test fun unsupportedGeometryStaysUnknownAndSquarePixelsAreAccepted() { - for (extension in listOf(box("clap", ByteArray(32)), box("pasp", int(2) + int(1)), - box("pasp", int(1) + int(1)) + box("pasp", int(1) + int(1)))) { - assertFalse(read(box("moov", track(extension = extension))).valueOrNull!![0].hasSimpleSampleGeometry) - } - assertTrue(read(box("moov", track(extension = box("pasp", int(1) + int(1))))) - .valueOrNull!![0].hasSimpleSampleGeometry) - } - - @Test fun malformedSizesAndBoxLimitsStayUnknown() { - for (data in listOf(byteArrayOf(), int(4096) + "moov".toByteArray(), - int(4) + "moov".toByteArray(), int(1) + "moov".toByteArray() + ByteArray(8) { -1 })) { - assertEquals(MediaFact.Unknown, read(data)) - } - val many = ByteArray(4097 * 8) - val free = box("free", byteArrayOf()) - for (i in 0..4096) free.copyInto(many, i * 8) - assertEquals(MediaFact.Unknown, read(many)) - } - - @Test fun permitsOnlyZeroQuickTimeSampleDescriptionTerminator() { - assertTrue(read(box("moov", track(extension = box("dvvC", ByteArray(4)) + ByteArray(4)))) - .valueOrNull!![0].hasDolbyVisionConfiguration) - assertEquals(MediaFact.Unknown, read(box("moov", track(extension = int(1))))) - } - - @Test fun noTrackOrDuplicateMovieBoxDoesNotInventAbsence() { - assertEquals(MediaFact.Unknown, read(box("moov", byteArrayOf()))) - assertEquals(MediaFact.Unknown, read(box("moov", track()) + box("moov", track()))) - } - - @Test fun everyUnitMatrixHasTheSharedNormalizedDegreeValue() { - for ((axes, degrees) in listOf( - listOf(65536, 0, 0, 65536) to 0, - listOf(0, 65536, -65536, 0) to 270, - listOf(-65536, 0, 0, -65536) to 180, - listOf(0, -65536, 65536, 0) to 90)) { - val matrix = intArrayOf(axes[0], axes[1], 0, axes[2], axes[3], 0, 1920 * 65536, 0, 0x40000000) - assertEquals(MediaFact.Known(degrees), read(box("moov", track(matrix = matrix))).valueOrNull!![0].rotationDegrees) - } - } - - @Test fun fragmentHeadersDoNotConsumeMovieMetadataBudget() { - val fragment = box("moof", byteArrayOf()) + box("mdat", byteArrayOf()) - val many = ByteArray(7200 * fragment.size) - repeat(7200) { fragment.copyInto(many, it * fragment.size) } - val movie = box("moov", track()) - assertNotEquals(MediaFact.Unknown, read(movie + many)) - assertNotEquals(MediaFact.Unknown, read(many + movie)) - assertEquals(MediaFact.Unknown, read(movie + many + movie)) - assertEquals(MediaFact.Unknown, read(movie + byteArrayOf(1))) - } - - @Test fun quickTimeLeadingBoxesAreRecognizedAsIso() { - for (type in listOf("skip", "uuid", "pnot")) { - val file = File.createTempFile("iso-sniff", ".mov") - try { - file.writeBytes(box(type, ByteArray(16)) + box("moov", track())) - assertEquals(MediaFact.Known(ContainerKind.IsoBaseMedia), MediaMetadataInspector().container(file)) - } finally { file.delete() } - } - } -} diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/MediaMetadataReaderTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/MediaMetadataReaderTests.kt index 08ecb85c..bdbbf1a4 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/MediaMetadataReaderTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/MediaMetadataReaderTests.kt @@ -1,11 +1,9 @@ package com.mux.video.upload.internal.standardization +import android.media.AudioFormat as PlatformAudioFormat +import android.media.MediaCodecInfo.CodecCapabilities +import android.media.MediaCodecInfo.CodecProfileLevel import android.media.MediaFormat -import androidx.media3.container.NalUnitUtil -import androidx.media3.extractor.AacUtil -import io.mockk.every -import io.mockk.mockkStatic -import io.mockk.unmockkStatic import com.mux.exoplayeradapter.AbsRobolectricTest import org.junit.Assert.* import org.junit.Test @@ -14,91 +12,76 @@ import java.nio.ByteBuffer @Config(sdk = [23]) class MediaMetadataReaderTests : AbsRobolectricTest() { - private fun hex(value: String) = value.chunked(2).map { it.toInt(16).toByte() }.toByteArray() - // SPS initialization bytes from checksum-verified shared synthetic fixtures, not media samples. - private val avc = hex("000000016742c028da01e0089f97016a02020280000003008000001e078c1950") - private val hevcSdr = hex("000001420101016000000300900000030000030078a003c0801107cb965654a4c2f016a020202080000003008000000f04") - private val hevcHlg = hex("00000001420101022000000300900000030000030078a003c0801107cad965654a4c2f016a12241208000003000800000300f040") - private val hevcPq = hex("00000001420101022000000300900000030000030078a003c0801107cad965654a4c2f016a12201208000003000800000300f040") - - private fun video(mime: String = "video/avc", csd: ByteArray? = avc, rotation: Int? = 0): MediaFormat = + private fun video(mime: String = "video/avc", rotation: Int? = 0) = MediaFormat.createVideoFormat(mime, 1920, 1080).apply { rotation?.let { setInteger(MediaFormat.KEY_ROTATION, it) } - csd?.let { setByteBuffer("csd-0", ByteBuffer.wrap(it)) } } - - private fun facts(format: MediaFormat, api: Int = 23) = + private fun audio(channels: Int, mask: Int? = null) = + MediaFormat.createAudioFormat("audio/mp4a-latm", 48000, channels).apply { + mask?.let { setInteger(MediaFormat.KEY_CHANNEL_MASK, it) } + } + private fun facts(format: MediaFormat, api: Int = 24) = MediaTrackMetadataReader.facts(listOf(MediaTrackMetadataReader.read(0, format, api))) - @Test fun incompatibleMedia3CodecHelpersLeaveFactsUnknown() { - mockkStatic(NalUnitUtil::class) - mockkStatic(AacUtil::class) - try { - every { NalUnitUtil.parseSpsNalUnit(any(), any(), any()) } throws NoSuchMethodError() - every { NalUnitUtil.parseH265SpsNalUnit(any(), any(), any(), any()) } throws NoSuchMethodError() - every { AacUtil.parseAudioSpecificConfig(any()) } throws NoSuchMethodError() - assertEquals(CodecMetadata(), CodecMetadataReader.video(VideoCodec.H264, listOf(avc))) - assertEquals(CodecMetadata(), CodecMetadataReader.video(VideoCodec.Hevc, listOf(hevcHlg))) - assertEquals(MediaFact.Unknown, CodecMetadataReader.aacLayout(hex("1190"), 2)) - every { NalUnitUtil.findNalUnit(any(), any(), any(), any()) } throws NoSuchMethodError() - assertEquals(CodecMetadata(), CodecMetadataReader.video(VideoCodec.Hevc, listOf(hevcHlg))) - } finally { - unmockkStatic(NalUnitUtil::class) - unmockkStatic(AacUtil::class) + @Test fun readsPlatformMetadataWithoutCodecInitializationData() { + val format = video().apply { + setInteger(MediaFormat.KEY_PROFILE, CodecProfileLevel.AVCProfileHigh) + setInteger(MediaFormat.KEY_COLOR_TRANSFER, MediaFormat.COLOR_TRANSFER_SDR_VIDEO) + setInteger(MediaFormat.KEY_COLOR_FORMAT, CodecCapabilities.COLOR_FormatYUV420Flexible) } - } - - @Test fun readsAvcConfigurationAndGeometryOnApi23() { - val f = facts(video()) - assertEquals(MediaFact.Known(VideoProfile.H264Baseline), f.videoProfile) + val f = facts(format) + assertEquals(MediaFact.Known(VideoProfile.H264High), f.videoProfile) assertEquals(MediaFact.Known(PixelFormat(8, ChromaSubsampling.Yuv420)), f.pixelFormat) assertEquals(MediaFact.Known(DynamicRange.Sdr), f.dynamicRange) assertEquals(MediaFact.Known(Dimensions(1920, 1080)), f.displayDimensions) - } - - @Test fun readsHevcMainMain10AndHdrFromConfigurationOnApi23() { - for ((csd, profile, depth, range) in listOf( - listOf(hevcSdr, VideoProfile.HevcMain, 8, DynamicRange.Sdr), - listOf(hevcHlg, VideoProfile.HevcMain10, 10, DynamicRange.Hlg), - listOf(hevcPq, VideoProfile.HevcMain10, 10, DynamicRange.Pq))) { - val f = facts(video("video/hevc", csd as ByteArray)) - assertEquals(MediaFact.Known(profile), f.videoProfile) - assertEquals(MediaFact.Known(PixelFormat(depth as Int, ChromaSubsampling.Yuv420)), f.pixelFormat) - assertEquals(MediaFact.Known(range), f.dynamicRange) + // Initialization bytes are not a second metadata source and are never parsed. + val bytes = ByteBuffer.wrap(byteArrayOf(0, 0, 1, 0x67)) + format.setByteBuffer("csd-0", bytes) + format.setByteBuffer("csd-1", ByteBuffer.allocate(128 * 1024)) + assertEquals(f, facts(format)) + assertEquals(0, bytes.position()) + } + + @Test fun readsHdrAndProfileFromPlatformProperties() { + for ((transfer, expected) in listOf(MediaFormat.COLOR_TRANSFER_HLG to DynamicRange.Hlg, + MediaFormat.COLOR_TRANSFER_ST2084 to DynamicRange.Pq)) { + val format = video("video/hevc").apply { + setInteger(MediaFormat.KEY_PROFILE, CodecProfileLevel.HEVCProfileMain10) + setInteger(MediaFormat.KEY_COLOR_TRANSFER, transfer) + } + assertEquals(MediaFact.Known(VideoProfile.HevcMain10), facts(format).videoProfile) + assertEquals(MediaFact.Known(expected), facts(format).dynamicRange) + assertEquals(MediaFact.Unknown, facts(format).pixelFormat) } } - @Test fun missingColorIsUnknownAndApi23IgnoresPlatformColor() { - val format = video(csd = null).apply { setInteger("color-transfer", 3) } - assertEquals(MediaFact.Unknown, facts(format).dynamicRange) - assertEquals(MediaFact.Known(DynamicRange.Sdr), facts(format, 24).dynamicRange) - format.setInteger("color-transfer", 0) - assertEquals(MediaFact.Unknown, facts(format, 24).dynamicRange) - } - - @Test fun conflictingColorAndProfileRemainUnknown() { - val format = video().apply { - setInteger("color-transfer", 7) - setInteger(MediaFormat.KEY_PROFILE, 8) // High contradicts the baseline SPS. + @Test fun api23AndMissingPlatformColorRemainUnknownRatherThanAssumingSdr() { + val format = video("video/hevc").apply { + setInteger(MediaFormat.KEY_PROFILE, CodecProfileLevel.HEVCProfileMain10) + setInteger(MediaFormat.KEY_COLOR_TRANSFER, MediaFormat.COLOR_TRANSFER_HLG) } - val f = facts(format, 24) - assertEquals(MediaFact.Unknown, f.dynamicRange) - assertEquals(MediaFact.Unknown, f.videoProfile) + assertEquals(MediaFact.Unknown, facts(format, 23).dynamicRange) + assertEquals(MediaFact.Unknown, facts(format, 23).videoProfile) + assertEquals(MediaFact.Unknown, facts(video()).dynamicRange) + format.setInteger(MediaFormat.KEY_COLOR_TRANSFER, 0) + assertEquals(MediaFact.Unknown, facts(format).dynamicRange) } - @Test fun dolbyVisionMimeOverridesBaseTransfer() { - val format = video("video/dolby-vision").apply { setInteger("color-transfer", 6) } - assertEquals(MediaFact.Known(DynamicRange.DolbyVision), facts(format, 24).dynamicRange) + @Test fun pixelFormatRequiresAnExplicitRecognizedPlatformValue() { + val format = video().apply { setInteger(MediaFormat.KEY_COLOR_FORMAT, CodecCapabilities.COLOR_FormatYUVP010) } + assertEquals(MediaFact.Known(PixelFormat(10, ChromaSubsampling.Yuv420)), facts(format, 33).pixelFormat) + assertEquals(MediaFact.Unknown, facts(format, 32).pixelFormat) + format.setInteger(MediaFormat.KEY_COLOR_FORMAT, CodecCapabilities.COLOR_FormatSurface) + assertEquals(MediaFact.Unknown, facts(format).pixelFormat) + assertEquals(MediaFact.Unknown, facts(video()).pixelFormat) } @Test fun metadataRateBitrateAndDurationDoNotBecomeSampleFacts() { - val format = video().apply { + val track = MediaTrackMetadataReader.read(0, video().apply { setFloat(MediaFormat.KEY_FRAME_RATE, 29.97f) setInteger(MediaFormat.KEY_BIT_RATE, 123456) setLong(MediaFormat.KEY_DURATION, 2000000) - setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1) - } - val track = MediaTrackMetadataReader.read(0, format) + }) assertEquals(29.97, track.reportedFrameRate.valueOrNull!!, 0.001) assertEquals(MediaFact.Known(123456), track.reportedBitrate) assertEquals(MediaFact.Known(2.0), track.reportedDurationSeconds) @@ -106,266 +89,72 @@ class MediaMetadataReaderTests : AbsRobolectricTest() { assertEquals(MediaFact.Unknown, f.frameRate) assertEquals(MediaFact.Unknown, f.averageBitrate) assertEquals(MediaFact.Unknown, f.durationSeconds) - assertEquals(MediaFact.Unknown, f.maximumKeyframeIntervalSeconds) + assertEquals(MediaFact.Unknown, f.audioVideoStartOffsetSeconds) assertEquals(MediaFact.Unknown, f.gopStructure) assertEquals(MediaFact.Unknown, f.editList) - assertEquals(MediaFact.Unknown, f.timestamps) } - @Test fun normalizesSignedRotationAndSwapsPortraitAxes() { - val format = video().apply { setInteger(MediaFormat.KEY_ROTATION, -90) } - val f = facts(format) - assertEquals(MediaFact.Known(90), f.rotationDegrees) - assertEquals(MediaFact.Known(Dimensions(1080, 1920)), f.displayDimensions) - } - - @Test fun absentRotationMeansZeroButNonQuarterRotationStaysUnknown() { - val missing = video(rotation = null) - assertEquals(MediaFact.Known(0), facts(missing).rotationDegrees) - assertEquals(MediaFact.Known(Dimensions(1920, 1080)), facts(missing).displayDimensions) - assertEquals(MediaFact.Unknown, MediaTrackMetadataReader.read(0, missing).video!!.reportedPlatformRotationDegrees) - val format = video().apply { setInteger(MediaFormat.KEY_ROTATION, 45) } - assertEquals(MediaFact.Unknown, facts(format).displayDimensions) + @Test fun platformRotationDeterminesDisplayAxesUnderSharedConvention() { + for ((raw, normalized) in listOf(0 to 0, 90 to 270, 180 to 180, 270 to 90, -90 to 90)) { + val v = MediaTrackMetadataReader.read(0, video(rotation = raw)).video!! + assertEquals(MediaFact.Known(raw.let { (it % 360 + 360) % 360 }), v.reportedPlatformRotationDegrees) + assertEquals(MediaFact.Known(normalized), v.rotationDegrees) + val expected = if (normalized == 90 || normalized == 270) Dimensions(1080, 1920) else Dimensions(1920, 1080) + assertEquals(MediaFact.Known(expected), v.displayDimensions) + } + assertEquals(MediaFact.Known(0), facts(video(rotation = null)).rotationDegrees) + assertEquals(MediaFact.Unknown, facts(video(rotation = 45)).displayDimensions) } - @Test fun appliesOrdinaryCropAndRejectsNonSquarePixels() { - val format = video(csd = null).apply { + @Test fun appliesPlatformCropAndRejectsIncompleteOrNonSquareGeometry() { + val format = video(rotation = 90).apply { setInteger("crop-left", 10); setInteger("crop-right", 1909) setInteger("crop-top", 0); setInteger("crop-bottom", 1079) setInteger("sar-width", 1); setInteger("sar-height", 1) - setInteger(MediaFormat.KEY_ROTATION, 90) } assertEquals(MediaFact.Known(Dimensions(1080, 1900)), facts(format).displayDimensions) format.setInteger("sar-width", 2) assertEquals(MediaFact.Unknown, facts(format).displayDimensions) - format.setInteger("sar-width", 1) - format.setInteger("crop-right", 1920) - assertEquals(MediaFact.Unknown, facts(format).displayDimensions) - } - - @Test fun conflictingAspectRatioIsUnknown() { - val format = video().apply { setInteger("sar-width", 2); setInteger("sar-height", 1) } - assertEquals(MediaFact.Unknown, facts(format).displayDimensions) - } - - @Test fun malformedAndOversizedCsdStayUnknown() { - for (data in listOf(hex("00000167"), ByteArray(CodecMetadataReader.MAX_CONFIGURATION_BYTES + 1))) { - val f = facts(video(csd = data)) - assertEquals(MediaFact.Unknown, f.pixelFormat) - assertEquals(MediaFact.Unknown, f.dynamicRange) - } - assertEquals(CodecMetadata(), CodecMetadataReader.video(VideoCodec.H264, listOf(avc, ByteArray(65536)))) - } - - @Test fun disagreeingSpsDescriptionsStayUnknownAndBuffersAreNotMutated() { - val format = video("video/hevc", hevcHlg) - format.setByteBuffer("csd-1", ByteBuffer.wrap(hevcPq)) - val before = format.getByteBuffer("csd-0")!!.position() - assertEquals(MediaFact.Unknown, facts(format).pixelFormat) - assertEquals(before, format.getByteBuffer("csd-0")!!.position()) - } - - private fun audio(channels: Int, config: String? = null) = - MediaFormat.createAudioFormat("audio/mp4a-latm", 48000, channels).apply { - config?.let { setByteBuffer("csd-0", ByteBuffer.wrap(hex(it))) } - } - - @Test fun aacLayoutRequiresConfigurationEvidence() { - for ((count, config, layout) in listOf( - Triple(1, "1188", AudioChannelLayout.Mono), - Triple(2, "1190", AudioChannelLayout.Stereo), - Triple(6, "11b0", AudioChannelLayout.FivePointOne))) { - val track = MediaTrackMetadataReader.read(1, audio(count, config)) - assertEquals(MediaFact.Known(AudioFormat.Aac(layout)), track.audio!!.format) - } - assertEquals(MediaFact.Unknown, MediaTrackMetadataReader.read(1, audio(6)).audio!!.format) - assertEquals(MediaFact.Unknown, MediaTrackMetadataReader.read(1, audio(6, "1190")).audio!!.format) - assertEquals(MediaFact.Unknown, MediaTrackMetadataReader.read(1, audio(6, "1180")).audio!!.format) + assertEquals(MediaFact.Unknown, facts(video().apply { setInteger("sar-width", 1) }).displayDimensions) + assertEquals(MediaFact.Unknown, facts(video().apply { setInteger("crop-left", 1) }).displayDimensions) } - @Test fun audioTracksRetainContainerOrderAndMultipleVideoDoesNotSelectOne() { - val tracks = listOf(MediaTrackMetadataReader.read(0, video()), - MediaTrackMetadataReader.read(1, audio(2, "1190")), MediaTrackMetadataReader.read(2, audio(1, "1188"))) - assertEquals(MediaFact.Known(listOf(AudioTrack(MediaFact.Known(AudioFormat.Aac(AudioChannelLayout.Stereo))), - AudioTrack(MediaFact.Known(AudioFormat.Aac(AudioChannelLayout.Mono))))), - MediaTrackMetadataReader.facts(tracks).audioTracks) - val f = MediaTrackMetadataReader.facts(tracks + MediaTrackMetadataReader.read(3, video("video/hevc", hevcSdr))) - assertEquals(MediaFact.Known(2), f.videoTrackCount) - assertEquals(MediaFact.Unknown, f.videoCodec) - assertEquals(MediaFact.Unknown, f.displayDimensions) - } - - @Test fun readsExplicitAacFrequencyAndRejectsZeroFrequency() { - assertEquals(MediaFact.Known(AudioChannelLayout.Stereo), - CodecMetadataReader.aacLayout(hex("17805dc010"), 2)) - assertEquals(MediaFact.Unknown, CodecMetadataReader.aacLayout(hex("1780000010"), 2)) - // AAC configuration is not NAL-escaped; this byte sequence must not lose its 0x03 byte. - assertEquals(MediaFact.Known(AudioChannelLayout.Stereo), - CodecMetadataReader.aacLayout(hex("1780000310"), 2)) - } - - @Test fun audioDelayAndPaddingRemainObservationsIncludingExplicitZero() { - val format = audio(2, "1190").apply { - setInteger(MediaFormat.KEY_ENCODER_DELAY, 1024) - setInteger(MediaFormat.KEY_ENCODER_PADDING, 0) - } - val track = MediaTrackMetadataReader.read(0, format) - assertEquals(MediaFact.Known(1024), track.audio!!.encoderDelaySamples) - assertEquals(MediaFact.Known(0), track.audio.encoderPaddingSamples) - assertEquals(MediaFact.Unknown, MediaTrackMetadataReader.facts(listOf(track)).audioVideoStartOffsetSeconds) - } - - @Test fun boundsAllocationDrivingHevcReferenceCounts() { - fun ue(value: Int): String { - val binary = (value + 1).toString(2) - return "0".repeat(binary.length - 1) + binary - } - val bits = "00000001" + "00000001" + "0".repeat(88) + ue(0) + ue(1) + - ue(1920) + ue(1080) + "0" + ue(0).repeat(3) + "0" + ue(0).repeat(9) + - "0000" + ue(1) + ue(100000000) + ue(0) - val raw = bits.padEnd((bits.length + 7) / 8 * 8, '0').chunked(8).map { it.toInt(2).toByte() } - val escaped = mutableListOf(0x42, 1) - var zeroCount = 0 - for (byte in raw) { - if (zeroCount == 2 && byte.toInt() and 255 <= 3) { escaped.add(3); zeroCount = 0 } - escaped.add(byte) - zeroCount = if (byte == 0.toByte()) zeroCount + 1 else 0 + @Test fun aacLayoutUsesChannelCountAndAvailableChannelMask() { + for ((channels, mask, expected) in listOf(Triple(1, null, AudioChannelLayout.Mono), + Triple(2, null, AudioChannelLayout.Stereo), Triple(6, PlatformAudioFormat.CHANNEL_OUT_5POINT1, AudioChannelLayout.FivePointOne))) { + assertEquals(MediaFact.Known(AudioFormat.Aac(expected)), MediaTrackMetadataReader.read(0, audio(channels, mask)).audio!!.format) } - val csd = byteArrayOf(0, 0, 0, 1) + escaped.toByteArray() - assertEquals(CodecMetadata(), CodecMetadataReader.video(VideoCodec.Hevc, listOf(csd))) - } - - private fun isoTrack(handler: String, entry: String, id: Int, dolby: Boolean = false) = - IsoTrackMetadata(handler, MediaFact.Known(id), listOf(entry), dolby, MediaFact.Known(0), true) - - @Test fun dolbyAliasesKeepPhysicalTrackCountAndOriginalFallbackWithoutSelectingAView() { - val iso = MediaFact.Known(listOf(isoTrack("vide", "hvc1", 1, true), isoTrack("soun", "mp4a", 2))) - val views = listOf( - MediaTrackMetadataReader.read(0, video("video/dolby-vision", null)).copy(containerIndex = MediaFact.Known(0)), - MediaTrackMetadataReader.read(1, video("video/hevc", hevcHlg)).copy(containerIndex = MediaFact.Known(0)), - MediaTrackMetadataReader.read(2, audio(2, "1190")).copy(containerIndex = MediaFact.Known(1))) - val f = MediaContainerMetadataReader.facts(MediaFact.Known(ContainerKind.IsoBaseMedia), iso, views) - assertEquals(MediaFact.Known(1), f.videoTrackCount) - assertEquals(MediaFact.Unknown, f.videoCodec) - assertEquals(MediaFact.Unknown, f.displayDimensions) - assertEquals(StandardInputAction.Fallback(FallbackReason.UnsupportedHdr(DynamicRange.DolbyVision)), - StandardInputPlanner().plan(f).action) + assertEquals(MediaFact.Unknown, MediaTrackMetadataReader.read(0, audio(6)).audio!!.format) + assertEquals(MediaFact.Unknown, MediaTrackMetadataReader.read(0, audio(2, PlatformAudioFormat.CHANNEL_OUT_MONO)).audio!!.format) + } + + @Test fun audioOrderUsesExtractorOrderAndRetainsTrackIds() { + val first = MediaTrackMetadataReader.read(1, audio(1).apply { setInteger("track-id", 3) }) + val second = MediaTrackMetadataReader.read(2, audio(2).apply { setInteger("track-id", 2) }) + val f = MediaTrackMetadataReader.facts(listOf(MediaTrackMetadataReader.read(0, video()), first, second)) + assertEquals(MediaFact.Known(3), first.sourceTrackId) + assertEquals(MediaFact.Known(1), first.containerIndex) + assertEquals(listOf(AudioChannelLayout.Mono, AudioChannelLayout.Stereo), f.audioTracks.valueOrNull!!.map { + (it.format.valueOrNull as AudioFormat.Aac).layout }) + assertEquals(MediaFact.Unknown, facts(MediaFormat()).videoTrackCount) + } + + @Test fun multipleOrAmbiguousDolbyViewsKeepOriginalFallback() { + val dolby = MediaTrackMetadataReader.read(0, video("video/dolby-vision").apply { + setInteger(MediaFormat.KEY_COLOR_TRANSFER, MediaFormat.COLOR_TRANSFER_ST2084) + }) + val f = MediaTrackMetadataReader.facts(listOf(dolby, MediaTrackMetadataReader.read(1, video("video/hevc")))) assertEquals(MediaFact.Known(DynamicRange.DolbyVision), f.dynamicRange) - assertEquals(MediaFact.Known(listOf(AudioTrack(MediaFact.Known(AudioFormat.Aac(AudioChannelLayout.Stereo))))), f.audioTracks) - } - - @Test fun normalizesAudioByContainerIdentityAndKeepsMissingTracksUnknown() { - val iso = MediaFact.Known(listOf(isoTrack("vide", "avc1", 1), isoTrack("soun", "mp4a", 2), isoTrack("soun", "mp4a", 3))) - val views = listOf(MediaTrackMetadataReader.read(0, video()).copy(containerIndex = MediaFact.Known(0)), - MediaTrackMetadataReader.read(1, audio(1, "1188")).copy(containerIndex = MediaFact.Known(2)), - MediaTrackMetadataReader.read(2, audio(2, "1190")).copy(containerIndex = MediaFact.Known(1))) - val f = MediaContainerMetadataReader.facts(MediaFact.Known(ContainerKind.IsoBaseMedia), iso, views) - assertEquals(listOf(AudioChannelLayout.Stereo, AudioChannelLayout.Mono), - f.audioTracks.valueOrNull!!.map { (it.format.valueOrNull as AudioFormat.Aac).layout }) - assertEquals(MediaFact.Unknown, MediaContainerMetadataReader.facts( - MediaFact.Known(ContainerKind.IsoBaseMedia), iso, views.dropLast(1)).audioTracks) - } - - @Test fun unknownMimePreventsClaimingCompleteTrackStructure() { - val f = MediaTrackMetadataReader.facts(listOf(MediaTrackMetadataReader.read(0, video()), - MediaTrackMetadataReader.read(1, MediaFormat()))) - assertEquals(MediaFacts(), f) - } - - @Test fun normalizesAndroidQuarterTurnsBeforeReconcilingContainerRotation() { - for ((raw, normalized) in listOf(0 to 0, 90 to 270, 180 to 180, 270 to 90)) { - val format = video().apply { setInteger(MediaFormat.KEY_ROTATION, raw) } - val track = MediaTrackMetadataReader.read(0, format, 23, MediaFact.Known(normalized)) - assertEquals(MediaFact.Known(raw), track.video!!.reportedPlatformRotationDegrees) - assertEquals(MediaFact.Known(normalized), track.video.rotationDegrees) - val conflict = MediaTrackMetadataReader.read(0, format, 23, MediaFact.Known((normalized + 180) % 360)) - assertEquals(MediaFact.Unknown, conflict.video!!.rotationDegrees) - assertEquals(MediaFact.Unknown, conflict.video.displayDimensions) - } - } - - @Test fun absentPlatformRotationCanUseContainerButInvalidRotationCannot() { - val format = MediaFormat.createVideoFormat("video/avc", 1920, 1080) - assertEquals(MediaFact.Known(270), MediaTrackMetadataReader.read(0, format, 23, MediaFact.Known(270)).video!!.rotationDegrees) - format.setInteger(MediaFormat.KEY_ROTATION, 45) - assertEquals(MediaFact.Unknown, MediaTrackMetadataReader.read(0, format, 23, MediaFact.Known(270)).video!!.rotationDegrees) - } - - @Test fun failedIsoReadRetainsResolutionButDoesNotGuessMultipleAudioOrder() { - for (rotation in listOf(0, null)) { - val tracks = listOf(MediaTrackMetadataReader.read(0, video(rotation = rotation), containerRotation = MediaFact.Unknown), - MediaTrackMetadataReader.read(1, audio(2, "1190")), MediaTrackMetadataReader.read(2, audio(1, "1188"))) - val f = MediaContainerMetadataReader.facts(MediaFact.Known(ContainerKind.IsoBaseMedia), MediaFact.Unknown, tracks) - assertEquals(MediaFact.Known(VideoCodec.H264), f.videoCodec) - assertEquals(MediaFact.Known(Dimensions(1920, 1080)), f.encodedDimensions) - assertEquals(if (rotation == null) MediaFact.Unknown else MediaFact.Known(0), f.rotationDegrees) - assertEquals(if (rotation == null) MediaFact.Unknown else MediaFact.Known(Dimensions(1920, 1080)), f.displayDimensions) - assertEquals(MediaFact.Unknown, f.audioTracks) - assertEquals(MediaFact.Unknown, f.durationSeconds) - } - } - - @Test fun matroskaWithoutRotationRetainsKnownDisplayDimensions() { - val track = MediaTrackMetadataReader.read(0, video(rotation = null)) - val f = MediaContainerMetadataReader.facts(MediaFact.Known(ContainerKind.Matroska), MediaFact.Unknown, listOf(track)) - assertEquals(MediaFact.Known(0), f.rotationDegrees) - assertEquals(MediaFact.Known(Dimensions(1920, 1080)), f.displayDimensions) - } - - @Test fun dolbyEvidenceSurvivesNonIsoMissingIsoUnknownHandlerAndUnknownMime() { - val dolby = MediaTrackMetadataReader.read(0, video("video/dolby-vision", null)) - val base = MediaTrackMetadataReader.read(1, video("video/hevc", hevcHlg)) - for (container in listOf(ContainerKind.Matroska, ContainerKind.Other, ContainerKind.IsoBaseMedia)) { - val f = MediaContainerMetadataReader.facts(MediaFact.Known(container), MediaFact.Unknown, listOf(dolby, base)) - assertEquals(MediaFact.Known(DynamicRange.DolbyVision), f.dynamicRange) - assertEquals(MediaFact.Unknown, f.videoTrackCount) - assertEquals(StandardInputAction.Fallback(FallbackReason.UnsupportedHdr(DynamicRange.DolbyVision)), StandardInputPlanner().plan(f).action) - } - val unknownHandler = MediaFact.Known(listOf(isoTrack("vide", "dvh1", 1, true).copy(handler = null))) - assertEquals(MediaFact.Known(DynamicRange.DolbyVision), MediaContainerMetadataReader.facts( - MediaFact.Known(ContainerKind.IsoBaseMedia), unknownHandler, listOf(dolby)).dynamicRange) - assertEquals(MediaFact.Known(DynamicRange.DolbyVision), MediaTrackMetadataReader.facts( - listOf(dolby, MediaTrackMetadataReader.read(1, MediaFormat()))).dynamicRange) - } - - @Test fun distinctDolbyTrackIdentitiesStillProveUnsafeMultipleTracks() { - val tracks = listOf(MediaTrackMetadataReader.read(0, video("video/dolby-vision", null)).copy(sourceTrackId = MediaFact.Known(1)), - MediaTrackMetadataReader.read(1, video("video/hevc", hevcHlg)).copy(sourceTrackId = MediaFact.Known(2))) - assertEquals(MediaFact.Known(2), MediaTrackMetadataReader.facts(tracks).videoTrackCount) - } - - @Test fun otherCodecWithoutSquarePixelEvidenceKeepsOriginalFallback() { - val format = video("video/x-vnd.on2.vp9", null).apply { setInteger("color-transfer", 3) } - val f = facts(format, 24).copy(frameRate = MediaFact.Known(30.0)) - assertEquals(MediaFact.Unknown, f.displayDimensions) - assertTrue(StandardInputPlanner().plan(f, capabilities = fullCapabilities()).action is StandardInputAction.Fallback) - } - - @Test fun predictedHevcReferenceSetsRetainMain10HlgEvidence() { - val nal = hex("420101022000000300900000030000030078a003c0801107cad965654a4c1af7780b509120904000000300400000078200") - val f = facts(video("video/hevc", byteArrayOf(0, 0, 0, 1) + nal)) - assertEquals(MediaFact.Known(VideoProfile.HevcMain10), f.videoProfile) - assertEquals(MediaFact.Known(PixelFormat(10, ChromaSubsampling.Yuv420)), f.pixelFormat) - assertEquals(MediaFact.Known(DynamicRange.Hlg), f.dynamicRange) - assertTrue(StandardInputPlanner().plan(f).action is StandardInputAction.UploadOriginal) - } - - @Test fun supportsMixedAnnexBPrefixesAndKeepsAacExtensionsUnknown() { - val mixed = avc + byteArrayOf(0, 0, 1, 0x68, 1) - assertEquals(CodecMetadataReader.video(VideoCodec.H264, listOf(avc)), CodecMetadataReader.video(VideoCodec.H264, listOf(mixed))) - for (config in listOf("2b9208", "eb9208", "1180", "11f0")) - assertEquals(MediaFact.Unknown, CodecMetadataReader.aacLayout(hex(config), 2)) + assertEquals(MediaFact.Unknown, f.videoTrackCount) + assertEquals(StandardInputAction.Fallback(FallbackReason.UnsupportedHdr(DynamicRange.DolbyVision)), StandardInputPlanner().plan(f).action) + val multiple = MediaTrackMetadataReader.facts(listOf(MediaTrackMetadataReader.read(0, video()), MediaTrackMetadataReader.read(1, video()))) + assertEquals(MediaFact.Known(2), multiple.videoTrackCount) + assertEquals(MediaFact.Unknown, multiple.videoCodec) } @Test fun emptyAndExcessiveTrackCountsHaveDifferentFailures() { assertEquals(MetadataFailure.Malformed, MediaMetadataInspector.trackCountFailure(0)) assertEquals(MetadataFailure.LimitExceeded, MediaMetadataInspector.trackCountFailure(65)) - assertNull(MediaMetadataInspector.trackCountFailure(1)) assertNull(MediaMetadataInspector.trackCountFailure(64)) } - - @Test fun complexPredictedHevcChainsStayUnknown() { - val csd = hex("00000001420101022000000300900000030000030078a003c0801107cad965654a4c08bdec780b509120904000000300400000078200") - assertEquals(CodecMetadata(), CodecMetadataReader.video(VideoCodec.Hevc, listOf(csd))) - } } From e61e2fe3c77aebef4eda0194011453f33b32fff3 Mon Sep 17 00:00:00 2001 From: Eugene Morozov Date: Fri, 16 Oct 2026 11:26:00 -0700 Subject: [PATCH 09/49] Retain missing pixel and color evidence without reconciling platform values --- .../internal/standardization/CodecMetadata.kt | 161 ++++++++++++++++++ .../standardization/MediaMetadataInspector.kt | 21 ++- .../MediaMetadataReaderTests.kt | 54 +++++- 3 files changed, 232 insertions(+), 4 deletions(-) create mode 100644 library/src/main/java/com/mux/video/upload/internal/standardization/CodecMetadata.kt diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/CodecMetadata.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/CodecMetadata.kt new file mode 100644 index 00000000..d31cc9c9 --- /dev/null +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/CodecMetadata.kt @@ -0,0 +1,161 @@ +package com.mux.video.upload.internal.standardization + +import androidx.media3.container.NalUnitUtil +import androidx.media3.container.ParsableNalUnitBitArray + +internal data class CodecMetadata( + val pixelFormat: MediaFact = MediaFact.Unknown, + val color: ColorMetadata = ColorMetadata(), +) + +/** Fills missing pixel/color facts only; platform metadata is authoritative when present. */ +internal object CodecMetadataReader { + const val MAX_CONFIGURATION_BYTES = 64 * 1024 + + fun video(codec: VideoCodec?, configurations: List): CodecMetadata { + if (configurations.sumOf { it.size.toLong() } > MAX_CONFIGURATION_BYTES) return CodecMetadata() + return try { + val results = configurations.flatMap(::annexBNalUnits).mapNotNull { nal -> + when { + codec == VideoCodec.H264 && nal.isNotEmpty() && nal[0].toInt() and 31 == 7 -> avc(nal) + codec == VideoCodec.Hevc && nal.size >= 2 && (nal[0].toInt() and 126) shr 1 == 33 -> hevc(nal) + else -> null + } + } + // Several SPS descriptions may signal changing formats; do not choose an arbitrary one. + results.distinct().singleOrNull() ?: CodecMetadata() + } catch (_: LinkageError) { CodecMetadata() } + catch (_: RuntimeException) { CodecMetadata() } + } + + private fun avc(nal: ByteArray): CodecMetadata { + val sps = NalUnitUtil.parseSpsNalUnit(nal, 0, nal.size) + val bits = ParsableNalUnitBitArray(nal, 1, nal.size) + val profile = bits.readBits(8) + bits.skipBits(16) + bits.readUnsignedExpGolombCodedInt() // seq_parameter_set_id + val chroma = when (profile) { + 66, 77, 88 -> 1 + 100, 110, 122, 244, 44, 83, 86, 118, 128, 138 -> bits.readUnsignedExpGolombCodedInt() + else -> return CodecMetadata() + } + return CodecMetadata( + pixelFormat = pixels(sps.bitDepthLumaMinus8, sps.bitDepthChromaMinus8, chroma), + color = color(sps.colorSpace, sps.colorTransfer, sps.colorRange), + ) + } + + private fun hevc(nal: ByteArray): CodecMetadata { + // Single-layer SPS only; multilayer configuration needs separate evidence. + if ((nal[0].toInt() and 1) != 0 || (nal[1].toInt() and 248) != 0) return CodecMetadata() + if (!boundedHevcSyntax(nal)) return CodecMetadata() + val sps = NalUnitUtil.parseH265SpsNalUnit(nal, 0, nal.size, null) + return CodecMetadata( + pixelFormat = pixels(sps.bitDepthLumaMinus8, sps.bitDepthChromaMinus8, sps.chromaFormatIdc), + color = color(sps.colorSpace, sps.colorTransfer, sps.colorRange), + ) + } + + /** Bound allocation-driving SPS fields before calling the shared parser. */ + private fun boundedHevcSyntax(nal: ByteArray): Boolean { + val bits = ParsableNalUnitBitArray(nal, 2, nal.size) + bits.skipBits(4) + val layers = bits.readBits(3) + if (layers > 6) return false + bits.skipBit() + bits.skipBits(96) // general profile/tier/level + val profilePresent = BooleanArray(layers) + val levelPresent = BooleanArray(layers) + for (i in 0 until layers) { + profilePresent[i] = bits.readBit() + levelPresent[i] = bits.readBit() + } + if (layers > 0) bits.skipBits((8 - layers) * 2) + for (i in 0 until layers) { + if (profilePresent[i]) bits.skipBits(88) + if (levelPresent[i]) bits.skipBits(8) + } + if (bits.readUnsignedExpGolombCodedInt() !in 0..15) return false + val chroma = bits.readUnsignedExpGolombCodedInt() + if (chroma !in 0..3) return false + if (chroma == 3) bits.skipBit() + repeat(2) { if (bits.readUnsignedExpGolombCodedInt() !in 1..65535) return false } + if (bits.readBit()) repeat(4) { bits.readUnsignedExpGolombCodedInt() } + repeat(2) { if (bits.readUnsignedExpGolombCodedInt() !in 0..8) return false } + val pocBits = bits.readUnsignedExpGolombCodedInt() + if (pocBits !in 0..12) return false + val start = if (bits.readBit()) 0 else layers + for (i in start..layers) repeat(3) { bits.readUnsignedExpGolombCodedInt() } + repeat(6) { bits.readUnsignedExpGolombCodedInt() } + if (bits.readBit() && bits.readBit()) { + for (sizeId in 0..3) { + for (matrixId in 0..5 step if (sizeId == 3) 3 else 1) { + if (!bits.readBit()) bits.readUnsignedExpGolombCodedInt() else { + if (sizeId > 1) bits.readSignedExpGolombCodedInt() + repeat(minOf(64, 1 shl (4 + 2 * sizeId))) { bits.readSignedExpGolombCodedInt() } + } + } + } + } + bits.skipBits(2) + if (bits.readBit()) { + bits.skipBits(8) + repeat(2) { bits.readUnsignedExpGolombCodedInt() } + bits.skipBit() + } + val sets = bits.readUnsignedExpGolombCodedInt() + if (sets !in 0..64) return false + var previousCount = 0 + for (i in 0 until sets) { + if (i > 0 && bits.readBit()) { + // Media3 derives predicted sets. Only a final predicted set can be checked without + // duplicating those calculations; longer predicted chains remain unknown. + if (i != sets - 1) return false + bits.skipBit() + bits.readUnsignedExpGolombCodedInt() + repeat(previousCount + 1) { if (!bits.readBit()) bits.skipBit() } + } else { + val negative = bits.readUnsignedExpGolombCodedInt() + val positive = bits.readUnsignedExpGolombCodedInt() + if (negative !in 0..16 || positive !in 0..16 || negative + positive > 16) return false + previousCount = negative + positive + repeat(previousCount) { + bits.readUnsignedExpGolombCodedInt() + bits.skipBit() + } + } + } + if (bits.readBit()) { + val count = bits.readUnsignedExpGolombCodedInt() + if (count !in 0..32) return false + repeat(count) { bits.skipBits(pocBits + 5) } + } + return true + } + + private fun pixels(luma: Int, chroma: Int, subsampling: Int): MediaFact = + if (luma != chroma || luma !in 0..8 || subsampling !in 0..3) MediaFact.Unknown + else MediaFact.Known(PixelFormat(luma + 8, + if (subsampling == 1) ChromaSubsampling.Yuv420 else ChromaSubsampling.Other)) + + private fun color(standard: Int, transfer: Int, range: Int) = ColorMetadata( + positiveCode(standard), positiveCode(transfer), positiveCode(range)) + + /** MediaExtractor supplies AVC/HEVC CSD in Annex B form, with three- or four-byte prefixes. */ + private fun annexBNalUnits(data: ByteArray): List { + val units = mutableListOf() + val flags = BooleanArray(3) + var prefix = NalUnitUtil.findNalUnit(data, 0, data.size, flags) + while (prefix < data.size) { + val payload = prefix + 3 + NalUnitUtil.clearPrefixFlags(flags) + val next = NalUnitUtil.findNalUnit(data, payload, data.size, flags) + // A four-byte prefix includes a leading zero outside the three-byte start code. + val end = if (next > payload && next < data.size && data[next - 1] == 0.toByte()) next - 1 else next + if (payload < end) units.add(data.copyOfRange(payload, end)) + prefix = next + } + return units + } + +} diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/MediaMetadataInspector.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/MediaMetadataInspector.kt index 74a55a75..d10aa766 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/MediaMetadataInspector.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/MediaMetadataInspector.kt @@ -8,6 +8,7 @@ import android.media.MediaFormat import android.os.Build import java.io.File import java.io.IOException +import java.nio.ByteBuffer internal enum class ContainerKind { IsoBaseMedia, Matroska, Other } internal enum class TrackKind { Video, Audio, Other, Unknown } @@ -32,7 +33,7 @@ internal data class VideoTrackMetadata( val rotationDegrees: MediaFact, val reportedPlatformRotationDegrees: MediaFact, val pixelFormat: MediaFact, - val platformColor: ColorMetadata, + val color: ColorMetadata, val dynamicRange: MediaFact, ) @@ -171,12 +172,20 @@ internal object MediaTrackMetadataReader { positiveCode(format.int(MediaFormat.KEY_COLOR_TRANSFER)), positiveCode(format.int(MediaFormat.KEY_COLOR_RANGE)), ) else ColorMetadata() + val platformPixels = pixels(format.int(MediaFormat.KEY_COLOR_FORMAT), apiLevel) + val fallback = if (platformPixels == MediaFact.Unknown || platformColor.transfer == MediaFact.Unknown) + CodecMetadataReader.video(codec.valueOrNull, (0..2).mapNotNull { format.bytes("csd-$it") }) else CodecMetadata() + val color = ColorMetadata( + platformColor.standard.takeUnless { it == MediaFact.Unknown } ?: fallback.color.standard, + platformColor.transfer.takeUnless { it == MediaFact.Unknown } ?: fallback.color.transfer, + platformColor.range.takeUnless { it == MediaFact.Unknown } ?: fallback.color.range, + ) val range = if (mime == "video/dolby-vision") MediaFact.Known(DynamicRange.DolbyVision) - else dynamicRange(platformColor.transfer) + else dynamicRange(color.transfer) return VideoTrackMetadata(codec, platformProfile(codec.valueOrNull, if (apiLevel >= 24) format.int(MediaFormat.KEY_PROFILE) else null), encoded, display(format, encoded, rotation), rotation, platformRotation, - pixels(format.int(MediaFormat.KEY_COLOR_FORMAT), apiLevel), platformColor, range) + platformPixels.takeUnless { it == MediaFact.Unknown } ?: fallback.pixelFormat, color, range) } private fun audio(format: MediaFormat, mime: String?): AudioTrackMetadata { @@ -262,6 +271,12 @@ private fun MediaFormat.long(key: String): Long? = try { if (containsKey(key)) g private fun MediaFormat.string(key: String): String? = try { if (containsKey(key)) getString(key) else null } catch (_: RuntimeException) { null } private fun MediaFormat.number(key: String): Double? = try { if (containsKey(key)) getFloat(key).toDouble() else null } catch (_: RuntimeException) { int(key)?.toDouble() } +private fun MediaFormat.bytes(key: String): ByteArray? = try { + val buffer: ByteBuffer? = getByteBuffer(key)?.duplicate() + if (buffer == null || buffer.remaining() !in 1..CodecMetadataReader.MAX_CONFIGURATION_BYTES) null + else ByteArray(buffer.remaining()).also { buffer.get(it) } +} catch (_: RuntimeException) { null } + private fun nonNegative(value: Int?): MediaFact = if (value != null && value >= 0) MediaFact.Known(value) else MediaFact.Unknown diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/MediaMetadataReaderTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/MediaMetadataReaderTests.kt index bdbbf1a4..6fdd8bae 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/MediaMetadataReaderTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/MediaMetadataReaderTests.kt @@ -4,6 +4,10 @@ import android.media.AudioFormat as PlatformAudioFormat import android.media.MediaCodecInfo.CodecCapabilities import android.media.MediaCodecInfo.CodecProfileLevel import android.media.MediaFormat +import androidx.media3.container.NalUnitUtil +import io.mockk.every +import io.mockk.mockkStatic +import io.mockk.unmockkStatic import com.mux.exoplayeradapter.AbsRobolectricTest import org.junit.Assert.* import org.junit.Test @@ -42,6 +46,34 @@ class MediaMetadataReaderTests : AbsRobolectricTest() { assertEquals(0, bytes.position()) } + @Test fun missingPixelAndColorFactsUseHeadersWithoutOverridingPlatformValues() { + val avc = "000000016742c028da01e0089f97016a02020280000003008000001e078c1950" + .chunked(2).map { it.toInt(16).toByte() }.toByteArray() + val format = video().apply { setByteBuffer("csd-0", ByteBuffer.wrap(avc)) } + assertEquals(MediaFact.Known(PixelFormat(8, ChromaSubsampling.Yuv420)), facts(format).pixelFormat) + assertEquals(MediaFact.Known(DynamicRange.Sdr), facts(format, 23).dynamicRange) + format.setInteger(MediaFormat.KEY_COLOR_TRANSFER, MediaFormat.COLOR_TRANSFER_HLG) + format.setInteger(MediaFormat.KEY_PROFILE, CodecProfileLevel.AVCProfileHigh) + format.setInteger(MediaFormat.KEY_COLOR_FORMAT, CodecCapabilities.COLOR_FormatYUVP010) + assertEquals(MediaFact.Known(DynamicRange.Hlg), facts(format, 33).dynamicRange) + assertEquals(MediaFact.Known(VideoProfile.H264High), facts(format, 33).videoProfile) + assertEquals(MediaFact.Known(PixelFormat(10, ChromaSubsampling.Yuv420)), facts(format, 33).pixelFormat) + } + + @Test fun missingFactFallbackSurvivesIncompatibleMedia3() { + mockkStatic(NalUnitUtil::class) + try { + every { NalUnitUtil.findNalUnit(any(), any(), any(), any()) } throws NoSuchMethodError() + val f = facts(video().apply { + setByteBuffer("csd-0", ByteBuffer.wrap(byteArrayOf(0, 0, 1, 0x67))) + setInteger(MediaFormat.KEY_PROFILE, CodecProfileLevel.AVCProfileHigh) + }) + assertEquals(MediaFact.Unknown, f.pixelFormat) + assertEquals(MediaFact.Unknown, f.dynamicRange) + assertEquals(MediaFact.Known(VideoProfile.H264High), f.videoProfile) + } finally { unmockkStatic(NalUnitUtil::class) } + } + @Test fun readsHdrAndProfileFromPlatformProperties() { for ((transfer, expected) in listOf(MediaFormat.COLOR_TRANSFER_HLG to DynamicRange.Hlg, MediaFormat.COLOR_TRANSFER_ST2084 to DynamicRange.Pq)) { @@ -55,7 +87,7 @@ class MediaMetadataReaderTests : AbsRobolectricTest() { } } - @Test fun api23AndMissingPlatformColorRemainUnknownRatherThanAssumingSdr() { + @Test fun api23AndMissingPlatformColorRemainUnknownWithoutHeaderEvidence() { val format = video("video/hevc").apply { setInteger(MediaFormat.KEY_PROFILE, CodecProfileLevel.HEVCProfileMain10) setInteger(MediaFormat.KEY_COLOR_TRANSFER, MediaFormat.COLOR_TRANSFER_HLG) @@ -157,4 +189,24 @@ class MediaMetadataReaderTests : AbsRobolectricTest() { assertEquals(MetadataFailure.LimitExceeded, MediaMetadataInspector.trackCountFailure(65)) assertNull(MediaMetadataInspector.trackCountFailure(64)) } + @Test fun boundsAllocationDrivingHevcReferenceCounts() { + fun ue(value: Int): String { + val binary = (value + 1).toString(2) + return "0".repeat(binary.length - 1) + binary + } + val bits = "00000001" + "00000001" + "0".repeat(88) + ue(0) + ue(1) + + ue(1920) + ue(1080) + "0" + ue(0).repeat(3) + "0" + ue(0).repeat(9) + + "0000" + ue(1) + ue(100000000) + ue(0) + val raw = bits.padEnd((bits.length + 7) / 8 * 8, '0').chunked(8).map { it.toInt(2).toByte() } + val escaped = mutableListOf(0x42, 1) + var zeroCount = 0 + for (byte in raw) { + if (zeroCount == 2 && byte.toInt() and 255 <= 3) { escaped.add(3); zeroCount = 0 } + escaped.add(byte) + zeroCount = if (byte == 0.toByte()) zeroCount + 1 else 0 + } + val csd = byteArrayOf(0, 0, 0, 1) + escaped.toByteArray() + assertEquals(CodecMetadata(), CodecMetadataReader.video(VideoCodec.Hevc, listOf(csd))) + } + } From f541f79ed7472371069c9779652ab041536e33eb Mon Sep 17 00:00:00 2001 From: Eugene Morozov Date: Fri, 16 Oct 2026 11:31:50 -0700 Subject: [PATCH 10/49] Fill missing pre-API-24 profile evidence without overriding platform metadata --- .../internal/standardization/CodecMetadata.kt | 15 ++++++++++++++- .../standardization/MediaMetadataInspector.kt | 5 +++-- .../standardization/MediaMetadataReaderTests.kt | 1 + 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/CodecMetadata.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/CodecMetadata.kt index d31cc9c9..aad35580 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/CodecMetadata.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/CodecMetadata.kt @@ -4,11 +4,12 @@ import androidx.media3.container.NalUnitUtil import androidx.media3.container.ParsableNalUnitBitArray internal data class CodecMetadata( + val profile: MediaFact = MediaFact.Unknown, val pixelFormat: MediaFact = MediaFact.Unknown, val color: ColorMetadata = ColorMetadata(), ) -/** Fills missing pixel/color facts only; platform metadata is authoritative when present. */ +/** Fills missing codec facts only; platform metadata is authoritative when present. */ internal object CodecMetadataReader { const val MAX_CONFIGURATION_BYTES = 64 * 1024 @@ -40,6 +41,12 @@ internal object CodecMetadataReader { else -> return CodecMetadata() } return CodecMetadata( + profile = MediaFact.Known(when (profile) { + 66 -> VideoProfile.H264Baseline + 77 -> VideoProfile.H264Main + 100 -> VideoProfile.H264High + else -> VideoProfile.Other + }), pixelFormat = pixels(sps.bitDepthLumaMinus8, sps.bitDepthChromaMinus8, chroma), color = color(sps.colorSpace, sps.colorTransfer, sps.colorRange), ) @@ -50,7 +57,13 @@ internal object CodecMetadataReader { if ((nal[0].toInt() and 1) != 0 || (nal[1].toInt() and 248) != 0) return CodecMetadata() if (!boundedHevcSyntax(nal)) return CodecMetadata() val sps = NalUnitUtil.parseH265SpsNalUnit(nal, 0, nal.size, null) + val profile = sps.profileTierLevel ?: return CodecMetadata() return CodecMetadata( + profile = MediaFact.Known(when (if (profile.generalProfileSpace == 0) profile.generalProfileIdc else -1) { + 1 -> VideoProfile.HevcMain + 2 -> VideoProfile.HevcMain10 + else -> VideoProfile.Other + }), pixelFormat = pixels(sps.bitDepthLumaMinus8, sps.bitDepthChromaMinus8, sps.chromaFormatIdc), color = color(sps.colorSpace, sps.colorTransfer, sps.colorRange), ) diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/MediaMetadataInspector.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/MediaMetadataInspector.kt index d10aa766..e415291b 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/MediaMetadataInspector.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/MediaMetadataInspector.kt @@ -172,8 +172,9 @@ internal object MediaTrackMetadataReader { positiveCode(format.int(MediaFormat.KEY_COLOR_TRANSFER)), positiveCode(format.int(MediaFormat.KEY_COLOR_RANGE)), ) else ColorMetadata() + val profile = platformProfile(codec.valueOrNull, if (apiLevel >= 24) format.int(MediaFormat.KEY_PROFILE) else null) val platformPixels = pixels(format.int(MediaFormat.KEY_COLOR_FORMAT), apiLevel) - val fallback = if (platformPixels == MediaFact.Unknown || platformColor.transfer == MediaFact.Unknown) + val fallback = if (platformPixels == MediaFact.Unknown || platformColor.transfer == MediaFact.Unknown || profile == MediaFact.Unknown) CodecMetadataReader.video(codec.valueOrNull, (0..2).mapNotNull { format.bytes("csd-$it") }) else CodecMetadata() val color = ColorMetadata( platformColor.standard.takeUnless { it == MediaFact.Unknown } ?: fallback.color.standard, @@ -183,7 +184,7 @@ internal object MediaTrackMetadataReader { val range = if (mime == "video/dolby-vision") MediaFact.Known(DynamicRange.DolbyVision) else dynamicRange(color.transfer) return VideoTrackMetadata(codec, - platformProfile(codec.valueOrNull, if (apiLevel >= 24) format.int(MediaFormat.KEY_PROFILE) else null), + profile.takeUnless { it == MediaFact.Unknown } ?: fallback.profile, encoded, display(format, encoded, rotation), rotation, platformRotation, platformPixels.takeUnless { it == MediaFact.Unknown } ?: fallback.pixelFormat, color, range) } diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/MediaMetadataReaderTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/MediaMetadataReaderTests.kt index 6fdd8bae..3e36416b 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/MediaMetadataReaderTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/MediaMetadataReaderTests.kt @@ -52,6 +52,7 @@ class MediaMetadataReaderTests : AbsRobolectricTest() { val format = video().apply { setByteBuffer("csd-0", ByteBuffer.wrap(avc)) } assertEquals(MediaFact.Known(PixelFormat(8, ChromaSubsampling.Yuv420)), facts(format).pixelFormat) assertEquals(MediaFact.Known(DynamicRange.Sdr), facts(format, 23).dynamicRange) + assertEquals(MediaFact.Known(VideoProfile.H264Baseline), facts(format, 23).videoProfile) format.setInteger(MediaFormat.KEY_COLOR_TRANSFER, MediaFormat.COLOR_TRANSFER_HLG) format.setInteger(MediaFormat.KEY_PROFILE, CodecProfileLevel.AVCProfileHigh) format.setInteger(MediaFormat.KEY_COLOR_FORMAT, CodecCapabilities.COLOR_FormatYUVP010) From f02e5bf31ceeada01cbe18980b18cc972538cfbe Mon Sep 17 00:00:00 2001 From: Eugene Morozov Date: Tue, 6 Oct 2026 10:22:52 -0700 Subject: [PATCH 11/49] Inspect bounded compressed video samples for Standard Input --- .../standardization/IsoContainerMetadata.kt | 195 +++++++++++++++ .../standardization/MediaSampleInspector.kt | 235 ++++++++++++++++++ .../IsoContainerMetadataTests.kt | 150 +++++++++++ .../MediaSampleInspectorTests.kt | 210 ++++++++++++++++ 4 files changed, 790 insertions(+) create mode 100644 library/src/main/java/com/mux/video/upload/internal/standardization/IsoContainerMetadata.kt create mode 100644 library/src/main/java/com/mux/video/upload/internal/standardization/MediaSampleInspector.kt create mode 100644 library/src/test/java/com/mux/video/upload/internal/standardization/IsoContainerMetadataTests.kt create mode 100644 library/src/test/java/com/mux/video/upload/internal/standardization/MediaSampleInspectorTests.kt diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/IsoContainerMetadata.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/IsoContainerMetadata.kt new file mode 100644 index 00000000..35e7e1c0 --- /dev/null +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/IsoContainerMetadata.kt @@ -0,0 +1,195 @@ +package com.mux.video.upload.internal.standardization + +import java.io.File +import java.io.RandomAccessFile +import java.nio.ByteBuffer + +internal data class IsoMetadataBox(val type: String, val payload: Long, val end: Long) + +internal object IsoSampleEntries { + val avc = setOf("avc1", "avc3", "dva1", "dvav") + val hevc = setOf("hvc1", "hev1", "dvh1", "dvhe") + val dolby = setOf("dvh1", "dvhe", "dva1", "dvav") + val visual = avc + hevc +} + +internal data class IsoTrackMetadata( + val handler: String?, + val trackId: MediaFact, + val sampleEntries: List, + val hasDolbyVisionConfiguration: Boolean, + val rotationDegrees: MediaFact, + val hasSimpleSampleGeometry: Boolean, + val editList: MediaFact = MediaFact.Unknown, +) { + val hasSimpleOrientation: Boolean get() = rotationDegrees != MediaFact.Unknown + fun matches(kind: TrackKind): Boolean = (handler == "vide" && kind == TrackKind.Video) || + (handler == "soun" && kind == TrackKind.Audio) +} + +/** Seeks past media payloads; reads track identity, simple orientation and Dolby Vision signaling. */ +internal object IsoContainerMetadataReader { + private const val MAX_BOXES = 4096 + private const val MAX_TOP_LEVEL_BOXES = 131072 + private const val MAX_ENTRIES = 16 + + fun read(file: File): MediaFact> = try { + RandomAccessFile(file, "r").use { input -> + val reader = Reader(input) + var movie: IsoMetadataBox? = null + for (box in reader.boxes(0, input.length(), topLevel = true)) { + if (box.type == "moov") { require(movie == null); movie = box } + } + val moov = movie ?: return MediaFact.Unknown + val children = reader.boxes(moov.payload, moov.end).toList() + val tracks = children.filter { it.type == "trak" } + if (tracks.size !in 1..64) return MediaFact.Unknown + val metadata = tracks.map { reader.track(it) } + val ids = metadata.mapNotNull { it.trackId.valueOrNull } + require(ids.distinct().size == ids.size) + MediaFact.Known(metadata) + } + } catch (_: Exception) { MediaFact.Unknown } + + private class Reader(val input: RandomAccessFile) { + var boxCount = 0 + val boxHeader = ByteArray(16) + + fun boxes(start: Long, end: Long, allowQuickTimeTerminator: Boolean = false, + topLevel: Boolean = false): Sequence = sequence { + require(start <= end && end <= input.length()) + var topCount = 0 + var position = start + while (position < end) { + if (allowQuickTimeTerminator && end - position == 4L) { + input.seek(position) + require(input.readInt() == 0) + break + } + require(end - position >= 8) + require(if (topLevel) ++topCount <= MAX_TOP_LEVEL_BOXES else ++boxCount <= MAX_BOXES) + input.seek(position) + input.readFully(boxHeader, 0, minOf(16L, end - position).toInt()) + val bytes = ByteBuffer.wrap(boxHeader) + val size32 = bytes.int.toLong() and 0xffffffffL + val type = String(boxHeader, 4, 4, Charsets.US_ASCII) + val header = if (size32 == 1L) 16 else 8 + require(end - position >= header) + val size = when (size32) { 0L -> end - position; 1L -> bytes.getLong(8); else -> size32 } + require(size >= header && size <= end - position) + yield(IsoMetadataBox(type, position + header, position + size)) + position += size + } + } + + fun track(track: IsoMetadataBox): IsoTrackMetadata { + val children = boxes(track.payload, track.end).toList() + val tkhd = children.singleOrNull { it.type == "tkhd" } + val mdia = children.single { it.type == "mdia" } + val media = boxes(mdia.payload, mdia.end).toList() + val handler = media.singleOrNull { it.type == "hdlr" }?.let { + require(it.end - it.payload >= 12) + input.seek(it.payload + 8) + fourCc() + } + val trackId = tkhd?.let { + input.seek(it.payload) + val version = input.readUnsignedByte() + val offset = when (version) { 0 -> 12; 1 -> 20; else -> -1 } + if (offset < 0 || it.end - it.payload < offset + 4) MediaFact.Unknown else { + input.seek(it.payload + offset) + positiveCode(input.readInt()) + } + } ?: MediaFact.Unknown + val orientation = tkhd?.let(::orientation) ?: MediaFact.Unknown + if (handler !in listOf("vide", "soun")) + return IsoTrackMetadata(handler, trackId, emptyList(), false, orientation, false) + val minf = media.single { it.type == "minf" } + val stbl = boxes(minf.payload, minf.end).single { it.type == "stbl" } + val tables = boxes(stbl.payload, stbl.end).toList() + val stsd = tables.single { it.type == "stsd" } + require(stsd.end - stsd.payload >= 8) + input.seek(stsd.payload + 4) + val count = input.readInt() + require(count in 1..MAX_ENTRIES) + val entries = boxes(stsd.payload + 8, stsd.end).toList() + require(entries.size == count) + var dolby = false + var simpleGeometry = entries.size == 1 + if (handler == "vide") { + for (entry in entries) { + if (entry.type in IsoSampleEntries.dolby) dolby = true + if (entry.type in IsoSampleEntries.visual) { + require(entry.end - entry.payload >= 78) + val extensions = boxes(entry.payload + 78, entry.end, allowQuickTimeTerminator = true).toList() + if (extensions.any { it.type in setOf("dvcC", "dvvC", "dvwC") }) dolby = true + if (extensions.any { it.type == "clap" }) simpleGeometry = false + val aspects = extensions.filter { it.type == "pasp" } + if (aspects.size > 1) simpleGeometry = false + for (aspect in aspects) { + require(aspect.end - aspect.payload == 8L) + input.seek(aspect.payload) + val horizontal = input.readInt() + val vertical = input.readInt() + // Only square pixels are supported; unusual geometry keeps original-file fallback. + if (horizontal == 0 || horizontal != vertical) simpleGeometry = false + } + } else simpleGeometry = false + } + } + return IsoTrackMetadata(handler, trackId, entries.map { it.type }, dolby, orientation, simpleGeometry, + edits(children)) + } + + /** Classifies edit structure only; applying it to the presentation timeline is separate work. */ + private fun edits(children: List): MediaFact { + val editBoxes = children.filter { it.type == "edts" } + if (editBoxes.isEmpty()) return MediaFact.Known(EditList.None) + if (editBoxes.size != 1) return MediaFact.Unknown + val lists = boxes(editBoxes.single().payload, editBoxes.single().end).toList() + if (lists.size != 1 || lists.single().type != "elst") return MediaFact.Unknown + val list = lists.single() + require(list.end - list.payload >= 8) + input.seek(list.payload) + val version = input.readUnsignedByte() + if (version !in 0..1 || input.readUnsignedByte() != 0 || input.readUnsignedShort() != 0) + return MediaFact.Unknown + val count = input.readInt().toLong() and 0xffffffffL + val entrySize = if (version == 0) 12 else 20 + require(count <= 64 && list.end - list.payload == 8 + count * entrySize) + if (count == 0L) return MediaFact.Known(EditList.None) + var simple = count == 1L + repeat(count.toInt()) { + val duration = if (version == 0) input.readInt().toLong() and 0xffffffffL else input.readLong() + val mediaTime = if (version == 0) input.readInt().toLong() else input.readLong() + val rate = input.readInt() + if (duration <= 0 || mediaTime < 0 || rate != 0x00010000) simple = false + } + return MediaFact.Known(if (simple) EditList.Simple else EditList.Complex) + } + + private fun orientation(box: IsoMetadataBox): MediaFact { + input.seek(box.payload) + val version = input.readUnsignedByte() + val offset = when (version) { 0 -> 40; 1 -> 52; else -> return MediaFact.Unknown } + require(box.end - box.payload >= offset + 36) + input.seek(box.payload + offset) + val matrix = IntArray(9) { input.readInt() } + if (matrix[2] != 0 || matrix[5] != 0 || matrix[8] != 0x40000000) return MediaFact.Unknown + // Same normalized transform convention as Swift. Android's reported angle is negated separately. + return when (listOf(matrix[0], matrix[1], matrix[3], matrix[4])) { + listOf(65536, 0, 0, 65536) -> MediaFact.Known(0) + listOf(0, 65536, -65536, 0) -> MediaFact.Known(270) + listOf(-65536, 0, 0, -65536) -> MediaFact.Known(180) + listOf(0, -65536, 65536, 0) -> MediaFact.Known(90) + else -> MediaFact.Unknown + } + } + + private fun fourCc(): String { + val bytes = ByteArray(4) + input.readFully(bytes) + return String(bytes, Charsets.US_ASCII) + } + } +} diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/MediaSampleInspector.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/MediaSampleInspector.kt new file mode 100644 index 00000000..8b53a5ca --- /dev/null +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/MediaSampleInspector.kt @@ -0,0 +1,235 @@ +package com.mux.video.upload.internal.standardization + +import android.media.MediaExtractor +import android.os.Build +import androidx.media3.container.NalUnitUtil +import java.io.File +import java.nio.ByteBuffer +import kotlin.math.abs +import kotlin.math.roundToLong + +internal enum class SampleScanStatus { Complete, Unsupported, Unreadable, LimitExceeded, Cancelled } +internal data class SampleScanLimits( + val maximumSampleBytes: Int = 4 * 1024 * 1024, + val maximumSamples: Int = 250_000, + val maximumReadBytes: Long = 1024L * 1024 * 1024, + /** Checked between platform reads; a blocking native call cannot be preempted. */ + val maximumElapsedNanos: Long = 5_000_000_000, +) { + init { + require(maximumSampleBytes > 0 && maximumSamples > 0 && maximumReadBytes > 0 && maximumElapsedNanos > 0) + } +} + +internal data class MediaSampleInspection( + val facts: MediaFacts, + val status: SampleScanStatus, + val sampleCount: Int, + val bytesRead: Long, + val largestSampleBytes: Int, + val elapsedNanos: Long, +) + +internal data class CompressedVideoSample( + val presentationTimeUs: Long, + val byteSize: Int, + val isSync: Boolean, + val randomAccess: MediaFact = MediaFact.Unknown, +) + +/** Separate from metadata inspection. No decoding, exporting, or effective timeline calculation. */ +internal class MediaSampleInspector(private val limits: SampleScanLimits = SampleScanLimits(), + private val apiLevel: Int = Build.VERSION.SDK_INT) { + fun inspect(file: File, metadata: MediaMetadataInspection, + isCancelled: () -> Boolean = { Thread.currentThread().isInterrupted }): MediaSampleInspection { + val started = System.nanoTime() + val samples = ArrayList() + var bytes = 0L + var largestSample = 0 + fun result(status: SampleScanStatus, facts: MediaFacts = metadata.facts) = + MediaSampleInspection(facts, status, samples.size, bytes, largestSample, System.nanoTime() - started) + if (isCancelled()) return result(SampleScanStatus.Cancelled) + val track = metadata.tracks.filter { it.kind == TrackKind.Video }.singleOrNull() + ?: return result(SampleScanStatus.Unsupported) + if (metadata.facts.videoTrackCount != MediaFact.Known(1) || + metadata.facts.videoCodec.valueOrNull !in listOf(VideoCodec.H264, VideoCodec.Hevc)) + return result(SampleScanStatus.Unsupported) + if (metadata.container == MediaFact.Known(ContainerKind.IsoBaseMedia) && + track.containerIndex == MediaFact.Unknown) return result(SampleScanStatus.Unsupported) + val codec = metadata.facts.videoCodec.valueOrNull!! + val extractor = MediaExtractor() + return try { + extractor.setDataSource(file.absolutePath) + extractor.selectTrack(track.extractorIndex) + val data = ByteArray(limits.maximumSampleBytes) + val buffer = ByteBuffer.wrap(data) + while (extractor.sampleTrackIndex >= 0) { + if (isCancelled()) return result(SampleScanStatus.Cancelled) + if (samples.size >= limits.maximumSamples || bytes >= limits.maximumReadBytes || + System.nanoTime() - started >= limits.maximumElapsedNanos) return result(SampleScanStatus.LimitExceeded) + val flags = extractor.sampleFlags + // Partial frames and encrypted bytes cannot establish complete pictures or GOPs. + if (flags and MediaExtractor.SAMPLE_FLAG_SYNC.inv() != 0) return result(SampleScanStatus.Unsupported) + val expectedSize = if (apiLevel >= 28) extractor.sampleSize else null + if (expectedSize != null && (expectedSize <= 0 || expectedSize > data.size || + expectedSize > limits.maximumReadBytes - bytes)) return result(SampleScanStatus.LimitExceeded) + buffer.clear() + // API 23–27 has no size query. Cap the native buffer's capacity as well as its limit. + val remaining = limits.maximumReadBytes - bytes + val readBuffer = if (remaining < data.size) ByteBuffer.wrap(data, 0, remaining.toInt()).slice() else buffer + val size = extractor.readSampleData(readBuffer, 0) + if (size <= 0 || size > data.size || size.toLong() > limits.maximumReadBytes - bytes || + (expectedSize != null && size.toLong() != expectedSize)) return result(SampleScanStatus.LimitExceeded) + bytes += size + largestSample = maxOf(largestSample, size) + val sync = flags and MediaExtractor.SAMPLE_FLAG_SYNC != 0 + samples.add(CompressedVideoSample(extractor.sampleTime, size, sync, + if (sync) SampleRandomAccessReader.read(codec, data, size) else MediaFact.Unknown)) + extractor.advance() + } + if (isCancelled()) return result(SampleScanStatus.Cancelled) + if (System.nanoTime() - started >= limits.maximumElapsedNanos) return result(SampleScanStatus.LimitExceeded) + // Match the shared video-edit policy. Audio edits remain separate track observations. + val edit = metadata.isoTracks.valueOrNull?.getOrNull(track.containerIndex.valueOrNull ?: -1)?.editList + ?: MediaFact.Unknown + val facts = VideoSampleFactsReader.read(samples, track.reportedDurationSeconds, metadata.facts).copy(editList = edit) + if (isCancelled()) return result(SampleScanStatus.Cancelled) + if (System.nanoTime() - started >= limits.maximumElapsedNanos) return result(SampleScanStatus.LimitExceeded) + result(SampleScanStatus.Complete, facts) + } catch (_: Exception) { + result(SampleScanStatus.Unreadable) + } finally { + extractor.release() + } + } +} + +/** Only sync samples need relevant NAL inspection. Unrecognized framing or syntax stays unknown. */ +internal object SampleRandomAccessReader { + fun read(codec: VideoCodec, data: ByteArray, size: Int): MediaFact = try { + inspect(codec, data, size) + } catch (_: LinkageError) { MediaFact.Unknown } + catch (_: RuntimeException) { MediaFact.Unknown } + + private fun inspect(codec: VideoCodec, data: ByteArray, size: Int): MediaFact { + if (size !in 1..data.size) return MediaFact.Unknown + val flags = BooleanArray(3) + var prefix = NalUnitUtil.findNalUnit(data, 0, size, flags) + if (prefix !in 0..1 || (prefix == 1 && data[0] != 0.toByte())) return MediaFact.Unknown + var slices = 0 + var idr = true + var hasIdr = false + var open = false + var count = 0 + while (prefix < size) { + if (++count > 4096) return MediaFact.Unknown + val start = prefix + 3 + NalUnitUtil.clearPrefixFlags(flags) + val next = NalUnitUtil.findNalUnit(data, start, size, flags) + if (start >= next || data[start].toInt() and 128 != 0) return MediaFact.Unknown + when (codec) { + VideoCodec.H264 -> { + val type = data[start].toInt() and 31 + when (type) { + 1, 5 -> { + if (next - start < 2 || (type == 5 && data[start].toInt() and 96 == 0)) return MediaFact.Unknown + slices++; idr = idr && type == 5; hasIdr = hasIdr || type == 5 + } + 6, 7, 8, 9, 12 -> Unit + else -> return MediaFact.Unknown + } + } + VideoCodec.Hevc -> { + if (next - start < 3 || data[start].toInt() and 1 != 0 || + data[start + 1].toInt() and 248 != 0 || data[start + 1].toInt() and 7 == 0) + return MediaFact.Unknown + val type = (data[start].toInt() and 126) shr 1 + when (type) { + in 0..9, in 16..21 -> { + slices++ + idr = idr && type in 19..20 + hasIdr = hasIdr || type in 19..20 + if (type in 16..18 || type == 21) open = true + } + 32, 33, 34, 35, 38, 39, 40 -> Unit + else -> return MediaFact.Unknown + } + } + else -> return MediaFact.Unknown + } + prefix = next + } + return when { + slices == 0 -> MediaFact.Unknown + open && hasIdr -> MediaFact.Unknown + open -> MediaFact.Known(GopStructure.Open) + idr -> MediaFact.Known(GopStructure.ClosedWithIdr) + codec == VideoCodec.H264 && !hasIdr -> MediaFact.Known(GopStructure.Open) + else -> MediaFact.Unknown + } + } +} + +internal object VideoSampleFactsReader { + fun read(samples: List, reportedDuration: MediaFact, + base: MediaFacts = MediaFacts()): MediaFacts { + if (samples.size < 2 || samples.any { it.byteSize <= 0 }) return base + val ordered = LongArray(samples.size) { samples[it].presentationTimeUs }.also { it.sort() } + val intervals = LongArray(samples.size - 1) { ordered[it + 1] - ordered[it] } + if (intervals.any { it <= 0 }) return base + val span = (ordered.last() - ordered.first()) / 1e6 + if (!span.isFinite() || span <= 0) return base + val frameInterval = span / (samples.size - 1) + val constant = intervals.all { abs(it / 1e6 - frameInterval) <= maxOf(2e-6, frameInterval * 0.001) } + var facts = base.copy( + frameRate = MediaFact.Known(1 / frameInterval), + cadence = MediaFact.Known(if (constant) Cadence.Constant else Cadence.Variable), + timestamps = MediaFact.Known(TimestampFacts(ordered.first() / 1e6, ordered.last() / 1e6, + samples.size.toLong(), (1 until samples.size).all { + samples[it - 1].presentationTimeUs < samples[it].presentationTimeUs + })), + ) + // Raw track duration is a bitrate denominator, not proof of effective duration or A/V sync. + // Reject inconsistent metadata instead of extrapolating an unobserved final VFR interval. + val duration = reportedDuration.valueOrNull?.takeIf { + it.isFinite() && it > span && it - span <= intervals.max() / 1e6 * 2 + } + if (duration != null) facts = facts.copy(averageBitrate = bitrate(samples.sumOf { it.byteSize.toLong() }, duration)) + if (!samples.first().isSync) return facts + val boundaries = samples.indices.filter { samples[it].isSync } + val kinds = boundaries.map { samples[it].randomAccess } + val open = kinds.any { it == MediaFact.Known(GopStructure.Open) } + val proven = kinds.all { it == MediaFact.Known(GopStructure.ClosedWithIdr) } + facts = facts.copy(gopStructure = when { + open -> MediaFact.Known(GopStructure.Open) + proven -> MediaFact.Known(GopStructure.ClosedWithIdr) + else -> MediaFact.Unknown + }) + var maxBytes = 0L + var maxRate = 0L + var maxInterval = 0.0 + for (i in boundaries.indices) { + val start = boundaries[i] + val end = boundaries.getOrNull(i + 1) ?: samples.size + val startTime = samples[start].presentationTimeUs / 1e6 + val endTime = if (end < samples.size) samples[end].presentationTimeUs / 1e6 + else duration?.plus(ordered.first() / 1e6) ?: return facts + val seconds = endTime - startTime + if (!seconds.isFinite() || seconds <= 0) return facts + maxInterval = maxOf(maxInterval, seconds) + val bytes = samples.subList(start, end).sumOf { it.byteSize.toLong() } + maxBytes = maxOf(maxBytes, bytes) + maxRate = maxOf(maxRate, bitrate(bytes, seconds).valueOrNull ?: return facts) + } + return facts.copy(maximumKeyframeIntervalSeconds = MediaFact.Known(maxInterval), + // Leading pictures in open/unproven GOPs cannot be assigned reliably to a byte window. + maximumGopByteSize = if (proven) MediaFact.Known(maxBytes) else MediaFact.Unknown, + maximumGopBitrate = if (proven) MediaFact.Known(maxRate) else MediaFact.Unknown) + } + + private fun bitrate(bytes: Long, seconds: Double): MediaFact { + val rate = bytes.toDouble() * 8 / seconds + return if (rate.isFinite() && rate > 0 && rate < Long.MAX_VALUE.toDouble()) + MediaFact.Known(rate.roundToLong()) else MediaFact.Unknown + } +} diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/IsoContainerMetadataTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/IsoContainerMetadataTests.kt new file mode 100644 index 00000000..12f6cd53 --- /dev/null +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/IsoContainerMetadataTests.kt @@ -0,0 +1,150 @@ +package com.mux.video.upload.internal.standardization + +import com.mux.exoplayeradapter.AbsRobolectricTest +import org.robolectric.annotation.Config +import org.junit.Assert.* +import org.junit.Test +import java.io.File +import java.nio.ByteBuffer + +@Config(sdk = [23]) +class IsoContainerMetadataTests : AbsRobolectricTest() { + private fun int(value: Int) = ByteBuffer.allocate(4).putInt(value).array() + private fun box(type: String, payload: ByteArray) = int(payload.size + 8) + type.toByteArray() + payload + private fun track(handler: String = "vide", entry: String = "hvc1", extension: ByteArray = byteArrayOf(), + edits: ByteArray = byteArrayOf(), + matrix: IntArray = intArrayOf(65536, 0, 0, 0, 65536, 0, 0, 0, 0x40000000)): ByteArray { + val tkhd = ByteArray(84) + for (i in matrix.indices) int(matrix[i]).copyInto(tkhd, 40 + i * 4) + val hdlr = ByteArray(8) + handler.toByteArray() + val description = ByteArray(if (handler == "vide") 78 else 28) + if (handler == "vide") { + ByteBuffer.wrap(description).putShort(24, 1920.toShort()).putShort(26, 1080.toShort()) + int(1920 * 65536).copyInto(tkhd, 76) + int(1080 * 65536).copyInto(tkhd, 80) + } + val sampleEntry = box(entry, description + extension) + val stsd = box("stsd", ByteArray(4) + int(1) + sampleEntry) + return box("trak", box("tkhd", tkhd) + edits + + box("mdia", box("hdlr", hdlr) + box("minf", box("stbl", stsd)))) + } + + private fun read(data: ByteArray): MediaFact> { + val file = File.createTempFile("iso-metadata", ".mp4") + return try { file.writeBytes(data); IsoContainerMetadataReader.read(file) } finally { file.delete() } + } + + @Test fun readsContainerOrderAndDolbyConfigurationInOrdinaryHevcEntry() { + val tracks = read(box("ftyp", "isom".toByteArray()) + box("moov", + track(extension = box("dvvC", byteArrayOf(1, 0, 0, 0))) + track("soun", "mp4a"))).valueOrNull!! + assertEquals(listOf("vide", "soun"), tracks.map { it.handler }) + assertEquals(listOf("hvc1"), tracks[0].sampleEntries) + assertTrue(tracks[0].hasDolbyVisionConfiguration) + assertTrue(tracks[0].hasSimpleOrientation) + } + + @Test fun readsDolbySampleEntryEvenWithoutConfigurationChild() { + assertTrue(read(box("moov", track(entry = "dvh1"))).valueOrNull!![0].hasDolbyVisionConfiguration) + } + + private fun edit(version: Int = 0, entries: List> = listOf(3000L to 0L), + rate: Int = 65536): ByteArray { + val payload = byteArrayOf(version.toByte(), 0, 0, 0) + int(entries.size) + entries.fold(byteArrayOf()) { bytes, (duration, time) -> + bytes + (if (version == 0) int(duration.toInt()) + int(time.toInt()) + else ByteBuffer.allocate(16).putLong(duration).putLong(time).array()) + int(rate) + } + return box("edts", box("elst", payload)) + } + + @Test fun classifiesEditsWithoutApplyingTheirTimeline() { + fun classification(edits: ByteArray) = read(box("moov", track(edits = edits))).valueOrNull!!.single().editList + assertEquals(MediaFact.Known(EditList.None), classification(byteArrayOf())) + assertEquals(MediaFact.Known(EditList.None), classification(edit(entries = emptyList()))) + for (version in 0..1) { + assertEquals(MediaFact.Known(EditList.Simple), classification(edit(version, listOf(3000L to 123L)))) + assertEquals(MediaFact.Known(EditList.Complex), classification(edit(version, listOf(3000L to -1L)))) + } + assertEquals(MediaFact.Known(EditList.Complex), classification(edit(entries = listOf(3000L to 0L, 1000L to 2000L)))) + assertEquals(MediaFact.Known(EditList.Complex), classification(edit(rate = 0))) + assertEquals(MediaFact.Unknown, classification(edit(version = 2))) + assertEquals(MediaFact.Unknown, classification(edit() + edit())) + assertEquals(MediaFact.Unknown, read(box("moov", track(edits = box("edts", box("elst", ByteArray(5))))))) + } + + @Test fun doesNotClaimSimpleGeometryForReflectionShearOrScale() { + for (matrix in listOf( + intArrayOf(-65536, 0, 0, 0, 65536, 0, 0, 0, 0x40000000), + intArrayOf(65536, 100, 0, 0, 65536, 0, 0, 0, 0x40000000), + intArrayOf(131072, 0, 0, 0, 65536, 0, 0, 0, 0x40000000))) { + assertFalse(read(box("moov", track(matrix = matrix))).valueOrNull!![0].hasSimpleOrientation) + } + } + + @Test fun recognizesQuarterRotationWithTranslation() { + val matrix = intArrayOf(0, 65536, 0, -65536, 0, 0, 1920 * 65536, 0, 0x40000000) + assertTrue(read(box("moov", track(matrix = matrix))).valueOrNull!![0].hasSimpleOrientation) + } + + @Test fun unsupportedGeometryStaysUnknownAndSquarePixelsAreAccepted() { + for (extension in listOf(box("clap", ByteArray(32)), box("pasp", int(2) + int(1)), + box("pasp", int(1) + int(1)) + box("pasp", int(1) + int(1)))) { + assertFalse(read(box("moov", track(extension = extension))).valueOrNull!![0].hasSimpleSampleGeometry) + } + assertTrue(read(box("moov", track(extension = box("pasp", int(1) + int(1))))) + .valueOrNull!![0].hasSimpleSampleGeometry) + } + + @Test fun malformedSizesAndBoxLimitsStayUnknown() { + for (data in listOf(byteArrayOf(), int(4096) + "moov".toByteArray(), + int(4) + "moov".toByteArray(), int(1) + "moov".toByteArray() + ByteArray(8) { -1 })) { + assertEquals(MediaFact.Unknown, read(data)) + } + val many = ByteArray(4097 * 8) + val free = box("free", byteArrayOf()) + for (i in 0..4096) free.copyInto(many, i * 8) + assertEquals(MediaFact.Unknown, read(many)) + } + + @Test fun permitsOnlyZeroQuickTimeSampleDescriptionTerminator() { + assertTrue(read(box("moov", track(extension = box("dvvC", ByteArray(4)) + ByteArray(4)))) + .valueOrNull!![0].hasDolbyVisionConfiguration) + assertEquals(MediaFact.Unknown, read(box("moov", track(extension = int(1))))) + } + + @Test fun noTrackOrDuplicateMovieBoxDoesNotInventAbsence() { + assertEquals(MediaFact.Unknown, read(box("moov", byteArrayOf()))) + assertEquals(MediaFact.Unknown, read(box("moov", track()) + box("moov", track()))) + } + + @Test fun everyUnitMatrixHasTheSharedNormalizedDegreeValue() { + for ((axes, degrees) in listOf( + listOf(65536, 0, 0, 65536) to 0, + listOf(0, 65536, -65536, 0) to 270, + listOf(-65536, 0, 0, -65536) to 180, + listOf(0, -65536, 65536, 0) to 90)) { + val matrix = intArrayOf(axes[0], axes[1], 0, axes[2], axes[3], 0, 1920 * 65536, 0, 0x40000000) + assertEquals(MediaFact.Known(degrees), read(box("moov", track(matrix = matrix))).valueOrNull!![0].rotationDegrees) + } + } + + @Test fun fragmentHeadersDoNotConsumeMovieMetadataBudget() { + val fragment = box("moof", byteArrayOf()) + box("mdat", byteArrayOf()) + val many = ByteArray(7200 * fragment.size) + repeat(7200) { fragment.copyInto(many, it * fragment.size) } + val movie = box("moov", track()) + assertNotEquals(MediaFact.Unknown, read(movie + many)) + assertNotEquals(MediaFact.Unknown, read(many + movie)) + assertEquals(MediaFact.Unknown, read(movie + many + movie)) + assertEquals(MediaFact.Unknown, read(movie + byteArrayOf(1))) + } + + @Test fun quickTimeLeadingBoxesAreRecognizedAsIso() { + for (type in listOf("skip", "uuid", "pnot")) { + val file = File.createTempFile("iso-sniff", ".mov") + try { + file.writeBytes(box(type, ByteArray(16)) + box("moov", track())) + assertEquals(MediaFact.Known(ContainerKind.IsoBaseMedia), MediaMetadataInspector().container(file)) + } finally { file.delete() } + } + } +} diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/MediaSampleInspectorTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/MediaSampleInspectorTests.kt new file mode 100644 index 00000000..f828848f --- /dev/null +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/MediaSampleInspectorTests.kt @@ -0,0 +1,210 @@ +package com.mux.video.upload.internal.standardization + +import android.media.MediaExtractor +import android.os.Build +import androidx.media3.container.NalUnitUtil +import com.mux.exoplayeradapter.AbsRobolectricTest +import io.mockk.* +import org.junit.Assert.* +import org.junit.Test +import org.robolectric.annotation.Config +import java.io.File +import java.nio.ByteBuffer + +@Config(sdk = [23]) +class MediaSampleInspectorTests : AbsRobolectricTest() { + private val idr = MediaFact.Known(GopStructure.ClosedWithIdr) + private fun sample(time: Long, sync: Boolean = false, bytes: Int = 100, + kind: MediaFact = idr) = CompressedVideoSample(time, bytes, sync, kind) + private fun facts(samples: List, duration: Double = 3.0) = + VideoSampleFactsReader.read(samples, MediaFact.Known(duration)) + private fun nal(vararg bytes: Int) = byteArrayOf(0, 0, 0, 1) + bytes.map { it.toByte() }.toByteArray() + + @Test fun measuresWholeVideoSeparatelyFromMaximumGopAndIncludesFinalTail() { + val facts = facts(listOf(sample(0, true, 100), sample(1_000_000, bytes = 300), + sample(2_000_000, true, 500))) + assertEquals(MediaFact.Known(1.0), facts.frameRate) + assertEquals(MediaFact.Known(Cadence.Constant), facts.cadence) + assertEquals(MediaFact.Known(2400L), facts.averageBitrate) + assertEquals(MediaFact.Known(4000L), facts.maximumGopBitrate) + assertEquals(MediaFact.Known(500L), facts.maximumGopByteSize) + assertEquals(MediaFact.Known(2.0), facts.maximumKeyframeIntervalSeconds) + assertEquals(idr, facts.gopStructure) + assertEquals(MediaFact.Unknown, facts.durationSeconds) + assertEquals(MediaFact.Unknown, facts.audioVideoStartOffsetSeconds) + } + + @Test fun reorderedPresentationTimesDoNotInventVariableCadence() { + val facts = facts(listOf(sample(0, true), sample(2_000_000), sample(1_000_000))) + assertEquals(MediaFact.Known(Cadence.Constant), facts.cadence) + assertEquals(MediaFact.Known(TimestampFacts(0.0, 2.0, 3, false)), facts.timestamps) + } + + @Test fun variableIntervalsRemainVariableAndNegativePrerollIsObserved() { + val facts = facts(listOf(sample(-100_000, true), sample(100_000), sample(400_000)), 0.8) + assertEquals(MediaFact.Known(Cadence.Variable), facts.cadence) + assertEquals(MediaFact.Known(4.0), facts.frameRate) + assertEquals(-0.1, facts.timestamps.valueOrNull!!.firstPresentationSeconds, 0.0) + } + + @Test fun noSyncAtStartCannotProveGops() { + val facts = facts(listOf(sample(0), sample(1_000_000, true), sample(2_000_000))) + assertEquals(MediaFact.Unknown, facts.gopStructure) + assertEquals(MediaFact.Unknown, facts.maximumGopBitrate) + assertEquals(MediaFact.Known(800L), facts.averageBitrate) + } + + @Test fun singleGopTailCountsForLongKeyframeViolations() { + val facts = facts((0..20).map { sample(it * 1_000_000L, it == 0) }, 21.0) + assertEquals(MediaFact.Known(21.0), facts.maximumKeyframeIntervalSeconds) + } + + @Test fun openGopDoesNotProveByteAllocationAndOverridesUnknownBoundary() { + val facts = facts(listOf(sample(0, true, kind = MediaFact.Unknown), sample(1_000_000), + sample(2_000_000, true, kind = MediaFact.Known(GopStructure.Open)))) + assertEquals(MediaFact.Known(GopStructure.Open), facts.gopStructure) + assertEquals(MediaFact.Unknown, facts.maximumGopBitrate) + assertEquals(MediaFact.Unknown, facts.maximumGopByteSize) + assertEquals(MediaFact.Known(2.0), facts.maximumKeyframeIntervalSeconds) + } + + @Test fun contradictoryOrMissingDurationDoesNotExtrapolateVfrTail() { + val samples = listOf(sample(0, true), sample(1_000_000), sample(2_000_000)) + for (duration in listOf(MediaFact.Unknown, MediaFact.Known(1.0), MediaFact.Known(100.0))) { + val facts = VideoSampleFactsReader.read(samples, duration) + assertEquals(MediaFact.Known(1.0), facts.frameRate) + assertEquals(MediaFact.Unknown, facts.averageBitrate) + assertEquals(MediaFact.Unknown, facts.maximumKeyframeIntervalSeconds) + } + } + + @Test fun duplicateTimesEmptyOrInvalidBytesCannotInventRates() { + for (samples in listOf(emptyList(), listOf(sample(0, true)), + listOf(sample(0, true), sample(0)), listOf(sample(0, true), sample(1_000_000, bytes = 0)))) + assertEquals(MediaFacts(), facts(samples)) + } + + @Test fun avcIdrRequiresEverySliceToBeIdrAndRecognizesRecoveryPoint() { + assertEquals(idr, SampleRandomAccessReader.read(VideoCodec.H264, nal(0x65, 0x80), 6)) + val mixed = nal(0x65, 0x80) + nal(0x41, 0x80) + assertEquals(MediaFact.Unknown, SampleRandomAccessReader.read(VideoCodec.H264, mixed, mixed.size)) + val recovery = nal(6, 6, 1, 0x80, 0x80) + nal(0x41, 0x80) + assertEquals(MediaFact.Known(GopStructure.Open), + SampleRandomAccessReader.read(VideoCodec.H264, recovery, recovery.size)) + } + + @Test fun hevcSyncRequiresIdrAndCraOrBlaAreOpen() { + for (type in 16..21) { + val data = nal(type shl 1, 1, 0x80) + assertEquals(if (type in 19..20) idr else MediaFact.Known(GopStructure.Open), + SampleRandomAccessReader.read(VideoCodec.Hevc, data, data.size)) + } + val ordinary = nal(2, 1, 0x80) + assertEquals(MediaFact.Unknown, SampleRandomAccessReader.read(VideoCodec.Hevc, ordinary, ordinary.size)) + } + + @Test fun malformedFramingHeadersAndMultilayerStayUnknown() { + for (data in listOf(byteArrayOf(), byteArrayOf(0, 0, 0, 2, 0x26, 1), + nal(0xa6, 1, 0x80), nal(0x26, 0, 0x80), nal(0x27, 1, 0x80), nal(0x26, 9, 0x80), nal(0x26))) + assertEquals(MediaFact.Unknown, SampleRandomAccessReader.read(VideoCodec.Hevc, data, data.size)) + } + + @Test fun threeBytePrefixesAndSeveralSlicesAreSupported() { + val data = byteArrayOf(0, 0, 1, 0x65, 0x80.toByte(), 0, 0, 1, 0x65, 0x80.toByte()) + assertEquals(idr, SampleRandomAccessReader.read(VideoCodec.H264, data, data.size)) + } + + private fun metadata() = MediaMetadataInspection(MediaFact.Known(ContainerKind.Other), listOf( + TrackMetadata(0, MediaFact.Unknown, MediaFact.Unknown, TrackKind.Video, MediaFact.Known("video/avc"), + MediaFact.Known(3.0), MediaFact.Unknown, MediaFact.Unknown)), MediaFact.Unknown, + MediaFacts(videoCodec = MediaFact.Known(VideoCodec.H264), videoTrackCount = MediaFact.Known(1)), 0) + + private fun scan(limits: SampleScanLimits = SampleScanLimits(), cancel: () -> Boolean = { false }, + reader: ((ByteBuffer) -> Int)? = null, reportedSize: Long = 6, flagsOverride: Int? = null): MediaSampleInspection { + mockkConstructor(MediaExtractor::class) + var index = 0 + var released = false + val bytes = nal(0x65, 0x80) + every { anyConstructed().setDataSource(any()) } just Runs + every { anyConstructed().selectTrack(0) } just Runs + every { anyConstructed().sampleTrackIndex } answers { if (index < 3) 0 else -1 } + every { anyConstructed().sampleFlags } answers { flagsOverride ?: if (index == 0) 1 else 0 } + if (Build.VERSION.SDK_INT >= 28) every { anyConstructed().sampleSize } returns reportedSize + every { anyConstructed().sampleTime } answers { index * 1_000_000L } + every { anyConstructed().readSampleData(any(), 0) } answers { + reader?.invoke(firstArg()) ?: if (firstArg().capacity() < bytes.size) -1 + else bytes.size.also { firstArg().put(bytes) } + } + every { anyConstructed().advance() } answers { ++index < 3 } + every { anyConstructed().release() } answers { released = true } + return try { + MediaSampleInspector(limits).inspect(File("unused"), metadata(), cancel).also { + assertTrue(released) + } + } finally { unmockkConstructor(MediaExtractor::class) } + } + + @Test fun api23UsesBoundedReadsAndReleasesExtractor() { + val result = scan(SampleScanLimits(maximumSampleBytes = 16)) + assertEquals(SampleScanStatus.Complete, result.status) + assertEquals(18L, result.bytesRead) + assertEquals(3, result.sampleCount) + assertEquals(idr, result.facts.gopStructure) + } + + @Test fun limitsAndCancellationDiscardPartialProof() { + for (limits in listOf(SampleScanLimits(maximumSamples = 1), SampleScanLimits(maximumReadBytes = 8), + SampleScanLimits(maximumElapsedNanos = 1))) { + val result = scan(limits) + assertEquals(SampleScanStatus.LimitExceeded, result.status) + assertEquals(metadata().facts, result.facts) + } + var calls = 0 + val result = scan(cancel = { ++calls >= 3 }) + assertEquals(SampleScanStatus.Cancelled, result.status) + assertEquals(metadata().facts, result.facts) + } + + @Test fun oversizedAndFailedApi23ReadsNeverGrowBufferOrPublishPartialFacts() { + var capacity = 0 + val result = scan(SampleScanLimits(maximumSampleBytes = 16), reader = { + capacity = it.capacity(); 17 + }) + assertEquals(16, capacity) + assertEquals(SampleScanStatus.LimitExceeded, result.status) + assertEquals(metadata().facts, result.facts) + assertEquals(SampleScanStatus.Unreadable, scan(reader = { throw IllegalArgumentException() }).status) + } + + @Test @Config(sdk = [28]) fun api28ChecksSizeBeforeReadAndRejectsInconsistentCount() { + var read = false + assertEquals(SampleScanStatus.LimitExceeded, + scan(SampleScanLimits(maximumSampleBytes = 16), reader = { read = true; 6 }, reportedSize = 17).status) + assertFalse(read) + assertEquals(SampleScanStatus.LimitExceeded, scan(reportedSize = 7).status) + assertEquals(SampleScanStatus.Complete, scan().status) + } + + @Test fun encryptedAndPartialFramesCannotSupplySampleFacts() { + for (flags in listOf(2, 4)) { + val result = scan(flagsOverride = flags) + assertEquals(SampleScanStatus.Unsupported, result.status) + assertEquals(metadata().facts, result.facts) + } + } + + @Test fun cancellationWinsBeforeUnsupportedInputFallback() { + val result = MediaSampleInspector().inspect(File("unused"), metadata().copy(tracks = emptyList())) { true } + assertEquals(SampleScanStatus.Cancelled, result.status) + assertEquals(metadata().facts, result.facts) + } + + @Test fun incompatibleMedia3HelperLeavesRandomAccessUnknown() { + mockkStatic(NalUnitUtil::class) + try { + every { NalUnitUtil.findNalUnit(any(), any(), any(), any()) } throws NoSuchMethodError() + val data = nal(0x65, 0x80) + assertEquals(MediaFact.Unknown, SampleRandomAccessReader.read(VideoCodec.H264, data, data.size)) + } finally { unmockkStatic(NalUnitUtil::class) } + } +} From 0a10181e59e0c8c84f4291080f401f3818c389e2 Mon Sep 17 00:00:00 2001 From: Eugene Morozov Date: Tue, 6 Oct 2026 10:46:49 -0700 Subject: [PATCH 12/49] Fix sample inspection review findings --- .../standardization/IsoContainerMetadata.kt | 17 +++++++-- .../internal/standardization/MediaFacts.kt | 3 +- .../standardization/MediaSampleInspector.kt | 25 ++++++++---- .../IsoContainerMetadataTests.kt | 30 ++++++++++++++- .../MediaSampleInspectorTests.kt | 38 +++++++++++++++++-- 5 files changed, 96 insertions(+), 17 deletions(-) diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/IsoContainerMetadata.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/IsoContainerMetadata.kt index 35e7e1c0..557f3c7a 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/IsoContainerMetadata.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/IsoContainerMetadata.kt @@ -143,6 +143,15 @@ internal object IsoContainerMetadataReader { /** Classifies edit structure only; applying it to the presentation timeline is separate work. */ private fun edits(children: List): MediaFact { + val metadataBoxCount = boxCount + return try { classifyEdits(children) } catch (_: Exception) { + // A bad edit list must not discard track identity, orientation or Dolby signaling. + boxCount = metadataBoxCount + MediaFact.Unknown + } + } + + private fun classifyEdits(children: List): MediaFact { val editBoxes = children.filter { it.type == "edts" } if (editBoxes.isEmpty()) return MediaFact.Known(EditList.None) if (editBoxes.size != 1) return MediaFact.Unknown @@ -158,14 +167,16 @@ internal object IsoContainerMetadataReader { val entrySize = if (version == 0) 12 else 20 require(count <= 64 && list.end - list.payload == 8 + count * entrySize) if (count == 0L) return MediaFact.Known(EditList.None) - var simple = count == 1L + var classification = if (count == 1L) EditList.None else EditList.Complex repeat(count.toInt()) { val duration = if (version == 0) input.readInt().toLong() and 0xffffffffL else input.readLong() val mediaTime = if (version == 0) input.readInt().toLong() else input.readLong() val rate = input.readInt() - if (duration <= 0 || mediaTime < 0 || rate != 0x00010000) simple = false + if (duration <= 0) return MediaFact.Unknown + if (mediaTime < 0 || rate != 0x00010000) classification = EditList.Complex + else if (count == 1L && mediaTime > 0) classification = EditList.Simple } - return MediaFact.Known(if (simple) EditList.Simple else EditList.Complex) + return MediaFact.Known(classification) } private fun orientation(box: IsoMetadataBox): MediaFact { diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/MediaFacts.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/MediaFacts.kt index fa32e5b7..1b7605bf 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/MediaFacts.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/MediaFacts.kt @@ -51,7 +51,8 @@ internal data class TimestampFacts( val firstPresentationSeconds: Double, val lastPresentationSeconds: Double, val sampleCount: Long, - val isMonotonic: Boolean, + /** False is normal for B-frame reordering and does not imply invalid presentation timestamps. */ + val presentationOrderMatchesDecodeOrder: Boolean, ) /** diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/MediaSampleInspector.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/MediaSampleInspector.kt index 8b53a5ca..ed2fb74c 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/MediaSampleInspector.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/MediaSampleInspector.kt @@ -77,7 +77,10 @@ internal class MediaSampleInspector(private val limits: SampleScanLimits = Sampl // API 23–27 has no size query. Cap the native buffer's capacity as well as its limit. val remaining = limits.maximumReadBytes - bytes val readBuffer = if (remaining < data.size) ByteBuffer.wrap(data, 0, remaining.toInt()).slice() else buffer - val size = extractor.readSampleData(readBuffer, 0) + val size = try { extractor.readSampleData(readBuffer, 0) } catch (_: IllegalArgumentException) { + // Legacy extractors signal an insufficient sample buffer with IllegalArgumentException. + return result(if (apiLevel < 28) SampleScanStatus.LimitExceeded else SampleScanStatus.Unreadable) + } if (size <= 0 || size > data.size || size.toLong() > limits.maximumReadBytes - bytes || (expectedSize != null && size.toLong() != expectedSize)) return result(SampleScanStatus.LimitExceeded) bytes += size @@ -90,8 +93,13 @@ internal class MediaSampleInspector(private val limits: SampleScanLimits = Sampl if (isCancelled()) return result(SampleScanStatus.Cancelled) if (System.nanoTime() - started >= limits.maximumElapsedNanos) return result(SampleScanStatus.LimitExceeded) // Match the shared video-edit policy. Audio edits remain separate track observations. - val edit = metadata.isoTracks.valueOrNull?.getOrNull(track.containerIndex.valueOrNull ?: -1)?.editList - ?: MediaFact.Unknown + val edit = when (metadata.container.valueOrNull) { + ContainerKind.IsoBaseMedia -> metadata.isoTracks.valueOrNull + ?.getOrNull(track.containerIndex.valueOrNull ?: -1)?.editList ?: MediaFact.Unknown + // This proves absence of ISO edits only, not an effective Matroska presentation timeline. + ContainerKind.Matroska -> MediaFact.Known(EditList.None) + else -> MediaFact.Unknown + } val facts = VideoSampleFactsReader.read(samples, track.reportedDurationSeconds, metadata.facts).copy(editList = edit) if (isCancelled()) return result(SampleScanStatus.Cancelled) if (System.nanoTime() - started >= limits.maximumElapsedNanos) return result(SampleScanStatus.LimitExceeded) @@ -135,24 +143,25 @@ internal object SampleRandomAccessReader { if (next - start < 2 || (type == 5 && data[start].toInt() and 96 == 0)) return MediaFact.Unknown slices++; idr = idr && type == 5; hasIdr = hasIdr || type == 5 } - 6, 7, 8, 9, 12 -> Unit - else -> return MediaFact.Unknown + 2, 3, 4 -> return MediaFact.Unknown + else -> Unit // Non-VCL NALs do not establish or invalidate random-access kind. } } VideoCodec.Hevc -> { - if (next - start < 3 || data[start].toInt() and 1 != 0 || + if (next - start < 2 || data[start].toInt() and 1 != 0 || data[start + 1].toInt() and 248 != 0 || data[start + 1].toInt() and 7 == 0) return MediaFact.Unknown val type = (data[start].toInt() and 126) shr 1 when (type) { in 0..9, in 16..21 -> { + if (next - start < 3) return MediaFact.Unknown slices++ idr = idr && type in 19..20 hasIdr = hasIdr || type in 19..20 if (type in 16..18 || type == 21) open = true } - 32, 33, 34, 35, 38, 39, 40 -> Unit - else -> return MediaFact.Unknown + in 10..15, in 22..31 -> return MediaFact.Unknown + else -> Unit // Includes EOS/EOB and Dolby RPU signaling; HDR eligibility is separate. } } else -> return MediaFact.Unknown diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/IsoContainerMetadataTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/IsoContainerMetadataTests.kt index 12f6cd53..0ec49dc8 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/IsoContainerMetadataTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/IsoContainerMetadataTests.kt @@ -12,9 +12,10 @@ class IsoContainerMetadataTests : AbsRobolectricTest() { private fun int(value: Int) = ByteBuffer.allocate(4).putInt(value).array() private fun box(type: String, payload: ByteArray) = int(payload.size + 8) + type.toByteArray() + payload private fun track(handler: String = "vide", entry: String = "hvc1", extension: ByteArray = byteArrayOf(), - edits: ByteArray = byteArrayOf(), + edits: ByteArray = byteArrayOf(), trackId: Int = 0, matrix: IntArray = intArrayOf(65536, 0, 0, 0, 65536, 0, 0, 0, 0x40000000)): ByteArray { val tkhd = ByteArray(84) + int(trackId).copyInto(tkhd, 12) for (i in matrix.indices) int(matrix[i]).copyInto(tkhd, 40 + i * 4) val hdlr = ByteArray(8) + handler.toByteArray() val description = ByteArray(if (handler == "vide") 78 else 28) @@ -61,6 +62,9 @@ class IsoContainerMetadataTests : AbsRobolectricTest() { assertEquals(MediaFact.Known(EditList.None), classification(byteArrayOf())) assertEquals(MediaFact.Known(EditList.None), classification(edit(entries = emptyList()))) for (version in 0..1) { + assertEquals(MediaFact.Unknown, classification(edit(version, listOf(0L to 0L)))) + assertEquals(MediaFact.Unknown, classification(edit(version, listOf(3000L to 0L, 0L to 1000L)))) + assertEquals(MediaFact.Known(EditList.None), classification(edit(version, listOf(3000L to 0L)))) assertEquals(MediaFact.Known(EditList.Simple), classification(edit(version, listOf(3000L to 123L)))) assertEquals(MediaFact.Known(EditList.Complex), classification(edit(version, listOf(3000L to -1L)))) } @@ -68,7 +72,29 @@ class IsoContainerMetadataTests : AbsRobolectricTest() { assertEquals(MediaFact.Known(EditList.Complex), classification(edit(rate = 0))) assertEquals(MediaFact.Unknown, classification(edit(version = 2))) assertEquals(MediaFact.Unknown, classification(edit() + edit())) - assertEquals(MediaFact.Unknown, read(box("moov", track(edits = box("edts", box("elst", ByteArray(5))))))) + assertEquals(MediaFact.Unknown, classification(box("edts", box("elst", ByteArray(5))))) + } + + @Test fun malformedOrOversizedEditsPreserveAllOtherTrackMetadata() { + val quarterTurn = intArrayOf(0, 65536, 0, -65536, 0, 0, 1920 * 65536, 0, 0x40000000) + val tooManyChildren = box("edts", List(4097) { box("free", byteArrayOf()) }.fold(byteArrayOf()) { a, b -> a + b }) + for (edits in listOf( + box("edts", box("elst", ByteArray(5))), + edit(entries = List(65) { 3000L to 0L }), + box("edts", int(100) + "elst".toByteArray() + ByteArray(4)), + tooManyChildren)) { + val tracks = read(box("moov", track(edits = edits, trackId = 7, matrix = quarterTurn, + extension = box("dvvC", ByteArray(4))) + track("soun", "mp4a", trackId = 8))).valueOrNull!! + assertEquals(listOf("vide", "soun"), tracks.map { it.handler }) + val video = tracks.first() + assertEquals(MediaFact.Known(7), video.trackId) + assertEquals(MediaFact.Known(270), video.rotationDegrees) + assertEquals(listOf("hvc1"), video.sampleEntries) + assertTrue(video.hasSimpleSampleGeometry) + assertTrue(video.hasDolbyVisionConfiguration) + assertEquals(MediaFact.Unknown, video.editList) + assertEquals(MediaFact.Known(8), tracks.last().trackId) + } } @Test fun doesNotClaimSimpleGeometryForReflectionShearOrScale() { diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/MediaSampleInspectorTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/MediaSampleInspectorTests.kt index f828848f..82ca0ce2 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/MediaSampleInspectorTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/MediaSampleInspectorTests.kt @@ -30,6 +30,7 @@ class MediaSampleInspectorTests : AbsRobolectricTest() { assertEquals(MediaFact.Known(500L), facts.maximumGopByteSize) assertEquals(MediaFact.Known(2.0), facts.maximumKeyframeIntervalSeconds) assertEquals(idr, facts.gopStructure) + assertTrue(facts.timestamps.valueOrNull!!.presentationOrderMatchesDecodeOrder) assertEquals(MediaFact.Unknown, facts.durationSeconds) assertEquals(MediaFact.Unknown, facts.audioVideoStartOffsetSeconds) } @@ -38,6 +39,7 @@ class MediaSampleInspectorTests : AbsRobolectricTest() { val facts = facts(listOf(sample(0, true), sample(2_000_000), sample(1_000_000))) assertEquals(MediaFact.Known(Cadence.Constant), facts.cadence) assertEquals(MediaFact.Known(TimestampFacts(0.0, 2.0, 3, false)), facts.timestamps) + assertFalse(facts.timestamps.valueOrNull!!.presentationOrderMatchesDecodeOrder) } @Test fun variableIntervalsRemainVariableAndNegativePrerollIsObserved() { @@ -109,6 +111,21 @@ class MediaSampleInspectorTests : AbsRobolectricTest() { assertEquals(MediaFact.Unknown, SampleRandomAccessReader.read(VideoCodec.Hevc, data, data.size)) } + @Test fun nonVclNalsDoNotHideIdrButMalformedAndReservedHeadersStayUnknown() { + val avc = nal(0x65, 0x80) + nal(10) + nal(11) + assertEquals(idr, SampleRandomAccessReader.read(VideoCodec.H264, avc, avc.size)) + val hevc = nal(0x26, 1, 0x80) + nal(62 shl 1, 1, 0x80) + nal(36 shl 1, 1) + nal(37 shl 1, 1) + assertEquals(idr, SampleRandomAccessReader.read(VideoCodec.Hevc, hevc, hevc.size)) + for (extra in listOf(nal(0xfc, 1), nal(0x7c), nal(0x7d, 1), nal(0x7c, 0), nal(44, 1, 0x80))) { + val data = nal(0x26, 1, 0x80) + extra + assertEquals(MediaFact.Unknown, SampleRandomAccessReader.read(VideoCodec.Hevc, data, data.size)) + } + val partition = nal(0x65, 0x80) + nal(2, 0x80) + assertEquals(MediaFact.Unknown, SampleRandomAccessReader.read(VideoCodec.H264, partition, partition.size)) + val nonVclOnly = nal(62 shl 1, 1, 0x80) + assertEquals(MediaFact.Unknown, SampleRandomAccessReader.read(VideoCodec.Hevc, nonVclOnly, nonVclOnly.size)) + } + @Test fun threeBytePrefixesAndSeveralSlicesAreSupported() { val data = byteArrayOf(0, 0, 1, 0x65, 0x80.toByte(), 0, 0, 1, 0x65, 0x80.toByte()) assertEquals(idr, SampleRandomAccessReader.read(VideoCodec.H264, data, data.size)) @@ -120,7 +137,8 @@ class MediaSampleInspectorTests : AbsRobolectricTest() { MediaFacts(videoCodec = MediaFact.Known(VideoCodec.H264), videoTrackCount = MediaFact.Known(1)), 0) private fun scan(limits: SampleScanLimits = SampleScanLimits(), cancel: () -> Boolean = { false }, - reader: ((ByteBuffer) -> Int)? = null, reportedSize: Long = 6, flagsOverride: Int? = null): MediaSampleInspection { + reader: ((ByteBuffer) -> Int)? = null, reportedSize: Long = 6, flagsOverride: Int? = null, + input: MediaMetadataInspection = metadata()): MediaSampleInspection { mockkConstructor(MediaExtractor::class) var index = 0 var released = false @@ -138,7 +156,7 @@ class MediaSampleInspectorTests : AbsRobolectricTest() { every { anyConstructed().advance() } answers { ++index < 3 } every { anyConstructed().release() } answers { released = true } return try { - MediaSampleInspector(limits).inspect(File("unused"), metadata(), cancel).also { + MediaSampleInspector(limits).inspect(File("unused"), input, cancel).also { assertTrue(released) } } finally { unmockkConstructor(MediaExtractor::class) } @@ -173,7 +191,10 @@ class MediaSampleInspectorTests : AbsRobolectricTest() { assertEquals(16, capacity) assertEquals(SampleScanStatus.LimitExceeded, result.status) assertEquals(metadata().facts, result.facts) - assertEquals(SampleScanStatus.Unreadable, scan(reader = { throw IllegalArgumentException() }).status) + val oversized = scan(reader = { throw IllegalArgumentException() }) + assertEquals(SampleScanStatus.LimitExceeded, oversized.status) + assertEquals(metadata().facts, oversized.facts) + assertEquals(SampleScanStatus.Unreadable, scan(reader = { throw IllegalStateException() }).status) } @Test @Config(sdk = [28]) fun api28ChecksSizeBeforeReadAndRejectsInconsistentCount() { @@ -183,6 +204,17 @@ class MediaSampleInspectorTests : AbsRobolectricTest() { assertFalse(read) assertEquals(SampleScanStatus.LimitExceeded, scan(reportedSize = 7).status) assertEquals(SampleScanStatus.Complete, scan().status) + assertEquals(SampleScanStatus.Unreadable, scan(reader = { throw IllegalArgumentException() }).status) + } + + @Test fun matroskaHasNoIsoEditListWhileOtherContainersStayUnknown() { + val result = scan(input = metadata().copy(container = MediaFact.Known(ContainerKind.Matroska))) + assertEquals(SampleScanStatus.Complete, result.status) + assertEquals(MediaFact.Known(EditList.None), result.facts.editList) + assertEquals(MediaFact.Unknown, result.facts.durationSeconds) + assertEquals(MediaFact.Unknown, result.facts.audioVideoStartOffsetSeconds) + for (container in listOf(MediaFact.Unknown, MediaFact.Known(ContainerKind.Other))) + assertEquals(MediaFact.Unknown, scan(input = metadata().copy(container = container)).facts.editList) } @Test fun encryptedAndPartialFramesCannotSupplySampleFacts() { From c9f486aafca913ce3a62e294b0f62b015c072ab3 Mon Sep 17 00:00:00 2001 From: Eugene Morozov Date: Fri, 16 Oct 2026 11:26:02 -0700 Subject: [PATCH 13/49] Limit ISO inspection to sample edit-list evidence --- .../standardization/IsoContainerMetadata.kt | 108 +++---------- .../standardization/MediaSampleInspector.kt | 5 +- .../IsoContainerMetadataTests.kt | 142 ++---------------- .../MediaSampleInspectorTests.kt | 2 +- 4 files changed, 35 insertions(+), 222 deletions(-) diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/IsoContainerMetadata.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/IsoContainerMetadata.kt index 557f3c7a..f5d400a7 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/IsoContainerMetadata.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/IsoContainerMetadata.kt @@ -6,32 +6,16 @@ import java.nio.ByteBuffer internal data class IsoMetadataBox(val type: String, val payload: Long, val end: Long) -internal object IsoSampleEntries { - val avc = setOf("avc1", "avc3", "dva1", "dvav") - val hevc = setOf("hvc1", "hev1", "dvh1", "dvhe") - val dolby = setOf("dvh1", "dvhe", "dva1", "dvav") - val visual = avc + hevc -} - internal data class IsoTrackMetadata( val handler: String?, val trackId: MediaFact, - val sampleEntries: List, - val hasDolbyVisionConfiguration: Boolean, - val rotationDegrees: MediaFact, - val hasSimpleSampleGeometry: Boolean, - val editList: MediaFact = MediaFact.Unknown, -) { - val hasSimpleOrientation: Boolean get() = rotationDegrees != MediaFact.Unknown - fun matches(kind: TrackKind): Boolean = (handler == "vide" && kind == TrackKind.Video) || - (handler == "soun" && kind == TrackKind.Audio) -} + val editList: MediaFact, +) -/** Seeks past media payloads; reads track identity, simple orientation and Dolby Vision signaling. */ +/** Edit-list inspection for sample/timeline analysis only; metadata uses MediaFormat directly. */ internal object IsoContainerMetadataReader { private const val MAX_BOXES = 4096 private const val MAX_TOP_LEVEL_BOXES = 131072 - private const val MAX_ENTRIES = 16 fun read(file: File): MediaFact> = try { RandomAccessFile(file, "r").use { input -> @@ -51,6 +35,13 @@ internal object IsoContainerMetadataReader { } } catch (_: Exception) { MediaFact.Unknown } + fun videoEdit(file: File, trackId: Int?): MediaFact { + if (trackId == null) return MediaFact.Unknown + return read(file).valueOrNull?.singleOrNull { + it.handler == "vide" && it.trackId.valueOrNull == trackId + }?.editList ?: MediaFact.Unknown + } + private class Reader(val input: RandomAccessFile) { var boxCount = 0 val boxHeader = ByteArray(16) @@ -86,59 +77,19 @@ internal object IsoContainerMetadataReader { val children = boxes(track.payload, track.end).toList() val tkhd = children.singleOrNull { it.type == "tkhd" } val mdia = children.single { it.type == "mdia" } - val media = boxes(mdia.payload, mdia.end).toList() - val handler = media.singleOrNull { it.type == "hdlr" }?.let { + val handler = boxes(mdia.payload, mdia.end).singleOrNull { it.type == "hdlr" }?.let { require(it.end - it.payload >= 12) input.seek(it.payload + 8) - fourCc() + ByteArray(4).also { input.readFully(it) }.toString(Charsets.US_ASCII) } - val trackId = tkhd?.let { + val id = tkhd?.let { input.seek(it.payload) - val version = input.readUnsignedByte() - val offset = when (version) { 0 -> 12; 1 -> 20; else -> -1 } - if (offset < 0 || it.end - it.payload < offset + 4) MediaFact.Unknown else { - input.seek(it.payload + offset) - positiveCode(input.readInt()) - } + val offset = when (input.readUnsignedByte()) { 0 -> 12; 1 -> 20; else -> return@let MediaFact.Unknown } + require(it.end - it.payload >= offset + 4) + input.seek(it.payload + offset) + positiveCode(input.readInt()) } ?: MediaFact.Unknown - val orientation = tkhd?.let(::orientation) ?: MediaFact.Unknown - if (handler !in listOf("vide", "soun")) - return IsoTrackMetadata(handler, trackId, emptyList(), false, orientation, false) - val minf = media.single { it.type == "minf" } - val stbl = boxes(minf.payload, minf.end).single { it.type == "stbl" } - val tables = boxes(stbl.payload, stbl.end).toList() - val stsd = tables.single { it.type == "stsd" } - require(stsd.end - stsd.payload >= 8) - input.seek(stsd.payload + 4) - val count = input.readInt() - require(count in 1..MAX_ENTRIES) - val entries = boxes(stsd.payload + 8, stsd.end).toList() - require(entries.size == count) - var dolby = false - var simpleGeometry = entries.size == 1 - if (handler == "vide") { - for (entry in entries) { - if (entry.type in IsoSampleEntries.dolby) dolby = true - if (entry.type in IsoSampleEntries.visual) { - require(entry.end - entry.payload >= 78) - val extensions = boxes(entry.payload + 78, entry.end, allowQuickTimeTerminator = true).toList() - if (extensions.any { it.type in setOf("dvcC", "dvvC", "dvwC") }) dolby = true - if (extensions.any { it.type == "clap" }) simpleGeometry = false - val aspects = extensions.filter { it.type == "pasp" } - if (aspects.size > 1) simpleGeometry = false - for (aspect in aspects) { - require(aspect.end - aspect.payload == 8L) - input.seek(aspect.payload) - val horizontal = input.readInt() - val vertical = input.readInt() - // Only square pixels are supported; unusual geometry keeps original-file fallback. - if (horizontal == 0 || horizontal != vertical) simpleGeometry = false - } - } else simpleGeometry = false - } - } - return IsoTrackMetadata(handler, trackId, entries.map { it.type }, dolby, orientation, simpleGeometry, - edits(children)) + return IsoTrackMetadata(handler, id, edits(children)) } /** Classifies edit structure only; applying it to the presentation timeline is separate work. */ @@ -179,28 +130,5 @@ internal object IsoContainerMetadataReader { return MediaFact.Known(classification) } - private fun orientation(box: IsoMetadataBox): MediaFact { - input.seek(box.payload) - val version = input.readUnsignedByte() - val offset = when (version) { 0 -> 40; 1 -> 52; else -> return MediaFact.Unknown } - require(box.end - box.payload >= offset + 36) - input.seek(box.payload + offset) - val matrix = IntArray(9) { input.readInt() } - if (matrix[2] != 0 || matrix[5] != 0 || matrix[8] != 0x40000000) return MediaFact.Unknown - // Same normalized transform convention as Swift. Android's reported angle is negated separately. - return when (listOf(matrix[0], matrix[1], matrix[3], matrix[4])) { - listOf(65536, 0, 0, 65536) -> MediaFact.Known(0) - listOf(0, 65536, -65536, 0) -> MediaFact.Known(270) - listOf(-65536, 0, 0, -65536) -> MediaFact.Known(180) - listOf(0, -65536, 65536, 0) -> MediaFact.Known(90) - else -> MediaFact.Unknown - } - } - - private fun fourCc(): String { - val bytes = ByteArray(4) - input.readFully(bytes) - return String(bytes, Charsets.US_ASCII) - } } } diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/MediaSampleInspector.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/MediaSampleInspector.kt index ed2fb74c..0fbd560d 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/MediaSampleInspector.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/MediaSampleInspector.kt @@ -54,8 +54,6 @@ internal class MediaSampleInspector(private val limits: SampleScanLimits = Sampl if (metadata.facts.videoTrackCount != MediaFact.Known(1) || metadata.facts.videoCodec.valueOrNull !in listOf(VideoCodec.H264, VideoCodec.Hevc)) return result(SampleScanStatus.Unsupported) - if (metadata.container == MediaFact.Known(ContainerKind.IsoBaseMedia) && - track.containerIndex == MediaFact.Unknown) return result(SampleScanStatus.Unsupported) val codec = metadata.facts.videoCodec.valueOrNull!! val extractor = MediaExtractor() return try { @@ -94,8 +92,7 @@ internal class MediaSampleInspector(private val limits: SampleScanLimits = Sampl if (System.nanoTime() - started >= limits.maximumElapsedNanos) return result(SampleScanStatus.LimitExceeded) // Match the shared video-edit policy. Audio edits remain separate track observations. val edit = when (metadata.container.valueOrNull) { - ContainerKind.IsoBaseMedia -> metadata.isoTracks.valueOrNull - ?.getOrNull(track.containerIndex.valueOrNull ?: -1)?.editList ?: MediaFact.Unknown + ContainerKind.IsoBaseMedia -> IsoContainerMetadataReader.videoEdit(file, track.sourceTrackId.valueOrNull) // This proves absence of ISO edits only, not an effective Matroska presentation timeline. ContainerKind.Matroska -> MediaFact.Known(EditList.None) else -> MediaFact.Unknown diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/IsoContainerMetadataTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/IsoContainerMetadataTests.kt index 0ec49dc8..b2301e62 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/IsoContainerMetadataTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/IsoContainerMetadataTests.kt @@ -11,43 +11,14 @@ import java.nio.ByteBuffer class IsoContainerMetadataTests : AbsRobolectricTest() { private fun int(value: Int) = ByteBuffer.allocate(4).putInt(value).array() private fun box(type: String, payload: ByteArray) = int(payload.size + 8) + type.toByteArray() + payload - private fun track(handler: String = "vide", entry: String = "hvc1", extension: ByteArray = byteArrayOf(), - edits: ByteArray = byteArrayOf(), trackId: Int = 0, - matrix: IntArray = intArrayOf(65536, 0, 0, 0, 65536, 0, 0, 0, 0x40000000)): ByteArray { - val tkhd = ByteArray(84) - int(trackId).copyInto(tkhd, 12) - for (i in matrix.indices) int(matrix[i]).copyInto(tkhd, 40 + i * 4) - val hdlr = ByteArray(8) + handler.toByteArray() - val description = ByteArray(if (handler == "vide") 78 else 28) - if (handler == "vide") { - ByteBuffer.wrap(description).putShort(24, 1920.toShort()).putShort(26, 1080.toShort()) - int(1920 * 65536).copyInto(tkhd, 76) - int(1080 * 65536).copyInto(tkhd, 80) - } - val sampleEntry = box(entry, description + extension) - val stsd = box("stsd", ByteArray(4) + int(1) + sampleEntry) - return box("trak", box("tkhd", tkhd) + edits + - box("mdia", box("hdlr", hdlr) + box("minf", box("stbl", stsd)))) - } - + private fun track(id: Int = 1, edits: ByteArray = byteArrayOf(), handler: String = "vide") = + box("trak", box("tkhd", ByteArray(12) + int(id)) + edits + + box("mdia", box("hdlr", ByteArray(8) + handler.toByteArray()))) private fun read(data: ByteArray): MediaFact> { - val file = File.createTempFile("iso-metadata", ".mp4") + val file = File.createTempFile("iso-edits", ".mp4") return try { file.writeBytes(data); IsoContainerMetadataReader.read(file) } finally { file.delete() } } - @Test fun readsContainerOrderAndDolbyConfigurationInOrdinaryHevcEntry() { - val tracks = read(box("ftyp", "isom".toByteArray()) + box("moov", - track(extension = box("dvvC", byteArrayOf(1, 0, 0, 0))) + track("soun", "mp4a"))).valueOrNull!! - assertEquals(listOf("vide", "soun"), tracks.map { it.handler }) - assertEquals(listOf("hvc1"), tracks[0].sampleEntries) - assertTrue(tracks[0].hasDolbyVisionConfiguration) - assertTrue(tracks[0].hasSimpleOrientation) - } - - @Test fun readsDolbySampleEntryEvenWithoutConfigurationChild() { - assertTrue(read(box("moov", track(entry = "dvh1"))).valueOrNull!![0].hasDolbyVisionConfiguration) - } - private fun edit(version: Int = 0, entries: List> = listOf(3000L to 0L), rate: Int = 65536): ByteArray { val payload = byteArrayOf(version.toByte(), 0, 0, 0) + int(entries.size) + entries.fold(byteArrayOf()) { bytes, (duration, time) -> @@ -75,102 +46,19 @@ class IsoContainerMetadataTests : AbsRobolectricTest() { assertEquals(MediaFact.Unknown, classification(box("edts", box("elst", ByteArray(5))))) } - @Test fun malformedOrOversizedEditsPreserveAllOtherTrackMetadata() { - val quarterTurn = intArrayOf(0, 65536, 0, -65536, 0, 0, 1920 * 65536, 0, 0x40000000) - val tooManyChildren = box("edts", List(4097) { box("free", byteArrayOf()) }.fold(byteArrayOf()) { a, b -> a + b }) - for (edits in listOf( - box("edts", box("elst", ByteArray(5))), - edit(entries = List(65) { 3000L to 0L }), - box("edts", int(100) + "elst".toByteArray() + ByteArray(4)), - tooManyChildren)) { - val tracks = read(box("moov", track(edits = edits, trackId = 7, matrix = quarterTurn, - extension = box("dvvC", ByteArray(4))) + track("soun", "mp4a", trackId = 8))).valueOrNull!! - assertEquals(listOf("vide", "soun"), tracks.map { it.handler }) - val video = tracks.first() - assertEquals(MediaFact.Known(7), video.trackId) - assertEquals(MediaFact.Known(270), video.rotationDegrees) - assertEquals(listOf("hvc1"), video.sampleEntries) - assertTrue(video.hasSimpleSampleGeometry) - assertTrue(video.hasDolbyVisionConfiguration) - assertEquals(MediaFact.Unknown, video.editList) - assertEquals(MediaFact.Known(8), tracks.last().trackId) - } - } - - @Test fun doesNotClaimSimpleGeometryForReflectionShearOrScale() { - for (matrix in listOf( - intArrayOf(-65536, 0, 0, 0, 65536, 0, 0, 0, 0x40000000), - intArrayOf(65536, 100, 0, 0, 65536, 0, 0, 0, 0x40000000), - intArrayOf(131072, 0, 0, 0, 65536, 0, 0, 0, 0x40000000))) { - assertFalse(read(box("moov", track(matrix = matrix))).valueOrNull!![0].hasSimpleOrientation) - } - } - - @Test fun recognizesQuarterRotationWithTranslation() { - val matrix = intArrayOf(0, 65536, 0, -65536, 0, 0, 1920 * 65536, 0, 0x40000000) - assertTrue(read(box("moov", track(matrix = matrix))).valueOrNull!![0].hasSimpleOrientation) - } - - @Test fun unsupportedGeometryStaysUnknownAndSquarePixelsAreAccepted() { - for (extension in listOf(box("clap", ByteArray(32)), box("pasp", int(2) + int(1)), - box("pasp", int(1) + int(1)) + box("pasp", int(1) + int(1)))) { - assertFalse(read(box("moov", track(extension = extension))).valueOrNull!![0].hasSimpleSampleGeometry) - } - assertTrue(read(box("moov", track(extension = box("pasp", int(1) + int(1))))) - .valueOrNull!![0].hasSimpleSampleGeometry) - } - - @Test fun malformedSizesAndBoxLimitsStayUnknown() { - for (data in listOf(byteArrayOf(), int(4096) + "moov".toByteArray(), - int(4) + "moov".toByteArray(), int(1) + "moov".toByteArray() + ByteArray(8) { -1 })) { - assertEquals(MediaFact.Unknown, read(data)) - } - val many = ByteArray(4097 * 8) - val free = box("free", byteArrayOf()) - for (i in 0..4096) free.copyInto(many, i * 8) - assertEquals(MediaFact.Unknown, read(many)) - } - - @Test fun permitsOnlyZeroQuickTimeSampleDescriptionTerminator() { - assertTrue(read(box("moov", track(extension = box("dvvC", ByteArray(4)) + ByteArray(4)))) - .valueOrNull!![0].hasDolbyVisionConfiguration) - assertEquals(MediaFact.Unknown, read(box("moov", track(extension = int(1))))) + @Test fun malformedEditsDoNotDiscardTrackIdentityAndDuplicateIdsStayUnknown() { + val data = box("moov", track(7, box("edts", box("elst", ByteArray(5)))) + track(8, handler = "soun")) + val tracks = read(data).valueOrNull!! + assertEquals(listOf("vide", "soun"), tracks.map { it.handler }) + assertEquals(MediaFact.Known(7), tracks[0].trackId) + assertEquals(MediaFact.Unknown, tracks[0].editList) + assertEquals(MediaFact.Known(8), tracks[1].trackId) + assertEquals(MediaFact.Unknown, read(box("moov", track() + track()))) } - @Test fun noTrackOrDuplicateMovieBoxDoesNotInventAbsence() { - assertEquals(MediaFact.Unknown, read(box("moov", byteArrayOf()))) + @Test fun malformedBoxesAndMissingMoviesRemainUnknown() { + assertEquals(MediaFact.Unknown, read(byteArrayOf())) + assertEquals(MediaFact.Unknown, read(int(100) + "moov".toByteArray())) assertEquals(MediaFact.Unknown, read(box("moov", track()) + box("moov", track()))) } - - @Test fun everyUnitMatrixHasTheSharedNormalizedDegreeValue() { - for ((axes, degrees) in listOf( - listOf(65536, 0, 0, 65536) to 0, - listOf(0, 65536, -65536, 0) to 270, - listOf(-65536, 0, 0, -65536) to 180, - listOf(0, -65536, 65536, 0) to 90)) { - val matrix = intArrayOf(axes[0], axes[1], 0, axes[2], axes[3], 0, 1920 * 65536, 0, 0x40000000) - assertEquals(MediaFact.Known(degrees), read(box("moov", track(matrix = matrix))).valueOrNull!![0].rotationDegrees) - } - } - - @Test fun fragmentHeadersDoNotConsumeMovieMetadataBudget() { - val fragment = box("moof", byteArrayOf()) + box("mdat", byteArrayOf()) - val many = ByteArray(7200 * fragment.size) - repeat(7200) { fragment.copyInto(many, it * fragment.size) } - val movie = box("moov", track()) - assertNotEquals(MediaFact.Unknown, read(movie + many)) - assertNotEquals(MediaFact.Unknown, read(many + movie)) - assertEquals(MediaFact.Unknown, read(movie + many + movie)) - assertEquals(MediaFact.Unknown, read(movie + byteArrayOf(1))) - } - - @Test fun quickTimeLeadingBoxesAreRecognizedAsIso() { - for (type in listOf("skip", "uuid", "pnot")) { - val file = File.createTempFile("iso-sniff", ".mov") - try { - file.writeBytes(box(type, ByteArray(16)) + box("moov", track())) - assertEquals(MediaFact.Known(ContainerKind.IsoBaseMedia), MediaMetadataInspector().container(file)) - } finally { file.delete() } - } - } } diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/MediaSampleInspectorTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/MediaSampleInspectorTests.kt index 82ca0ce2..a340cecd 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/MediaSampleInspectorTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/MediaSampleInspectorTests.kt @@ -133,7 +133,7 @@ class MediaSampleInspectorTests : AbsRobolectricTest() { private fun metadata() = MediaMetadataInspection(MediaFact.Known(ContainerKind.Other), listOf( TrackMetadata(0, MediaFact.Unknown, MediaFact.Unknown, TrackKind.Video, MediaFact.Known("video/avc"), - MediaFact.Known(3.0), MediaFact.Unknown, MediaFact.Unknown)), MediaFact.Unknown, + MediaFact.Known(3.0), MediaFact.Unknown, MediaFact.Unknown)), MediaFacts(videoCodec = MediaFact.Known(VideoCodec.H264), videoTrackCount = MediaFact.Known(1)), 0) private fun scan(limits: SampleScanLimits = SampleScanLimits(), cancel: () -> Boolean = { false }, From 0bb70e486260015cf91ab10601e0e521b88d244e Mon Sep 17 00:00:00 2001 From: Eugene Morozov Date: Tue, 6 Oct 2026 11:40:06 -0700 Subject: [PATCH 14/49] Validate generated media before upload --- .../standardization/IsoContainerMetadata.kt | 2 +- .../standardization/MediaMetadataInspector.kt | 2 +- .../standardization/MediaSampleInspector.kt | 9 +- .../StandardInputOutputValidator.kt | 179 +++++++++++++ .../standardization/StandardInputPlanner.kt | 1 + .../StandardInputTimelineInspector.kt | 253 ++++++++++++++++++ .../MediaSampleInspectorTests.kt | 14 +- .../StandardInputOutputValidatorTests.kt | 167 ++++++++++++ .../StandardInputTimelineInspectorTests.kt | 142 ++++++++++ 9 files changed, 763 insertions(+), 6 deletions(-) create mode 100644 library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputOutputValidator.kt create mode 100644 library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspector.kt create mode 100644 library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputOutputValidatorTests.kt create mode 100644 library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspectorTests.kt diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/IsoContainerMetadata.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/IsoContainerMetadata.kt index f5d400a7..b0ed8050 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/IsoContainerMetadata.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/IsoContainerMetadata.kt @@ -42,7 +42,7 @@ internal object IsoContainerMetadataReader { }?.editList ?: MediaFact.Unknown } - private class Reader(val input: RandomAccessFile) { + internal class Reader(val input: RandomAccessFile) { var boxCount = 0 val boxHeader = ByteArray(16) diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/MediaMetadataInspector.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/MediaMetadataInspector.kt index e415291b..3dc8901f 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/MediaMetadataInspector.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/MediaMetadataInspector.kt @@ -60,7 +60,7 @@ internal data class TrackMetadata( val audio: AudioTrackMetadata? = null, ) -/** Effective duration and A/V offset remain unknown until timeline validation is implemented. */ +/** Metadata alone does not prove duration or A/V offset; use StandardInputTimelineInspector. */ internal data class MediaMetadataInspection( val container: MediaFact, val tracks: List, diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/MediaSampleInspector.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/MediaSampleInspector.kt index 0fbd560d..250176b2 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/MediaSampleInspector.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/MediaSampleInspector.kt @@ -41,7 +41,12 @@ internal data class CompressedVideoSample( internal class MediaSampleInspector(private val limits: SampleScanLimits = SampleScanLimits(), private val apiLevel: Int = Build.VERSION.SDK_INT) { fun inspect(file: File, metadata: MediaMetadataInspection, - isCancelled: () -> Boolean = { Thread.currentThread().isInterrupted }): MediaSampleInspection { + isCancelled: () -> Boolean = { Thread.currentThread().isInterrupted }): MediaSampleInspection = + inspect(file, metadata, null, isCancelled) + + /** Output validation supplies proven video duration; source-only scans retain the observation. */ + fun inspect(file: File, metadata: MediaMetadataInspection, provenVideoDurationSeconds: MediaFact?, + isCancelled: () -> Boolean): MediaSampleInspection { val started = System.nanoTime() val samples = ArrayList() var bytes = 0L @@ -97,7 +102,7 @@ internal class MediaSampleInspector(private val limits: SampleScanLimits = Sampl ContainerKind.Matroska -> MediaFact.Known(EditList.None) else -> MediaFact.Unknown } - val facts = VideoSampleFactsReader.read(samples, track.reportedDurationSeconds, metadata.facts).copy(editList = edit) + val facts = VideoSampleFactsReader.read(samples, provenVideoDurationSeconds ?: track.reportedDurationSeconds, metadata.facts).copy(editList = edit) if (isCancelled()) return result(SampleScanStatus.Cancelled) if (System.nanoTime() - started >= limits.maximumElapsedNanos) return result(SampleScanStatus.LimitExceeded) result(SampleScanStatus.Complete, facts) diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputOutputValidator.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputOutputValidator.kt new file mode 100644 index 00000000..a7ec6f1e --- /dev/null +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputOutputValidator.kt @@ -0,0 +1,179 @@ +package com.mux.video.upload.internal.standardization + +import java.io.File +import java.util.concurrent.CancellationException +import kotlin.math.abs + +internal enum class OutputExpectation { + VideoPresence, VideoCodec, DimensionsAndOrientation, PixelFormat, DynamicRange, + Audio, Duration, AudioVideoStartOffset, Cadence, Timestamps, +} + +internal sealed interface OutputRejection { + data object EmptyOrUnreadable : OutputRejection + data object ChangedDuringInspection : OutputRejection + data class Metadata(val reason: MetadataFailure) : OutputRejection + data class SampleInspection(val status: SampleScanStatus) : OutputRejection + data class TimelineInspection(val status: SampleScanStatus) : OutputRejection + data class NonCompliant(val requirements: Set) : OutputRejection + data class InsufficientPolicyEvidence(val requirements: Set) : OutputRejection + data class InsufficientPlanEvidence(val expectations: Set) : OutputRejection + data class DoesNotMatchPlan(val expectations: Set) : OutputRejection +} + +/** Rejection permits original-file fallback; Cancelled must terminate preparation without upload. */ +internal sealed interface StandardInputOutputValidation { + data class Accepted(val facts: MediaFacts, val evaluation: PolicyEvaluation) : StandardInputOutputValidation + data class Rejected(val reason: OutputRejection, val evaluation: PolicyEvaluation? = null) : StandardInputOutputValidation + data object Cancelled : StandardInputOutputValidation +} + +/** Internal seam for the later adapter; never selects/deletes payloads or starts an upload. */ +internal class StandardInputOutputValidator( + private val evaluator: StandardInputPolicyEvaluator = StandardInputPolicyEvaluator(), + private val inspectMetadata: (File) -> MetadataInspectionResult = MediaMetadataInspector()::inspect, + private val inspectSamples: (File, MediaMetadataInspection, () -> Boolean, MediaFact) -> MediaSampleInspection = + { file, metadata, cancelled, duration -> MediaSampleInspector().inspect(file, metadata, duration, cancelled) }, + private val inspectTimeline: (File, MediaMetadataInspection, () -> Boolean) -> TimelineInspection = + { file, metadata, cancelled -> StandardInputTimelineInspector().inspect(file, metadata, cancelled) }, +) { + fun validateGeneratedOutput(file: File, source: MediaFacts, sourceTimeline: StandardInputTimelineFacts, + conversion: StandardInputConversion, + isCancelled: () -> Boolean = { Thread.currentThread().isInterrupted }): StandardInputOutputValidation { + fun rejected(reason: OutputRejection) = StandardInputOutputValidation.Rejected(reason) + return try { + if (isCancelled()) return StandardInputOutputValidation.Cancelled + if (!file.isFile || !file.canRead() || file.length() == 0L) return rejected(OutputRejection.EmptyOrUnreadable) + val initialSize = file.length() + val initialModified = file.lastModified() + val metadataResult = inspectMetadata(file) + if (isCancelled()) return StandardInputOutputValidation.Cancelled + val metadata = when (metadataResult) { + is MetadataInspectionResult.Success -> metadataResult.inspection + is MetadataInspectionResult.Failure -> return rejected(OutputRejection.Metadata(metadataResult.reason)) + } + val timeline = inspectTimeline(file, metadata, isCancelled) + if (isCancelled() || timeline.status == SampleScanStatus.Cancelled) return StandardInputOutputValidation.Cancelled + if (timeline.status != SampleScanStatus.Complete) return rejected(OutputRejection.TimelineInspection(timeline.status)) + val duration = timeline.timeline.videoDurationSeconds.valueOrNull + if (duration == null || !duration.isFinite() || duration <= 0) + return rejected(OutputRejection.InsufficientPlanEvidence(setOf(OutputExpectation.Duration))) + val samples = inspectSamples(file, metadata, isCancelled, timeline.timeline.videoDurationSeconds) + if (isCancelled() || samples.status == SampleScanStatus.Cancelled) return StandardInputOutputValidation.Cancelled + if (samples.status != SampleScanStatus.Complete) return rejected(OutputRejection.SampleInspection(samples.status)) + if (isCancelled()) return StandardInputOutputValidation.Cancelled + if (file.length() != initialSize || file.lastModified() != initialModified) + return rejected(OutputRejection.ChangedDuringInspection) + val result = validateFacts(samples.facts, source, sourceTimeline, timeline.timeline, conversion) + if (isCancelled()) StandardInputOutputValidation.Cancelled else result + } catch (_: CancellationException) { StandardInputOutputValidation.Cancelled } + catch (_: LinkageError) { + if (isCancelled()) StandardInputOutputValidation.Cancelled else rejected(OutputRejection.EmptyOrUnreadable) + } catch (_: Exception) { + if (isCancelled()) StandardInputOutputValidation.Cancelled else rejected(OutputRejection.EmptyOrUnreadable) + } + } + + fun validateFacts(output: MediaFacts, source: MediaFacts, sourceTimeline: StandardInputTimelineFacts, + outputTimeline: StandardInputTimelineFacts, conversion: StandardInputConversion): StandardInputOutputValidation { + val evaluation = evaluator.evaluate(output, conversion.selection, MediaRole.GeneratedOutput) + fun rejected(reason: OutputRejection) = StandardInputOutputValidation.Rejected(reason, evaluation) + if (evaluation.nonCompliantRequirements.isNotEmpty()) + return rejected(OutputRejection.NonCompliant(evaluation.nonCompliantRequirements)) + if (evaluation.unknownRequirements.isNotEmpty()) + return rejected(OutputRejection.InsufficientPolicyEvidence(evaluation.unknownRequirements)) + val missing = mutableSetOf() + val mismatches = mutableSetOf() + fun expect(fact: MediaFact, expected: T, field: OutputExpectation) { + val actual = fact.valueOrNull + if (actual == null) missing.add(field) else if (actual != expected) mismatches.add(field) + } + expect(output.videoTrackCount, 1, OutputExpectation.VideoPresence) + expect(output.videoCodec, conversion.outputCodec, OutputExpectation.VideoCodec) + // The plan's dimensions are display-oriented. The adapter must bake orientation into pixels. + expect(output.encodedDimensions, conversion.outputDimensions, OutputExpectation.DimensionsAndOrientation) + expect(output.displayDimensions, conversion.outputDimensions, OutputExpectation.DimensionsAndOrientation) + expect(output.rotationDegrees, 0, OutputExpectation.DimensionsAndOrientation) + expect(output.pixelFormat, conversion.outputPixelFormat, OutputExpectation.PixelFormat) + expect(output.dynamicRange, conversion.outputDynamicRange, OutputExpectation.DynamicRange) + if (!preservesAspectRatio(conversion.sourceDisplayDimensions, conversion.outputDimensions)) + mismatches.add(OutputExpectation.DimensionsAndOrientation) + + val sourceAudio = source.audioTracks.valueOrNull + val outputAudio = output.audioTracks.valueOrNull + val wantsAudio = when (conversion.outputAudio) { + OutputAudio.None -> false + OutputAudio.AacFromFirstTrack -> true + OutputAudio.AacFromFirstTrackIfPresent -> sourceAudio?.isNotEmpty() + } + if (wantsAudio == null || outputAudio == null) missing.add(OutputExpectation.Audio) + else if (outputAudio.size != if (wantsAudio) 1 else 0) mismatches.add(OutputExpectation.Audio) + else if (wantsAudio) { + val sourceFormat = sourceAudio?.firstOrNull()?.format?.valueOrNull + if (sourceFormat is AudioFormat.Aac && outputAudio.single().format.valueOrNull != sourceFormat) + mismatches.add(OutputExpectation.Audio) + } + val rate = output.frameRate.valueOrNull?.takeIf { it.isFinite() && it > 0 } + if (rate == null) missing.add(OutputExpectation.Cadence) + else if (!conversion.outputFrameRate.isFinite() || conversion.outputFrameRate <= 0 || + abs(rate - conversion.outputFrameRate) > conversion.outputFrameRate * 0.001) + mismatches.add(OutputExpectation.Cadence) + when (conversion.outputCadence) { + OutputCadence.ConstantFrameRate -> expect(output.cadence, Cadence.Constant, OutputExpectation.Cadence) + OutputCadence.PreserveSourceTimestamps -> { + val cadence = source.cadence.valueOrNull + if (cadence == null) missing.add(OutputExpectation.Cadence) + else expect(output.cadence, cadence, OutputExpectation.Cadence) + val sourceTimes = sourceTimeline.videoPresentationSeconds.valueOrNull + val outputTimes = outputTimeline.videoPresentationSeconds.valueOrNull + if (sourceTimes == null || outputTimes == null || sourceTimes.isEmpty() || outputTimes.isEmpty()) + missing.add(OutputExpectation.Timestamps) + else if (sourceTimes.size != outputTimes.size || sourceTimes.indices.any { + !sourceTimes[it].isFinite() || !outputTimes[it].isFinite() || abs(sourceTimes[it] - outputTimes[it]) > 2e-6 + }) mismatches.add(OutputExpectation.Timestamps) + } + } + fun compareDuration(sourceFact: MediaFact, outputFact: MediaFact) { + val sourceDuration = sourceFact.valueOrNull?.takeIf { it.isFinite() && it > 0 } + val outputDuration = outputFact.valueOrNull?.takeIf { it.isFinite() && it > 0 } + if (sourceDuration == null || outputDuration == null || rate == null) missing.add(OutputExpectation.Duration) + else if (abs(sourceDuration - outputDuration) > maxOf(MINIMUM_DURATION_DELTA, 1 / rate) + 1e-12) + mismatches.add(OutputExpectation.Duration) + } + compareDuration(sourceTimeline.durationSeconds, outputTimeline.durationSeconds) + compareDuration(sourceTimeline.videoDurationSeconds, outputTimeline.videoDurationSeconds) + if (wantsAudio == true) compareDuration(sourceTimeline.firstAudioDurationSeconds, outputTimeline.firstAudioDurationSeconds) + val sourceOffset = sourceTimeline.audioVideoStartOffset.valueOrNull + val outputOffset = outputTimeline.audioVideoStartOffset.valueOrNull + when { + sourceOffset == null || outputOffset == null -> missing.add(OutputExpectation.AudioVideoStartOffset) + sourceOffset is AudioVideoStartOffset.Seconds && outputOffset is AudioVideoStartOffset.Seconds -> { + if (!sourceOffset.value.isFinite() || !outputOffset.value.isFinite()) missing.add(OutputExpectation.AudioVideoStartOffset) + else if (abs(sourceOffset.value - outputOffset.value) > MAXIMUM_AV_START_DELTA + 1e-12) + mismatches.add(OutputExpectation.AudioVideoStartOffset) + } + sourceOffset != outputOffset -> mismatches.add(OutputExpectation.AudioVideoStartOffset) + } + return when { + mismatches.isNotEmpty() -> rejected(OutputRejection.DoesNotMatchPlan(mismatches)) + missing.isNotEmpty() -> rejected(OutputRejection.InsufficientPlanEvidence(missing)) + else -> StandardInputOutputValidation.Accepted(output.copy(durationSeconds = outputTimeline.durationSeconds, + audioVideoStartOffsetSeconds = (outputOffset as? AudioVideoStartOffset.Seconds)?.let { MediaFact.Known(it.value) } + ?: MediaFact.Unknown), evaluation) + } + } + + companion object { + const val MINIMUM_DURATION_DELTA = 0.050 + const val MAXIMUM_AV_START_DELTA = 0.050 + const val MAXIMUM_ASPECT_DIMENSION_ERROR = 2.0 + + internal fun preservesAspectRatio(source: Dimensions, output: Dimensions): Boolean { + if (!source.isValid || !output.isValid) return false + val error = MAXIMUM_ASPECT_DIMENSION_ERROR + val minimum = maxOf(0.0, (output.width - error) / source.width, (output.height - error) / source.height) + val maximum = minOf((output.width + error) / source.width, (output.height + error) / source.height) + return minimum <= maximum + } + } +} diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputPlanner.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputPlanner.kt index 87f69ee3..2ec0ab8d 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputPlanner.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputPlanner.kt @@ -69,6 +69,7 @@ internal data class StandardInputConversion( val selection: PolicySelection, /** Other policy violations requiring proof; explicit AAC preparation and tone mapping supply their own. */ val requirementsToRemediate: Set, + /** Display-oriented pixels with rotation baked in; generated output must report rotation zero. */ val outputDimensions: Dimensions, /** Nominal measured rate when preserving timestamps; an exact target when resampling. */ val outputFrameRate: Double, diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspector.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspector.kt new file mode 100644 index 00000000..266bef0b --- /dev/null +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspector.kt @@ -0,0 +1,253 @@ +package com.mux.video.upload.internal.standardization + +import android.media.MediaExtractor +import android.os.Build +import java.io.File +import java.io.RandomAccessFile +import java.nio.ByteBuffer +import java.util.concurrent.CancellationException +import kotlin.math.abs + +internal sealed interface AudioVideoStartOffset { + data object NotApplicable : AudioVideoStartOffset + data class Seconds(val value: Double) : AudioVideoStartOffset +} + +internal data class StandardInputTimelineFacts( + val durationSeconds: MediaFact = MediaFact.Unknown, + val audioVideoStartOffset: MediaFact = MediaFact.Unknown, + /** Effective video PTS in presentation order, excluding encoder priming outside the edit. */ + val videoPresentationSeconds: MediaFact> = MediaFact.Unknown, + val videoDurationSeconds: MediaFact = MediaFact.Unknown, + val firstAudioDurationSeconds: MediaFact = MediaFact.Unknown, +) + +internal data class TimelineInspection( + val timeline: StandardInputTimelineFacts = StandardInputTimelineFacts(), + val status: SampleScanStatus, +) + +/** + * Effective ISO presentation time, shared by source inspection and generated-output validation. + * Timing tables prove the final sample's duration; full platform reads prove those samples exist. + * Fragmented, non-ISO, ambiguous, or unsupported edit timelines remain unknown. No decoding/export. + */ +internal class StandardInputTimelineInspector( + private val limits: SampleScanLimits = SampleScanLimits(), + private val apiLevel: Int = Build.VERSION.SDK_INT, +) { + fun inspect(file: File, metadata: MediaMetadataInspection, + isCancelled: () -> Boolean = { Thread.currentThread().isInterrupted }): TimelineInspection { + val started = System.nanoTime() + fun failure(status: SampleScanStatus) = TimelineInspection(status = status) + fun checkBudget(samples: Int, bytes: Long) { + if (isCancelled()) throw TimelineStop(SampleScanStatus.Cancelled) + if (samples > limits.maximumSamples || bytes > limits.maximumReadBytes || + System.nanoTime() - started >= limits.maximumElapsedNanos) throw TimelineStop(SampleScanStatus.LimitExceeded) + } + return try { + checkBudget(0, 0) + if (metadata.container != MediaFact.Known(ContainerKind.IsoBaseMedia) || + metadata.facts.videoTrackCount != MediaFact.Known(1) || metadata.facts.audioTracks == MediaFact.Unknown) + return failure(SampleScanStatus.Unsupported) + val tracks = IsoTimelineReader.read(file, limits.maximumSamples) { checkBudget(0, 0) } + val media = metadata.tracks.filter { it.kind == TrackKind.Video || it.kind == TrackKind.Audio } + if (media.size != tracks.size || media.any { it.containerIndex.valueOrNull !in tracks.keys }) + return failure(SampleScanStatus.Unsupported) + var bytes = 0L + var count = 0 + val buffer = ByteBuffer.allocate(limits.maximumSampleBytes) + for (track in media) { + val timing = tracks.getValue(track.containerIndex.valueOrNull!!) + if (track.kind == TrackKind.Audio && !timing.hasEdit && track.audio?.let { + (it.encoderDelaySamples.valueOrNull ?: 0) > 0 || (it.encoderPaddingSamples.valueOrNull ?: 0) > 0 + } == true) return failure(SampleScanStatus.Unsupported) + val extractor = MediaExtractor() + try { + extractor.setDataSource(file.absolutePath) + extractor.selectTrack(track.extractorIndex) + var index = 0 + while (extractor.sampleTrackIndex >= 0) { + checkBudget(count, bytes) + if (count >= limits.maximumSamples || bytes >= limits.maximumReadBytes) + throw TimelineStop(SampleScanStatus.LimitExceeded) + if (extractor.sampleFlags and MediaExtractor.SAMPLE_FLAG_SYNC.inv() != 0) + throw TimelineStop(SampleScanStatus.Unsupported) + if (index >= timing.presentationSeconds.size || + abs(extractor.sampleTime / 1e6 - timing.presentationSeconds[index]) > 2e-6) + throw TimelineStop(SampleScanStatus.Unsupported) + val expectedSize = if (apiLevel >= 28) extractor.sampleSize else null + if (expectedSize != null && (expectedSize <= 0 || expectedSize > buffer.capacity() || + expectedSize > limits.maximumReadBytes - bytes)) throw TimelineStop(SampleScanStatus.LimitExceeded) + buffer.clear() + val remaining = minOf(buffer.capacity().toLong(), limits.maximumReadBytes - bytes).toInt() + val readBuffer = buffer.duplicate().apply { limit(remaining) }.slice() + val size = try { extractor.readSampleData(readBuffer, 0) } catch (_: IllegalArgumentException) { + throw TimelineStop(if (apiLevel < 28) SampleScanStatus.LimitExceeded else SampleScanStatus.Unreadable) + } + if (size <= 0 || size > remaining || (expectedSize != null && size.toLong() != expectedSize)) + throw TimelineStop(SampleScanStatus.Unreadable) + bytes += size; count++; index++ + extractor.advance() + } + if (index != timing.presentationSeconds.size) throw TimelineStop(SampleScanStatus.Unreadable) + } finally { extractor.release() } + } + checkBudget(count, bytes) + val video = media.single { it.kind == TrackKind.Video } + val videoTime = tracks.getValue(video.containerIndex.valueOrNull!!) + val firstAudio = media.filter { it.kind == TrackKind.Audio }.minByOrNull { it.containerIndex.valueOrNull!! } + val offset = firstAudio?.let { + AudioVideoStartOffset.Seconds(tracks.getValue(it.containerIndex.valueOrNull!!).startSeconds - videoTime.startSeconds) + } ?: AudioVideoStartOffset.NotApplicable + val audioTime = firstAudio?.let { tracks.getValue(it.containerIndex.valueOrNull!!) } + TimelineInspection(StandardInputTimelineFacts(MediaFact.Known(tracks.values.maxOf { it.endSeconds }), + MediaFact.Known(offset), MediaFact.Known(videoTime.effectivePresentationSeconds), + MediaFact.Known(videoTime.endSeconds - videoTime.startSeconds), + audioTime?.let { MediaFact.Known(it.endSeconds - it.startSeconds) } ?: MediaFact.Unknown), SampleScanStatus.Complete) + } catch (stop: TimelineStop) { failure(stop.status) } + catch (_: CancellationException) { failure(SampleScanStatus.Cancelled) } + catch (_: LinkageError) { failure(SampleScanStatus.Unsupported) } + catch (_: Exception) { failure(if (isCancelled()) SampleScanStatus.Cancelled else SampleScanStatus.Unreadable) } + } +} + +internal class TimelineStop(val status: SampleScanStatus) : RuntimeException() + +private fun checkedAdd(a: Long, b: Long): Long { + val result = a + b + require(((a xor result) and (b xor result)) >= 0) { "Timeline overflow" } + return result +} + +internal data class IsoTrackTimeline( + /** Platform sample times include priming samples outside a single edit's presented range. */ + val presentationSeconds: DoubleArray, + val startSeconds: Double, + val endSeconds: Double, + val effectivePresentationSeconds: List, + val hasEdit: Boolean, +) + +/** Timing-only extension of the existing bounded ISO box walker; never parses sample sizes or NALs. */ +internal object IsoTimelineReader { + fun read(file: File, maximumSamples: Int, checkBudget: () -> Unit = {}): Map = + RandomAccessFile(file, "r").use { input -> + val reader = IsoContainerMetadataReader.Reader(input) + val top = reader.boxes(0, input.length(), topLevel = true).onEach { checkBudget() }.toList() + require(top.none { it.type == "moof" }) + val movie = top.single { it.type == "moov" } + val children = reader.boxes(movie.payload, movie.end).toList() + require(children.none { it.type == "mvex" }) + fun children(box: IsoMetadataBox) = reader.boxes(box.payload, box.end).toList() + fun uint() = input.readInt().toLong() and 0xffffffffL + fun timescale(box: IsoMetadataBox): Long { + input.seek(box.payload) + val version = input.readUnsignedByte() + require(version in 0..1) + val offset = if (version == 0) 12 else 20 + require(box.end - box.payload >= offset + 4) + input.seek(box.payload + offset) + return uint().also { require(it > 0) } + } + val movieScale = timescale(children.single { it.type == "mvhd" }) + var totalSamples = 0L + children.filter { it.type == "trak" }.mapIndexedNotNull { index, track -> + checkBudget() + val trackChildren = children(track) + val mdia = children(trackChildren.single { it.type == "mdia" }) + val handler = mdia.single { it.type == "hdlr" } + require(handler.end - handler.payload >= 12) + input.seek(handler.payload + 8) + val kind = ByteArray(4).also { input.readFully(it) }.toString(Charsets.US_ASCII) + if (kind !in listOf("vide", "soun")) return@mapIndexedNotNull null + val scale = timescale(mdia.single { it.type == "mdhd" }) + val tables = children(children(mdia.single { it.type == "minf" }).single { it.type == "stbl" }) + fun runs(box: IsoMetadataBox, composition: Boolean): List> { + require(box.end - box.payload >= 8) + input.seek(box.payload) + val version = input.readUnsignedByte() + require(version in (if (composition) 0..1 else 0..0)) + require(input.readUnsignedByte() == 0 && input.readUnsignedShort() == 0) + val entries = uint() + if (entries > maximumSamples) throw TimelineStop(SampleScanStatus.LimitExceeded) + require(entries > 0 && box.end - box.payload == 8 + entries * 8) + var samples = 0L + return List(entries.toInt()) { + checkBudget() + val n = uint() + // Match Media3/Android's signed composition offsets, including QuickTime v0 files. + // Full extractor reads below must agree before any timeline becomes Known. + val value = if (composition) input.readInt().toLong() else uint() + samples += n + if (samples > maximumSamples) throw TimelineStop(SampleScanStatus.LimitExceeded) + require(n > 0 && (composition || value > 0)) + n.toInt() to value + } + } + val durations = runs(tables.single { it.type == "stts" }, false) + val sampleCount = durations.sumOf { it.first } + totalSamples += sampleCount + if (totalSamples > maximumSamples) throw TimelineStop(SampleScanStatus.LimitExceeded) + val deltas = LongArray(sampleCount) + var position = 0 + for ((count, delta) in durations) repeat(count) { deltas[position++] = delta } + val offsets = LongArray(sampleCount) + val composition = tables.filter { it.type == "ctts" } + require(composition.size <= 1) + composition.singleOrNull()?.let { + val entries = runs(it, true) + require(entries.sumOf { it.first } == sampleCount) + position = 0 + for ((count, offset) in entries) repeat(count) { offsets[position++] = offset } + } + var decodeTime = 0L + val starts = LongArray(sampleCount) { i -> + checkBudget() + val pts = checkedAdd(decodeTime, offsets[i]) + decodeTime = checkedAdd(decodeTime, deltas[i]) + pts + } + val ends = LongArray(sampleCount) { checkedAdd(starts[it], deltas[it]) } + require(starts.distinct().size == sampleCount) + // Single unit-rate edits cover phone encoder priming and simple trims. Complex edits are + // unproven here, including empty leading edits; do not guess how the platform applies them. + var mediaStart = 0.0 + var editDuration: Double? = null + val editBoxes = trackChildren.filter { it.type == "edts" } + require(editBoxes.size <= 1) + editBoxes.singleOrNull()?.let { + val list = children(it).single { it.type == "elst" } + require(list.end - list.payload >= 8) + input.seek(list.payload) + val version = input.readUnsignedByte() + require(version in 0..1 && input.readUnsignedByte() == 0 && input.readUnsignedShort() == 0) + val count = uint() + if (count > 1) throw TimelineStop(SampleScanStatus.Unsupported) + require(list.end - list.payload == 8 + count * (if (version == 0) 12 else 20)) + if (count == 1L) { + val duration = if (version == 0) uint() else input.readLong() + val time = if (version == 0) input.readInt().toLong() else input.readLong() + require(duration > 0 && time >= 0 && input.readInt() == 65536) + mediaStart = time.toDouble() / scale + editDuration = duration.toDouble() / movieScale + } + } + val rawStart = starts.min().toDouble() / scale + val rawEnd = ends.max().toDouble() / scale + val start = maxOf(0.0, rawStart - mediaStart) + val end = editDuration ?: rawEnd + require(start.isFinite() && end.isFinite() && end > start) + if (editDuration != null) { + // Table coverage must reach the edit endpoint; movie/track timescale rounding is bounded. + require(rawStart <= mediaStart + 2.0 / scale && + rawEnd + 2.0 / scale + 1.0 / movieScale >= mediaStart + end) + } else require(rawStart >= 0) + val platformTimes = DoubleArray(sampleCount) { starts[it].toDouble() / scale - mediaStart } + val effectiveTimes = starts.indices.filter { ends[it].toDouble() / scale > mediaStart && platformTimes[it] < end } + .map { maxOf(0.0, platformTimes[it]) }.sorted() + require(effectiveTimes.isNotEmpty()) + index to IsoTrackTimeline(platformTimes, start, end, effectiveTimes, editDuration != null) + }.toMap() + } +} diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/MediaSampleInspectorTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/MediaSampleInspectorTests.kt index a340cecd..68a5ce0a 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/MediaSampleInspectorTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/MediaSampleInspectorTests.kt @@ -138,7 +138,7 @@ class MediaSampleInspectorTests : AbsRobolectricTest() { private fun scan(limits: SampleScanLimits = SampleScanLimits(), cancel: () -> Boolean = { false }, reader: ((ByteBuffer) -> Int)? = null, reportedSize: Long = 6, flagsOverride: Int? = null, - input: MediaMetadataInspection = metadata()): MediaSampleInspection { + input: MediaMetadataInspection = metadata(), provenDuration: MediaFact? = null): MediaSampleInspection { mockkConstructor(MediaExtractor::class) var index = 0 var released = false @@ -156,7 +156,7 @@ class MediaSampleInspectorTests : AbsRobolectricTest() { every { anyConstructed().advance() } answers { ++index < 3 } every { anyConstructed().release() } answers { released = true } return try { - MediaSampleInspector(limits).inspect(File("unused"), input, cancel).also { + MediaSampleInspector(limits).inspect(File("unused"), input, provenDuration, cancel).also { assertTrue(released) } } finally { unmockkConstructor(MediaExtractor::class) } @@ -239,4 +239,14 @@ class MediaSampleInspectorTests : AbsRobolectricTest() { assertEquals(MediaFact.Unknown, SampleRandomAccessReader.read(VideoCodec.H264, data, data.size)) } finally { unmockkStatic(NalUnitUtil::class) } } + @Test fun provenOutputDurationSuppliesBitrateAndFinalGopDenominator() { + val source = scan() + assertEquals(MediaFact.Known(48L), source.facts.averageBitrate) + val output = scan(provenDuration = MediaFact.Known(2.5)) + assertEquals(SampleScanStatus.Complete, output.status) + assertEquals(MediaFact.Known(58L), output.facts.averageBitrate) + assertEquals(MediaFact.Known(58L), output.facts.maximumGopBitrate) + assertEquals(MediaFact.Known(2.5), output.facts.maximumKeyframeIntervalSeconds) + } + } diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputOutputValidatorTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputOutputValidatorTests.kt new file mode 100644 index 00000000..5cc5ba00 --- /dev/null +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputOutputValidatorTests.kt @@ -0,0 +1,167 @@ +package com.mux.video.upload.internal.standardization + +import org.junit.Assert.* +import org.junit.Test +import java.io.File +import java.util.concurrent.CancellationException + +class StandardInputOutputValidatorTests { + private val source = compliantFacts().copy(cadence = known(Cadence.Constant)) + private val plan = (StandardInputPlanner().plan(source.copy(averageBitrate = known(9_000_000L)), + options(), fullCapabilities()).action as StandardInputAction.Convert).conversion + private val output = source.copy(encodedDimensions = known(plan.outputDimensions), rotationDegrees = known(0)) + private fun timeline(duration: Double = 3.0, offset: AudioVideoStartOffset = AudioVideoStartOffset.Seconds(0.0)) = + StandardInputTimelineFacts(known(duration), known(offset), known(listOf(0.0, 1.0, 2.0)), known(duration), known(duration)) + private val validator = StandardInputOutputValidator() + private fun validate(facts: MediaFacts = output, input: MediaFacts = source, + sourceTime: StandardInputTimelineFacts = timeline(), outputTime: StandardInputTimelineFacts = timeline(), + conversion: StandardInputConversion = plan) = validator.validateFacts(facts, input, sourceTime, outputTime, conversion) + private fun reason(result: StandardInputOutputValidation) = (result as StandardInputOutputValidation.Rejected).reason + private fun mismatch(field: OutputExpectation, result: StandardInputOutputValidation) { + assertTrue((reason(result) as OutputRejection.DoesNotMatchPlan).expectations.contains(field)) + } + + @Test fun acceptsCompliantOutputAndPublishesEffectiveTimeline() { + val accepted = validate() as StandardInputOutputValidation.Accepted + assertEquals(known(3.0), accepted.facts.durationSeconds) + assertEquals(known(0.0), accepted.facts.audioVideoStartOffsetSeconds) + } + @Test fun everyRequiredUnknownPolicyFactRejectsOutput() { + for (facts in listOf(output.copy(videoCodec = MediaFact.Unknown), output.copy(displayDimensions = MediaFact.Unknown), + output.copy(frameRate = MediaFact.Unknown), output.copy(averageBitrate = MediaFact.Unknown), + output.copy(maximumGopBitrate = MediaFact.Unknown), output.copy(maximumKeyframeIntervalSeconds = MediaFact.Unknown), + output.copy(gopStructure = MediaFact.Unknown), output.copy(pixelFormat = MediaFact.Unknown), + output.copy(dynamicRange = MediaFact.Unknown), output.copy(audioTracks = MediaFact.Unknown), + output.copy(editList = MediaFact.Unknown))) { + assertTrue(reason(validate(facts)) is OutputRejection.InsufficientPolicyEvidence) + } + } + @Test fun rejectsKnownViolationsIncludingOpenGopAndComplexEdits() { + for (facts in listOf(output.copy(averageBitrate = known(9_000_000L)), output.copy(gopStructure = known(GopStructure.Open)), + output.copy(maximumKeyframeIntervalSeconds = known(21.0)), output.copy(editList = known(EditList.Complex)))) + assertTrue(reason(validate(facts)) is OutputRejection.NonCompliant) + } + @Test fun rejectsCodecFallbackEvenIfOtherwiseCompliant() { + mismatch(OutputExpectation.VideoCodec, validate(output.copy(videoCodec = known(VideoCodec.Hevc)))) + } + @Test fun requiresSingleVideoExactEncodedAndDisplayDimensionsAndBakedOrientation() { + mismatch(OutputExpectation.VideoPresence, validate(output.copy(videoTrackCount = known(0)))) + mismatch(OutputExpectation.VideoPresence, validate(output.copy(videoTrackCount = known(2)))) + for (facts in listOf(output.copy(encodedDimensions = known(Dimensions(1080, 1920))), + output.copy(displayDimensions = known(Dimensions(1918, 1080))), output.copy(rotationDegrees = known(180)))) + mismatch(OutputExpectation.DimensionsAndOrientation, validate(facts)) + assertTrue(reason(validate(output.copy(encodedDimensions = MediaFact.Unknown))) is OutputRejection.InsufficientPlanEvidence) + } + @Test fun compliantButUnplannedBitDepthRejects() { + val hevc = plan.copy(outputCodec = VideoCodec.Hevc, outputPixelFormat = PixelFormat(10, ChromaSubsampling.Yuv420)) + mismatch(OutputExpectation.PixelFormat, validate(output.copy(videoCodec = known(VideoCodec.Hevc)), conversion = hevc)) + } + @Test fun toneMappingMustProduceSdr() { + val hevc = plan.copy(outputCodec = VideoCodec.Hevc, sourceDynamicRange = DynamicRange.Hlg, toneMapsToSdr = true) + mismatch(OutputExpectation.DynamicRange, validate(output.copy(videoCodec = known(VideoCodec.Hevc), + pixelFormat = known(PixelFormat(10, ChromaSubsampling.Yuv420)), dynamicRange = known(DynamicRange.Hlg)), conversion = hevc)) + } + @Test fun expectedAudioCannotDisappearOrAppearAndKnownLayoutIsRetained() { + mismatch(OutputExpectation.Audio, validate(output.copy(audioTracks = known(emptyList())))) + mismatch(OutputExpectation.Audio, validate(conversion = plan.copy(outputAudio = OutputAudio.None))) + mismatch(OutputExpectation.Audio, validate(output.copy(audioTracks = known(listOf(AudioTrack(known(AudioFormat.Aac(AudioChannelLayout.Mono)))))))) + assertTrue(reason(validate(input = source.copy(audioTracks = MediaFact.Unknown), + conversion = plan.copy(outputAudio = OutputAudio.AacFromFirstTrackIfPresent))) is OutputRejection.InsufficientPlanEvidence) + assertTrue(validate(output.copy(audioTracks = known(emptyList())), source.copy(audioTracks = known(emptyList())), + timeline(offset = AudioVideoStartOffset.NotApplicable), timeline(offset = AudioVideoStartOffset.NotApplicable), + plan.copy(outputAudio = OutputAudio.AacFromFirstTrackIfPresent)) is StandardInputOutputValidation.Accepted) + } + @Test fun durationToleranceIncludes50msBoundaryAndUsesOneFrameAtLowRate() { + assertTrue(validate(outputTime = timeline(3.05)) is StandardInputOutputValidation.Accepted) + mismatch(OutputExpectation.Duration, validate(outputTime = timeline(3.050001))) + val low = plan.copy(outputFrameRate = 5.0) + assertTrue(validate(output.copy(frameRate = known(5.0)), outputTime = timeline(3.2), conversion = low) is StandardInputOutputValidation.Accepted) + mismatch(OutputExpectation.Duration, validate(output.copy(frameRate = known(5.0)), outputTime = timeline(3.200001), conversion = low)) + } + @Test fun avStartToleranceIs50msAndAbsenceIsExplicit() { + assertTrue(validate(outputTime = timeline(offset = AudioVideoStartOffset.Seconds(-0.05))) is StandardInputOutputValidation.Accepted) + mismatch(OutputExpectation.AudioVideoStartOffset, validate(outputTime = timeline(offset = AudioVideoStartOffset.Seconds(0.050001)))) + mismatch(OutputExpectation.AudioVideoStartOffset, validate(outputTime = timeline(offset = AudioVideoStartOffset.NotApplicable))) + } + @Test fun unknownOrInvalidTimelineNeverAccepts() { + for (time in listOf(StandardInputTimelineFacts(), timeline(0.0), timeline(Double.NaN), timeline(Double.POSITIVE_INFINITY), + timeline(offset = AudioVideoStartOffset.Seconds(Double.NaN)))) { + assertTrue(reason(validate(sourceTime = time)) is OutputRejection.InsufficientPlanEvidence) + assertTrue(reason(validate(outputTime = time)) is OutputRejection.InsufficientPlanEvidence) + } + } + @Test fun aspectAlignmentUsesIntersectionOfBothAxisScaleRanges() { + assertTrue(StandardInputOutputValidator.preservesAspectRatio(Dimensions(1920, 1080), Dimensions(1282, 718))) + assertFalse(StandardInputOutputValidator.preservesAspectRatio(Dimensions(1920, 1080), Dimensions(1283, 717))) + assertFalse(StandardInputOutputValidator.preservesAspectRatio(Dimensions(0, 1080), Dimensions(1280, 720))) + mismatch(OutputExpectation.DimensionsAndOrientation, validate(conversion = plan.copy(sourceDisplayDimensions = Dimensions(1000, 1000)))) + } + @Test fun preserveTimestampsChecksEveryPresentationTimeWithoutRejectingBFrameDecodeOrder() { + assertTrue(validate(output.copy(timestamps = known(TimestampFacts(0.0, 2.0, 3, false)))) is StandardInputOutputValidation.Accepted) + mismatch(OutputExpectation.Timestamps, validate(outputTime = timeline().copy(videoPresentationSeconds = known(listOf(0.0, 1.01, 2.0))))) + assertTrue(reason(validate(sourceTime = timeline().copy(videoPresentationSeconds = MediaFact.Unknown))) is OutputRejection.InsufficientPlanEvidence) + } + @Test fun resamplingRequiresConstantCadenceAndPlannedRate() { + val constant = plan.copy(outputCadence = OutputCadence.ConstantFrameRate) + assertTrue(validate(conversion = constant) is StandardInputOutputValidation.Accepted) + mismatch(OutputExpectation.Cadence, validate(output.copy(cadence = known(Cadence.Variable)), conversion = constant)) + mismatch(OutputExpectation.Cadence, validate(output.copy(frameRate = known(29.97)), conversion = constant)) + } + @Test fun emptyMissingAndAudioOnlyFilesRejectAndCancellationWins() { + val file = File.createTempFile("validation", ".mp4") + try { + assertEquals(OutputRejection.EmptyOrUnreadable, reason(validator.validateGeneratedOutput(file, source, timeline(), plan))) + assertEquals(StandardInputOutputValidation.Cancelled, validator.validateGeneratedOutput(file, source, timeline(), plan) { true }) + file.writeBytes(byteArrayOf(1)) + val unreadable = StandardInputOutputValidator(inspectMetadata = { throw CancellationException() }) + assertEquals(StandardInputOutputValidation.Cancelled, unreadable.validateGeneratedOutput(file, source, timeline(), plan)) + } finally { file.delete() } + } + @Test fun partialScansNeverReachComparisonAndLateCancellationWins() { + val file = File.createTempFile("validation", ".mp4") + val metadata = MediaMetadataInspection(known(ContainerKind.IsoBaseMedia), emptyList(), MediaFact.Unknown, output, 0) + try { + file.writeBytes(byteArrayOf(1)) + for (status in SampleScanStatus.entries.filter { it != SampleScanStatus.Complete }) { + val v = StandardInputOutputValidator(inspectMetadata = { MetadataInspectionResult.Success(metadata) }, + inspectSamples = { _, _, _, _ -> MediaSampleInspection(output, status, 1, 1, 1, 0) }, + inspectTimeline = { _, _, _ -> TimelineInspection(timeline(), SampleScanStatus.Complete) }) + val result = v.validateGeneratedOutput(file, source, timeline(), plan) + if (status == SampleScanStatus.Cancelled) assertEquals(StandardInputOutputValidation.Cancelled, result) + else assertEquals(OutputRejection.SampleInspection(status), reason(result)) + } + var cancelled = false + val v = StandardInputOutputValidator(inspectMetadata = { cancelled = true; MetadataInspectionResult.Failure(MetadataFailure.Malformed) }) + assertEquals(StandardInputOutputValidation.Cancelled, v.validateGeneratedOutput(file, source, timeline(), plan) { cancelled }) + } finally { file.delete() } + } + @Test fun timelineFailureCancellationAndFileChangesCannotSelectGeneratedBytes() { + val file = File.createTempFile("validation", ".mp4") + val metadata = MediaMetadataInspection(known(ContainerKind.IsoBaseMedia), emptyList(), MediaFact.Unknown, output, 0) + fun validator(timelineRead: () -> TimelineInspection) = StandardInputOutputValidator( + inspectMetadata = { MetadataInspectionResult.Success(metadata) }, + inspectSamples = { _, _, _, _ -> MediaSampleInspection(output, SampleScanStatus.Complete, 3, 3, 1, 0) }, + inspectTimeline = { _, _, _ -> timelineRead() }) + try { + file.writeBytes(byteArrayOf(1)) + for (status in SampleScanStatus.entries.filter { it != SampleScanStatus.Complete }) { + val result = validator { TimelineInspection(status = status) }.validateGeneratedOutput(file, source, timeline(), plan) + if (status == SampleScanStatus.Cancelled) assertEquals(StandardInputOutputValidation.Cancelled, result) + else assertEquals(OutputRejection.TimelineInspection(status), reason(result)) + } + val changed = validator { file.appendBytes(byteArrayOf(2)); TimelineInspection(timeline(), SampleScanStatus.Complete) } + assertEquals(OutputRejection.ChangedDuringInspection, reason(changed.validateGeneratedOutput(file, source, timeline(), plan))) + val accepted = validator { TimelineInspection(timeline(), SampleScanStatus.Complete) }.validateGeneratedOutput(file, source, timeline(), plan) + assertTrue(accepted is StandardInputOutputValidation.Accepted) + } finally { file.delete() } + } + + @Test fun shorterVideoOrAudioCannotHideBehindUnchangedAssetDuration() { + val constant = plan.copy(outputCadence = OutputCadence.ConstantFrameRate) + mismatch(OutputExpectation.Duration, validate(outputTime = timeline().copy(videoDurationSeconds = known(1.0)), conversion = constant)) + mismatch(OutputExpectation.Duration, validate(outputTime = timeline().copy(firstAudioDurationSeconds = known(1.0)), conversion = constant)) + assertTrue(reason(validate(outputTime = timeline().copy(videoDurationSeconds = MediaFact.Unknown))) is OutputRejection.InsufficientPlanEvidence) + assertTrue(reason(validate(sourceTime = timeline().copy(firstAudioDurationSeconds = MediaFact.Unknown))) is OutputRejection.InsufficientPlanEvidence) + } + +} diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspectorTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspectorTests.kt new file mode 100644 index 00000000..bcfba4c8 --- /dev/null +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspectorTests.kt @@ -0,0 +1,142 @@ +package com.mux.video.upload.internal.standardization + +import android.media.MediaExtractor +import com.mux.exoplayeradapter.AbsRobolectricTest +import io.mockk.* +import org.junit.Assert.* +import org.junit.Test +import org.robolectric.annotation.Config +import java.io.File +import java.nio.ByteBuffer + +@Config(sdk = [23]) +class StandardInputTimelineInspectorTests : AbsRobolectricTest() { + private fun int(value: Int) = ByteBuffer.allocate(4).putInt(value).array() + private fun long(value: Long) = ByteBuffer.allocate(8).putLong(value).array() + private fun box(type: String, payload: ByteArray) = int(payload.size + 8) + type.toByteArray() + payload + private fun header(scale: Int = 1000, version: Int = 0) = + byteArrayOf(version.toByte(), 0, 0, 0) + ByteArray(if (version == 0) 8 else 16) + int(scale) + int(999999) + private fun edit(duration: Long, start: Long, version: Int = 0, rate: Int = 65536) = + box("edts", box("elst", byteArrayOf(version.toByte(), 0, 0, 0) + int(1) + + (if (version == 0) int(duration.toInt()) + int(start.toInt()) else long(duration) + long(start)) + int(rate))) + private fun track(deltas: List = listOf(1000, 1000, 1000), offsets: List? = null, + edits: ByteArray = byteArrayOf(), kind: String = "vide", scale: Int = 1000, + timingVersion: Int = 0, headerVersion: Int = 0): ByteArray { + val stts = box("stts", ByteArray(4) + int(deltas.size) + deltas.flatMap { (int(1) + int(it)).toList() }.toByteArray()) + val ctts = offsets?.let { box("ctts", byteArrayOf(timingVersion.toByte(), 0, 0, 0) + int(it.size) + + it.flatMap { offset -> (int(1) + int(offset)).toList() }.toByteArray()) } ?: byteArrayOf() + return box("trak", edits + box("mdia", box("hdlr", ByteArray(8) + kind.toByteArray()) + + box("mdhd", header(scale, headerVersion)) + box("minf", box("stbl", stts + ctts)))) + } + private fun file(tracks: ByteArray = track(), scale: Int = 1000, extra: ByteArray = byteArrayOf()): File = + File.createTempFile("timeline", ".mp4").apply { writeBytes(box("moov", box("mvhd", header(scale)) + tracks) + extra) } + private fun read(tracks: ByteArray = track(), scale: Int = 1000, maximum: Int = 250000): IsoTrackTimeline { + val file = file(tracks, scale) + return try { IsoTimelineReader.read(file, maximum).getValue(0) } finally { file.delete() } + } + private fun invalid(tracks: ByteArray, scale: Int = 1000, maximum: Int = 250000) { + try { read(tracks, scale, maximum); fail("Unproven timeline was published") } catch (_: IllegalArgumentException) { } catch (stop: TimelineStop) { assertEquals(SampleScanStatus.LimitExceeded, stop.status) } + } + @Test fun provesFinalVfrSampleDurationWithoutUsingReportedHeaderDuration() { + val time = read(track(deltas = listOf(100, 200, 700))) + assertEquals(1.0, time.endSeconds, 0.0) + assertEquals(listOf(0.0, 0.1, 0.3), time.effectivePresentationSeconds) + } + @Test fun maxPresentationEndpointHandlesReorderingAndSignedCompositionOffsets() { + for (version in 0..1) { + val time = read(track(offsets = listOf(0, 1000, -1000), timingVersion = version)) + assertEquals(3.0, time.endSeconds, 0.0) + assertEquals(listOf(0.0, 1.0, 2.0), time.effectivePresentationSeconds) + assertEquals(listOf(0.0, 2.0, 1.0), time.presentationSeconds.toList()) + } + } + @Test fun appliesPrimingEditAndKeepsRawPlatformSamplesSeparateFromPresentedSamples() { + for (version in 0..1) { + val time = read(track(edits = edit(2000, 1000, version), headerVersion = version)) + assertEquals(listOf(-1.0, 0.0, 1.0), time.presentationSeconds.toList()) + assertEquals(listOf(0.0, 1.0), time.effectivePresentationSeconds) + assertEquals(0.0, time.startSeconds, 0.0) + assertEquals(2.0, time.endSeconds, 0.0) + } + } + @Test fun movieAndMediaTimescalesAreIndependentAndTrimsCannotClaimMissingTail() { + assertEquals(2.0, read(track(edits = edit(180000, 1000)), scale = 90000).endSeconds, 0.0) + invalid(track(edits = edit(4000, 0))) + invalid(track(edits = edit(2000, 5000))) + } + @Test fun zeroTimescalesZeroDurationsDuplicateTimesAndMismatchedRunsStayUnproven() { + invalid(track(scale = 0)) + invalid(track(), scale = 0) + invalid(track(deltas = listOf(0))) + invalid(track(offsets = listOf(0, -1000, 0), timingVersion = 1)) + invalid(track(offsets = listOf(0))) + invalid(track(edits = edit(0, 0))) + invalid(track(edits = edit(3000, -1))) + invalid(track(edits = edit(3000, 0, rate = 0))) + invalid(track(edits = edit(3000, 0) + edit(3000, 0))) + } + @Test fun oversizedSampleTablesAndFragmentedTimelinesStayUnproven() { + invalid(track(), maximum = 2) + val file = file(extra = box("moof", byteArrayOf())) + try { + try { IsoTimelineReader.read(file, 250000); fail("Fragment accepted") } catch (_: IllegalArgumentException) { } catch (stop: TimelineStop) { assertEquals(SampleScanStatus.LimitExceeded, stop.status) } + } finally { file.delete() } + } + + private fun metadata(audio: Boolean): MediaMetadataInspection { + fun track(index: Int, kind: TrackKind) = TrackMetadata(index, known(index), known(index + 1), kind, + MediaFact.Unknown, known(999.0), MediaFact.Unknown, MediaFact.Unknown) + return MediaMetadataInspection(known(ContainerKind.IsoBaseMedia), + listOf(track(0, TrackKind.Video)) + if (audio) listOf(track(1, TrackKind.Audio)) else emptyList(), + MediaFact.Unknown, MediaFacts(videoTrackCount = known(1), audioTracks = known(if (audio) listOf(AudioTrack()) else emptyList())), 0) + } + private fun inspect(audio: Boolean = false, audioOffset: Int = 0, sampleLimit: Int = 250000, + missingTail: Boolean = false, partial: Boolean = false, cancelAfterRead: Boolean = false, + legacyBufferFailure: Boolean = false): TimelineInspection { + val file = file(track() + if (audio) track(kind = "soun", offsets = List(3) { audioOffset }) else byteArrayOf()) + mockkConstructor(MediaExtractor::class) + var selected = 0; var index = 0; var cancelled = false; var releases = 0 + every { anyConstructed().setDataSource(any()) } just Runs + every { anyConstructed().selectTrack(any()) } answers { selected = firstArg(); index = 0 } + every { anyConstructed().sampleTrackIndex } answers { if (index < if (missingTail) 2 else 3) selected else -1 } + every { anyConstructed().sampleTime } answers { index * 1000000L + if (selected == 1) audioOffset * 1000L else 0L } + every { anyConstructed().sampleFlags } returns if (partial) 4 else 0 + every { anyConstructed().readSampleData(any(), 0) } answers { + if (legacyBufferFailure) throw IllegalArgumentException("Capped buffer") + if (cancelAfterRead) cancelled = true + 4 + } + every { anyConstructed().advance() } answers { ++index < 3 } + every { anyConstructed().release() } answers { releases++ } + return try { + StandardInputTimelineInspector(SampleScanLimits(maximumSamples = sampleLimit), 23).inspect(file, metadata(audio)) { cancelled } + .also { if (sampleLimit >= 3) assertTrue(releases > 0) } + } finally { unmockkConstructor(MediaExtractor::class); file.delete() } + } + @Test fun noAudioHasExplicitNotApplicableOffsetAndEffectiveEndpoint() { + val result = inspect() + assertEquals(SampleScanStatus.Complete, result.status) + assertEquals(known(3.0), result.timeline.durationSeconds) + assertEquals(known(AudioVideoStartOffset.NotApplicable), result.timeline.audioVideoStartOffset) + } + @Test fun offsetComesFromEffectiveAudioAndVideoStarts() { + val result = inspect(audio = true, audioOffset = 100) + assertEquals(SampleScanStatus.Complete, result.status) + assertEquals(known(3.1), result.timeline.durationSeconds) + assertEquals(known(AudioVideoStartOffset.Seconds(0.1)), result.timeline.audioVideoStartOffset) + } + @Test fun truncatedPartialCancelledOrFailedLegacyReadsNeverPublishTimeline() { + for ((result, status) in listOf(inspect(missingTail = true) to SampleScanStatus.Unreadable, + inspect(partial = true) to SampleScanStatus.Unsupported, + inspect(cancelAfterRead = true) to SampleScanStatus.Cancelled, + inspect(legacyBufferFailure = true) to SampleScanStatus.LimitExceeded)) { + assertEquals(status, result.status) + assertEquals(StandardInputTimelineFacts(), result.timeline) + } + } + @Test fun cancellationWinsBeforeUnsupportedMetadata() { + val result = StandardInputTimelineInspector(apiLevel = 23).inspect(File("unused"), + metadata(false).copy(container = known(ContainerKind.Matroska))) { true } + assertEquals(SampleScanStatus.Cancelled, result.status) + } +} From 5391f70a2aa457855afb681e83568d230a68dd2d Mon Sep 17 00:00:00 2001 From: Eugene Morozov Date: Tue, 6 Oct 2026 12:34:33 -0700 Subject: [PATCH 15/49] Fix output validation timing and bounded sample inspection --- .../standardization/BoundedSampleReader.kt | 51 +++++++ .../standardization/MediaSampleInspector.kt | 124 +++++++-------- .../StandardInputOutputValidator.kt | 30 ++-- .../StandardInputTimelineInspector.kt | 141 ++++++++++-------- .../MediaSampleInspectorTests.kt | 51 +++++-- .../StandardInputOutputValidatorTests.kt | 42 +++++- .../StandardInputTimelineInspectorTests.kt | 105 +++++++++++-- 7 files changed, 367 insertions(+), 177 deletions(-) create mode 100644 library/src/main/java/com/mux/video/upload/internal/standardization/BoundedSampleReader.kt diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/BoundedSampleReader.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/BoundedSampleReader.kt new file mode 100644 index 00000000..e2bfae48 --- /dev/null +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/BoundedSampleReader.kt @@ -0,0 +1,51 @@ +package com.mux.video.upload.internal.standardization + +import android.media.MediaExtractor +import java.nio.ByteBuffer + +internal class SampleScanStop(val status: SampleScanStatus) : RuntimeException() + +/** One capped buffer and cooperative budget shared by source and effective-timeline scans. */ +internal class BoundedSampleReader( + private val limits: SampleScanLimits, + private val apiLevel: Int, + private val isCancelled: () -> Boolean, +) { + private val started = System.nanoTime() + private val data = ByteArray(limits.maximumSampleBytes) + private val buffer = ByteBuffer.wrap(data) + var sampleCount = 0; private set + var bytesRead = 0L; private set + var largestSampleBytes = 0; private set + val elapsedNanos get() = System.nanoTime() - started + + fun checkBudget(beforeRead: Boolean = false) { + if (isCancelled()) throw SampleScanStop(SampleScanStatus.Cancelled) + if (elapsedNanos >= limits.maximumElapsedNanos || + (if (beforeRead) sampleCount >= limits.maximumSamples else sampleCount > limits.maximumSamples) || + (if (beforeRead) bytesRead >= limits.maximumReadBytes else bytesRead > limits.maximumReadBytes)) + throw SampleScanStop(SampleScanStatus.LimitExceeded) + } + + /** The byte array is reused: consume relevant NAL evidence before advancing the extractor. */ + fun read(extractor: MediaExtractor, consume: (Long, Int, Int, ByteArray) -> Unit) { + checkBudget(beforeRead = true) + val flags = extractor.sampleFlags + if (flags and MediaExtractor.SAMPLE_FLAG_SYNC.inv() != 0) throw SampleScanStop(SampleScanStatus.Unsupported) + val time = extractor.sampleTime + val expected = if (apiLevel >= 28) extractor.sampleSize else null + val remaining = minOf(data.size.toLong(), limits.maximumReadBytes - bytesRead).toInt() + if (expected != null && (expected <= 0 || expected > remaining)) throw SampleScanStop(SampleScanStatus.LimitExceeded) + buffer.clear() + val capped = if (remaining < data.size) ByteBuffer.wrap(data, 0, remaining).slice() else buffer + val size = try { extractor.readSampleData(capped, 0) } catch (_: IllegalArgumentException) { + throw SampleScanStop(if (apiLevel < 28) SampleScanStatus.LimitExceeded else SampleScanStatus.Unreadable) + } + if (size > remaining) throw SampleScanStop(SampleScanStatus.LimitExceeded) + if (size <= 0) throw SampleScanStop(if (remaining < data.size) SampleScanStatus.LimitExceeded else SampleScanStatus.Unreadable) + if (expected != null && size.toLong() != expected) throw SampleScanStop(SampleScanStatus.Unreadable) + bytesRead += size; sampleCount++; largestSampleBytes = maxOf(largestSampleBytes, size) + consume(time, size, flags, data) + checkBudget() + } +} diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/MediaSampleInspector.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/MediaSampleInspector.kt index 250176b2..6b8c946a 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/MediaSampleInspector.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/MediaSampleInspector.kt @@ -4,7 +4,7 @@ import android.media.MediaExtractor import android.os.Build import androidx.media3.container.NalUnitUtil import java.io.File -import java.nio.ByteBuffer +import java.util.concurrent.CancellationException import kotlin.math.abs import kotlin.math.roundToLong @@ -41,18 +41,11 @@ internal data class CompressedVideoSample( internal class MediaSampleInspector(private val limits: SampleScanLimits = SampleScanLimits(), private val apiLevel: Int = Build.VERSION.SDK_INT) { fun inspect(file: File, metadata: MediaMetadataInspection, - isCancelled: () -> Boolean = { Thread.currentThread().isInterrupted }): MediaSampleInspection = - inspect(file, metadata, null, isCancelled) - - /** Output validation supplies proven video duration; source-only scans retain the observation. */ - fun inspect(file: File, metadata: MediaMetadataInspection, provenVideoDurationSeconds: MediaFact?, - isCancelled: () -> Boolean): MediaSampleInspection { - val started = System.nanoTime() + isCancelled: () -> Boolean = { Thread.currentThread().isInterrupted }): MediaSampleInspection { + val reader = BoundedSampleReader(limits, apiLevel, isCancelled) val samples = ArrayList() - var bytes = 0L - var largestSample = 0 fun result(status: SampleScanStatus, facts: MediaFacts = metadata.facts) = - MediaSampleInspection(facts, status, samples.size, bytes, largestSample, System.nanoTime() - started) + MediaSampleInspection(facts, status, reader.sampleCount, reader.bytesRead, reader.largestSampleBytes, reader.elapsedNanos) if (isCancelled()) return result(SampleScanStatus.Cancelled) val track = metadata.tracks.filter { it.kind == TrackKind.Video }.singleOrNull() ?: return result(SampleScanStatus.Unsupported) @@ -64,56 +57,36 @@ internal class MediaSampleInspector(private val limits: SampleScanLimits = Sampl return try { extractor.setDataSource(file.absolutePath) extractor.selectTrack(track.extractorIndex) - val data = ByteArray(limits.maximumSampleBytes) - val buffer = ByteBuffer.wrap(data) while (extractor.sampleTrackIndex >= 0) { - if (isCancelled()) return result(SampleScanStatus.Cancelled) - if (samples.size >= limits.maximumSamples || bytes >= limits.maximumReadBytes || - System.nanoTime() - started >= limits.maximumElapsedNanos) return result(SampleScanStatus.LimitExceeded) - val flags = extractor.sampleFlags - // Partial frames and encrypted bytes cannot establish complete pictures or GOPs. - if (flags and MediaExtractor.SAMPLE_FLAG_SYNC.inv() != 0) return result(SampleScanStatus.Unsupported) - val expectedSize = if (apiLevel >= 28) extractor.sampleSize else null - if (expectedSize != null && (expectedSize <= 0 || expectedSize > data.size || - expectedSize > limits.maximumReadBytes - bytes)) return result(SampleScanStatus.LimitExceeded) - buffer.clear() - // API 23–27 has no size query. Cap the native buffer's capacity as well as its limit. - val remaining = limits.maximumReadBytes - bytes - val readBuffer = if (remaining < data.size) ByteBuffer.wrap(data, 0, remaining.toInt()).slice() else buffer - val size = try { extractor.readSampleData(readBuffer, 0) } catch (_: IllegalArgumentException) { - // Legacy extractors signal an insufficient sample buffer with IllegalArgumentException. - return result(if (apiLevel < 28) SampleScanStatus.LimitExceeded else SampleScanStatus.Unreadable) + reader.read(extractor) { time, size, flags, data -> + val sync = flags and MediaExtractor.SAMPLE_FLAG_SYNC != 0 + samples.add(CompressedVideoSample(time, size, sync, + if (sync) SampleRandomAccessReader.read(codec, data, size) else MediaFact.Unknown)) } - if (size <= 0 || size > data.size || size.toLong() > limits.maximumReadBytes - bytes || - (expectedSize != null && size.toLong() != expectedSize)) return result(SampleScanStatus.LimitExceeded) - bytes += size - largestSample = maxOf(largestSample, size) - val sync = flags and MediaExtractor.SAMPLE_FLAG_SYNC != 0 - samples.add(CompressedVideoSample(extractor.sampleTime, size, sync, - if (sync) SampleRandomAccessReader.read(codec, data, size) else MediaFact.Unknown)) extractor.advance() } - if (isCancelled()) return result(SampleScanStatus.Cancelled) - if (System.nanoTime() - started >= limits.maximumElapsedNanos) return result(SampleScanStatus.LimitExceeded) - // Match the shared video-edit policy. Audio edits remain separate track observations. - val edit = when (metadata.container.valueOrNull) { - ContainerKind.IsoBaseMedia -> IsoContainerMetadataReader.videoEdit(file, track.sourceTrackId.valueOrNull) - // This proves absence of ISO edits only, not an effective Matroska presentation timeline. - ContainerKind.Matroska -> MediaFact.Known(EditList.None) - else -> MediaFact.Unknown - } - val facts = VideoSampleFactsReader.read(samples, provenVideoDurationSeconds ?: track.reportedDurationSeconds, metadata.facts).copy(editList = edit) - if (isCancelled()) return result(SampleScanStatus.Cancelled) - if (System.nanoTime() - started >= limits.maximumElapsedNanos) return result(SampleScanStatus.LimitExceeded) + reader.checkBudget() + val facts = VideoSampleFactsReader.read(samples, track.reportedDurationSeconds, metadata.facts) + .copy(editList = videoEditList(metadata, track)) + reader.checkBudget() result(SampleScanStatus.Complete, facts) - } catch (_: Exception) { - result(SampleScanStatus.Unreadable) - } finally { - extractor.release() - } + } catch (stop: SampleScanStop) { result(stop.status) } + catch (_: CancellationException) { result(SampleScanStatus.Cancelled) } + catch (_: Exception) { result(if (isCancelled()) SampleScanStatus.Cancelled else SampleScanStatus.Unreadable) } + finally { extractor.release() } } + } +/** Match the shared video-edit policy; audio edits remain separate track observations. */ +internal fun videoEditList(metadata: MediaMetadataInspection, track: TrackMetadata): MediaFact = + when (metadata.container.valueOrNull) { + ContainerKind.IsoBaseMedia -> metadata.isoTracks.valueOrNull + ?.getOrNull(track.containerIndex.valueOrNull ?: -1)?.editList ?: MediaFact.Unknown + ContainerKind.Matroska -> MediaFact.Known(EditList.None) + else -> MediaFact.Unknown + } + /** Only sync samples need relevant NAL inspection. Unrecognized framing or syntax stays unknown. */ internal object SampleRandomAccessReader { fun read(codec: VideoCodec, data: ByteArray, size: Int): MediaFact = try { @@ -183,32 +156,44 @@ internal object SampleRandomAccessReader { internal object VideoSampleFactsReader { fun read(samples: List, reportedDuration: MediaFact, - base: MediaFacts = MediaFacts()): MediaFacts { + base: MediaFacts = MediaFacts(), timeline: IsoTrackTimeline? = null): MediaFacts { if (samples.size < 2 || samples.any { it.byteSize <= 0 }) return base - val ordered = LongArray(samples.size) { samples[it].presentationTimeUs }.also { it.sort() } - val intervals = LongArray(samples.size - 1) { ordered[it + 1] - ordered[it] } + if (timeline != null && timeline.presentationSeconds.size != samples.size) return base + val presented = timeline?.effectiveSampleIndices + if (presented != null && presented.size < 2) return base + fun time(index: Int) = timeline?.presentationSeconds?.get(index) ?: (samples[index].presentationTimeUs / 1e6) + val measured = if (timeline == null) samples else presented!!.map { samples[it].copy(presentationTimeUs = + (maxOf(timeline?.startSeconds ?: Double.NEGATIVE_INFINITY, time(it)) * 1e6).roundToLong()) } + val ordered = LongArray(measured.size) { measured[it].presentationTimeUs }.also { it.sort() } + val intervals = LongArray(measured.size - 1) { ordered[it + 1] - ordered[it] } if (intervals.any { it <= 0 }) return base val span = (ordered.last() - ordered.first()) / 1e6 if (!span.isFinite() || span <= 0) return base - val frameInterval = span / (samples.size - 1) + val frameInterval = span / (measured.size - 1) val constant = intervals.all { abs(it / 1e6 - frameInterval) <= maxOf(2e-6, frameInterval * 0.001) } var facts = base.copy( frameRate = MediaFact.Known(1 / frameInterval), cadence = MediaFact.Known(if (constant) Cadence.Constant else Cadence.Variable), timestamps = MediaFact.Known(TimestampFacts(ordered.first() / 1e6, ordered.last() / 1e6, - samples.size.toLong(), (1 until samples.size).all { - samples[it - 1].presentationTimeUs < samples[it].presentationTimeUs + measured.size.toLong(), (1 until measured.size).all { + measured[it - 1].presentationTimeUs < measured[it].presentationTimeUs })), ) - // Raw track duration is a bitrate denominator, not proof of effective duration or A/V sync. - // Reject inconsistent metadata instead of extrapolating an unobserved final VFR interval. - val duration = reportedDuration.valueOrNull?.takeIf { + // Only a proven effective window replaces the raw source-duration observation. + // Unproven metadata must not extrapolate the final VFR interval. + val duration = timeline?.let { it.endSeconds - it.startSeconds } ?: reportedDuration.valueOrNull?.takeIf { it.isFinite() && it > span && it - span <= intervals.max() / 1e6 * 2 } - if (duration != null) facts = facts.copy(averageBitrate = bitrate(samples.sumOf { it.byteSize.toLong() }, duration)) + if (duration != null) facts = facts.copy(averageBitrate = bitrate(measured.sumOf { it.byteSize.toLong() }, duration)) if (!samples.first().isSync) return facts val boundaries = samples.indices.filter { samples[it].isSync } - val kinds = boundaries.map { samples[it].randomAccess } + val presentedSet = presented?.toHashSet() + fun isPresented(index: Int) = presentedSet == null || index in presentedSet + val relevant = boundaries.indices.filter { i -> + presentedSet == null || (boundaries[i] until (boundaries.getOrNull(i + 1) ?: samples.size)).any { isPresented(it) } + } + // Keep the original IDR boundary, including decode preroll outside the effective edit. + val kinds = relevant.map { samples[boundaries[it]].randomAccess } val open = kinds.any { it == MediaFact.Known(GopStructure.Open) } val proven = kinds.all { it == MediaFact.Known(GopStructure.ClosedWithIdr) } facts = facts.copy(gopStructure = when { @@ -219,16 +204,17 @@ internal object VideoSampleFactsReader { var maxBytes = 0L var maxRate = 0L var maxInterval = 0.0 - for (i in boundaries.indices) { + for (i in relevant) { val start = boundaries[i] val end = boundaries.getOrNull(i + 1) ?: samples.size - val startTime = samples[start].presentationTimeUs / 1e6 - val endTime = if (end < samples.size) samples[end].presentationTimeUs / 1e6 - else duration?.plus(ordered.first() / 1e6) ?: return facts + val startTime = maxOf(timeline?.startSeconds ?: Double.NEGATIVE_INFINITY, time(start)) + val rawEnd = if (end < samples.size) time(end) + else timeline?.endSeconds ?: duration?.plus(ordered.first() / 1e6) ?: return facts + val endTime = minOf(timeline?.endSeconds ?: Double.POSITIVE_INFINITY, rawEnd) val seconds = endTime - startTime if (!seconds.isFinite() || seconds <= 0) return facts maxInterval = maxOf(maxInterval, seconds) - val bytes = samples.subList(start, end).sumOf { it.byteSize.toLong() } + val bytes = (start until end).sumOf { if (isPresented(it)) samples[it].byteSize.toLong() else 0L } maxBytes = maxOf(maxBytes, bytes) maxRate = maxOf(maxRate, bitrate(bytes, seconds).valueOrNull ?: return facts) } diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputOutputValidator.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputOutputValidator.kt index a7ec6f1e..0576c4d7 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputOutputValidator.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputOutputValidator.kt @@ -13,7 +13,6 @@ internal sealed interface OutputRejection { data object EmptyOrUnreadable : OutputRejection data object ChangedDuringInspection : OutputRejection data class Metadata(val reason: MetadataFailure) : OutputRejection - data class SampleInspection(val status: SampleScanStatus) : OutputRejection data class TimelineInspection(val status: SampleScanStatus) : OutputRejection data class NonCompliant(val requirements: Set) : OutputRejection data class InsufficientPolicyEvidence(val requirements: Set) : OutputRejection @@ -32,8 +31,6 @@ internal sealed interface StandardInputOutputValidation { internal class StandardInputOutputValidator( private val evaluator: StandardInputPolicyEvaluator = StandardInputPolicyEvaluator(), private val inspectMetadata: (File) -> MetadataInspectionResult = MediaMetadataInspector()::inspect, - private val inspectSamples: (File, MediaMetadataInspection, () -> Boolean, MediaFact) -> MediaSampleInspection = - { file, metadata, cancelled, duration -> MediaSampleInspector().inspect(file, metadata, duration, cancelled) }, private val inspectTimeline: (File, MediaMetadataInspection, () -> Boolean) -> TimelineInspection = { file, metadata, cancelled -> StandardInputTimelineInspector().inspect(file, metadata, cancelled) }, ) { @@ -41,6 +38,7 @@ internal class StandardInputOutputValidator( conversion: StandardInputConversion, isCancelled: () -> Boolean = { Thread.currentThread().isInterrupted }): StandardInputOutputValidation { fun rejected(reason: OutputRejection) = StandardInputOutputValidation.Rejected(reason) + fun failedInspection() = if (isCancelled()) StandardInputOutputValidation.Cancelled else rejected(OutputRejection.EmptyOrUnreadable) return try { if (isCancelled()) return StandardInputOutputValidation.Cancelled if (!file.isFile || !file.canRead() || file.length() == 0L) return rejected(OutputRejection.EmptyOrUnreadable) @@ -58,20 +56,14 @@ internal class StandardInputOutputValidator( val duration = timeline.timeline.videoDurationSeconds.valueOrNull if (duration == null || !duration.isFinite() || duration <= 0) return rejected(OutputRejection.InsufficientPlanEvidence(setOf(OutputExpectation.Duration))) - val samples = inspectSamples(file, metadata, isCancelled, timeline.timeline.videoDurationSeconds) - if (isCancelled() || samples.status == SampleScanStatus.Cancelled) return StandardInputOutputValidation.Cancelled - if (samples.status != SampleScanStatus.Complete) return rejected(OutputRejection.SampleInspection(samples.status)) - if (isCancelled()) return StandardInputOutputValidation.Cancelled + val facts = timeline.sampleFacts ?: return rejected(OutputRejection.TimelineInspection(SampleScanStatus.Unreadable)) if (file.length() != initialSize || file.lastModified() != initialModified) return rejected(OutputRejection.ChangedDuringInspection) - val result = validateFacts(samples.facts, source, sourceTimeline, timeline.timeline, conversion) + val result = validateFacts(facts, source, sourceTimeline, timeline.timeline, conversion) if (isCancelled()) StandardInputOutputValidation.Cancelled else result } catch (_: CancellationException) { StandardInputOutputValidation.Cancelled } - catch (_: LinkageError) { - if (isCancelled()) StandardInputOutputValidation.Cancelled else rejected(OutputRejection.EmptyOrUnreadable) - } catch (_: Exception) { - if (isCancelled()) StandardInputOutputValidation.Cancelled else rejected(OutputRejection.EmptyOrUnreadable) - } + catch (_: LinkageError) { failedInspection() } + catch (_: Exception) { failedInspection() } } fun validateFacts(output: MediaFacts, source: MediaFacts, sourceTimeline: StandardInputTimelineFacts, @@ -128,9 +120,15 @@ internal class StandardInputOutputValidator( val outputTimes = outputTimeline.videoPresentationSeconds.valueOrNull if (sourceTimes == null || outputTimes == null || sourceTimes.isEmpty() || outputTimes.isEmpty()) missing.add(OutputExpectation.Timestamps) - else if (sourceTimes.size != outputTimes.size || sourceTimes.indices.any { - !sourceTimes[it].isFinite() || !outputTimes[it].isFinite() || abs(sourceTimes[it] - outputTimes[it]) > 2e-6 - }) mismatches.add(OutputExpectation.Timestamps) + else { + val timescale = outputTimeline.videoTimescale.valueOrNull?.takeIf { it > 0 } + if (timescale == null) missing.add(OutputExpectation.Timestamps) + else if (sourceTimes.size != outputTimes.size || sourceTimes.indices.any { + // Extractor/codec timestamps truncate to microseconds; the muxer rounds to output ticks. + !sourceTimes[it].isFinite() || !outputTimes[it].isFinite() || + abs(sourceTimes[it] - outputTimes[it]) > 1e-6 + 0.5 / timescale + 1e-12 + }) mismatches.add(OutputExpectation.Timestamps) + } } } fun compareDuration(sourceFact: MediaFact, outputFact: MediaFact) { diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspector.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspector.kt index 266bef0b..5a3d12b3 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspector.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspector.kt @@ -4,9 +4,9 @@ import android.media.MediaExtractor import android.os.Build import java.io.File import java.io.RandomAccessFile -import java.nio.ByteBuffer import java.util.concurrent.CancellationException import kotlin.math.abs +import kotlin.math.roundToLong internal sealed interface AudioVideoStartOffset { data object NotApplicable : AudioVideoStartOffset @@ -20,11 +20,14 @@ internal data class StandardInputTimelineFacts( val videoPresentationSeconds: MediaFact> = MediaFact.Unknown, val videoDurationSeconds: MediaFact = MediaFact.Unknown, val firstAudioDurationSeconds: MediaFact = MediaFact.Unknown, + val videoTimescale: MediaFact = MediaFact.Unknown, ) internal data class TimelineInspection( val timeline: StandardInputTimelineFacts = StandardInputTimelineFacts(), val status: SampleScanStatus, + /** Facts proven by the same payload scan; never published after a partial read. */ + val sampleFacts: MediaFacts? = null, ) /** @@ -38,81 +41,73 @@ internal class StandardInputTimelineInspector( ) { fun inspect(file: File, metadata: MediaMetadataInspection, isCancelled: () -> Boolean = { Thread.currentThread().isInterrupted }): TimelineInspection { - val started = System.nanoTime() + val reader = BoundedSampleReader(limits, apiLevel, isCancelled) fun failure(status: SampleScanStatus) = TimelineInspection(status = status) - fun checkBudget(samples: Int, bytes: Long) { - if (isCancelled()) throw TimelineStop(SampleScanStatus.Cancelled) - if (samples > limits.maximumSamples || bytes > limits.maximumReadBytes || - System.nanoTime() - started >= limits.maximumElapsedNanos) throw TimelineStop(SampleScanStatus.LimitExceeded) - } return try { - checkBudget(0, 0) + reader.checkBudget() if (metadata.container != MediaFact.Known(ContainerKind.IsoBaseMedia) || metadata.facts.videoTrackCount != MediaFact.Known(1) || metadata.facts.audioTracks == MediaFact.Unknown) return failure(SampleScanStatus.Unsupported) - val tracks = IsoTimelineReader.read(file, limits.maximumSamples) { checkBudget(0, 0) } + val tracks = IsoTimelineReader.read(file, limits.maximumSamples) { reader.checkBudget() } val media = metadata.tracks.filter { it.kind == TrackKind.Video || it.kind == TrackKind.Audio } if (media.size != tracks.size || media.any { it.containerIndex.valueOrNull !in tracks.keys }) return failure(SampleScanStatus.Unsupported) - var bytes = 0L - var count = 0 - val buffer = ByteBuffer.allocate(limits.maximumSampleBytes) + val timingByExtractor = media.associate { it.extractorIndex to tracks.getValue(it.containerIndex.valueOrNull!!) } for (track in media) { - val timing = tracks.getValue(track.containerIndex.valueOrNull!!) - if (track.kind == TrackKind.Audio && !timing.hasEdit && track.audio?.let { + if (track.kind == TrackKind.Audio && !timingByExtractor.getValue(track.extractorIndex).hasEdit && track.audio?.let { (it.encoderDelaySamples.valueOrNull ?: 0) > 0 || (it.encoderPaddingSamples.valueOrNull ?: 0) > 0 } == true) return failure(SampleScanStatus.Unsupported) - val extractor = MediaExtractor() - try { - extractor.setDataSource(file.absolutePath) - extractor.selectTrack(track.extractorIndex) - var index = 0 - while (extractor.sampleTrackIndex >= 0) { - checkBudget(count, bytes) - if (count >= limits.maximumSamples || bytes >= limits.maximumReadBytes) - throw TimelineStop(SampleScanStatus.LimitExceeded) - if (extractor.sampleFlags and MediaExtractor.SAMPLE_FLAG_SYNC.inv() != 0) - throw TimelineStop(SampleScanStatus.Unsupported) - if (index >= timing.presentationSeconds.size || - abs(extractor.sampleTime / 1e6 - timing.presentationSeconds[index]) > 2e-6) - throw TimelineStop(SampleScanStatus.Unsupported) - val expectedSize = if (apiLevel >= 28) extractor.sampleSize else null - if (expectedSize != null && (expectedSize <= 0 || expectedSize > buffer.capacity() || - expectedSize > limits.maximumReadBytes - bytes)) throw TimelineStop(SampleScanStatus.LimitExceeded) - buffer.clear() - val remaining = minOf(buffer.capacity().toLong(), limits.maximumReadBytes - bytes).toInt() - val readBuffer = buffer.duplicate().apply { limit(remaining) }.slice() - val size = try { extractor.readSampleData(readBuffer, 0) } catch (_: IllegalArgumentException) { - throw TimelineStop(if (apiLevel < 28) SampleScanStatus.LimitExceeded else SampleScanStatus.Unreadable) - } - if (size <= 0 || size > remaining || (expectedSize != null && size.toLong() != expectedSize)) - throw TimelineStop(SampleScanStatus.Unreadable) - bytes += size; count++; index++ - extractor.advance() - } - if (index != timing.presentationSeconds.size) throw TimelineStop(SampleScanStatus.Unreadable) - } finally { extractor.release() } } - checkBudget(count, bytes) val video = media.single { it.kind == TrackKind.Video } - val videoTime = tracks.getValue(video.containerIndex.valueOrNull!!) + val videoTime = timingByExtractor.getValue(video.extractorIndex) + val matchers = timingByExtractor.mapValues { PlatformTimestampMatcher(it.value) } + val counts = timingByExtractor.mapValues { 0 }.toMutableMap() + val samples = ArrayList() + val extractor = MediaExtractor() + try { + extractor.setDataSource(file.absolutePath) + media.forEach { extractor.selectTrack(it.extractorIndex) } + while (extractor.sampleTrackIndex >= 0) { + val trackIndex = extractor.sampleTrackIndex + val timing = timingByExtractor[trackIndex] ?: throw SampleScanStop(SampleScanStatus.Unsupported) + val index = counts.getValue(trackIndex) + reader.read(extractor) { time, size, flags, data -> + if (index >= timing.presentationSeconds.size || !matchers.getValue(trackIndex).matches(index, time / 1e6)) + throw SampleScanStop(SampleScanStatus.Unsupported) + if (trackIndex == video.extractorIndex) { + val sync = flags and MediaExtractor.SAMPLE_FLAG_SYNC != 0 + val codec = metadata.facts.videoCodec.valueOrNull + samples.add(CompressedVideoSample((timing.presentationSeconds[index] * 1e6).roundToLong(), size, sync, + if (sync && codec != null) SampleRandomAccessReader.read(codec, data, size) else MediaFact.Unknown)) + } + } + counts[trackIndex] = index + 1 + extractor.advance() + } + if (counts.any { (index, count) -> count != timingByExtractor.getValue(index).presentationSeconds.size }) + throw SampleScanStop(SampleScanStatus.Unreadable) + } finally { extractor.release() } + reader.checkBudget() val firstAudio = media.filter { it.kind == TrackKind.Audio }.minByOrNull { it.containerIndex.valueOrNull!! } - val offset = firstAudio?.let { - AudioVideoStartOffset.Seconds(tracks.getValue(it.containerIndex.valueOrNull!!).startSeconds - videoTime.startSeconds) - } ?: AudioVideoStartOffset.NotApplicable - val audioTime = firstAudio?.let { tracks.getValue(it.containerIndex.valueOrNull!!) } + val audioTime = firstAudio?.let { timingByExtractor.getValue(it.extractorIndex) } + val offset = audioTime?.let { AudioVideoStartOffset.Seconds(it.startSeconds - videoTime.startSeconds) } + ?: AudioVideoStartOffset.NotApplicable + val facts = VideoSampleFactsReader.read(samples, MediaFact.Unknown, metadata.facts, videoTime) + .copy(editList = videoEditList(metadata, video)) + reader.checkBudget() TimelineInspection(StandardInputTimelineFacts(MediaFact.Known(tracks.values.maxOf { it.endSeconds }), MediaFact.Known(offset), MediaFact.Known(videoTime.effectivePresentationSeconds), MediaFact.Known(videoTime.endSeconds - videoTime.startSeconds), - audioTime?.let { MediaFact.Known(it.endSeconds - it.startSeconds) } ?: MediaFact.Unknown), SampleScanStatus.Complete) - } catch (stop: TimelineStop) { failure(stop.status) } + audioTime?.let { MediaFact.Known(it.endSeconds - it.startSeconds) } ?: MediaFact.Unknown, + MediaFact.Known(videoTime.timescale)), SampleScanStatus.Complete, facts) + } catch (stop: SampleScanStop) { failure(stop.status) } catch (_: CancellationException) { failure(SampleScanStatus.Cancelled) } catch (_: LinkageError) { failure(SampleScanStatus.Unsupported) } catch (_: Exception) { failure(if (isCancelled()) SampleScanStatus.Cancelled else SampleScanStatus.Unreadable) } } + } -internal class TimelineStop(val status: SampleScanStatus) : RuntimeException() private fun checkedAdd(a: Long, b: Long): Long { val result = a + b @@ -127,8 +122,34 @@ internal data class IsoTrackTimeline( val endSeconds: Double, val effectivePresentationSeconds: List, val hasEdit: Boolean, + val timescale: Long, + val mediaStartSeconds: Double, + /** Decode-order indices of samples whose presentation intervals overlap the effective edit. */ + val effectiveSampleIndices: List, ) +/** A single table-to-platform timestamp convention must agree for the entire track. */ +internal class PlatformTimestampMatcher(private val timing: IsoTrackTimeline) { + private var shifted = true + private var raw = true + private var clamped = true + private var firstSampleShifted = true + + fun matches(index: Int, actual: Double): Boolean { + val effective = timing.presentationSeconds[index] + val unshifted = effective + timing.mediaStartSeconds + val firstRaw = timing.presentationSeconds.first() + timing.mediaStartSeconds + val legacyShift = minOf(timing.mediaStartSeconds, maxOf(0.0, firstRaw)) + fun near(expected: Double) = actual.isFinite() && abs(actual - expected) <= 2e-6 + shifted = shifted && near(effective) + raw = raw && near(unshifted) + clamped = clamped && near(maxOf(0.0, effective)) + // Android Q limits the video shift to the first composition time and clamps early CTS. + firstSampleShifted = firstSampleShifted && near(maxOf(0.0, unshifted - legacyShift)) + return shifted || raw || clamped || firstSampleShifted + } +} + /** Timing-only extension of the existing bounded ISO box walker; never parses sample sizes or NALs. */ internal object IsoTimelineReader { fun read(file: File, maximumSamples: Int, checkBudget: () -> Unit = {}): Map = @@ -170,7 +191,7 @@ internal object IsoTimelineReader { require(version in (if (composition) 0..1 else 0..0)) require(input.readUnsignedByte() == 0 && input.readUnsignedShort() == 0) val entries = uint() - if (entries > maximumSamples) throw TimelineStop(SampleScanStatus.LimitExceeded) + if (entries > maximumSamples) throw SampleScanStop(SampleScanStatus.LimitExceeded) require(entries > 0 && box.end - box.payload == 8 + entries * 8) var samples = 0L return List(entries.toInt()) { @@ -180,7 +201,7 @@ internal object IsoTimelineReader { // Full extractor reads below must agree before any timeline becomes Known. val value = if (composition) input.readInt().toLong() else uint() samples += n - if (samples > maximumSamples) throw TimelineStop(SampleScanStatus.LimitExceeded) + if (samples > maximumSamples) throw SampleScanStop(SampleScanStatus.LimitExceeded) require(n > 0 && (composition || value > 0)) n.toInt() to value } @@ -188,7 +209,7 @@ internal object IsoTimelineReader { val durations = runs(tables.single { it.type == "stts" }, false) val sampleCount = durations.sumOf { it.first } totalSamples += sampleCount - if (totalSamples > maximumSamples) throw TimelineStop(SampleScanStatus.LimitExceeded) + if (totalSamples > maximumSamples) throw SampleScanStop(SampleScanStatus.LimitExceeded) val deltas = LongArray(sampleCount) var position = 0 for ((count, delta) in durations) repeat(count) { deltas[position++] = delta } @@ -223,7 +244,7 @@ internal object IsoTimelineReader { val version = input.readUnsignedByte() require(version in 0..1 && input.readUnsignedByte() == 0 && input.readUnsignedShort() == 0) val count = uint() - if (count > 1) throw TimelineStop(SampleScanStatus.Unsupported) + if (count > 1) throw SampleScanStop(SampleScanStatus.Unsupported) require(list.end - list.payload == 8 + count * (if (version == 0) 12 else 20)) if (count == 1L) { val duration = if (version == 0) uint() else input.readLong() @@ -244,10 +265,10 @@ internal object IsoTimelineReader { rawEnd + 2.0 / scale + 1.0 / movieScale >= mediaStart + end) } else require(rawStart >= 0) val platformTimes = DoubleArray(sampleCount) { starts[it].toDouble() / scale - mediaStart } - val effectiveTimes = starts.indices.filter { ends[it].toDouble() / scale > mediaStart && platformTimes[it] < end } - .map { maxOf(0.0, platformTimes[it]) }.sorted() + val effectiveIndices = starts.indices.filter { ends[it].toDouble() / scale > mediaStart && platformTimes[it] < end } + val effectiveTimes = effectiveIndices.map { maxOf(start, platformTimes[it]) }.sorted() require(effectiveTimes.isNotEmpty()) - index to IsoTrackTimeline(platformTimes, start, end, effectiveTimes, editDuration != null) + index to IsoTrackTimeline(platformTimes, start, end, effectiveTimes, editDuration != null, scale, mediaStart, effectiveIndices) }.toMap() } } diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/MediaSampleInspectorTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/MediaSampleInspectorTests.kt index 68a5ce0a..659d6d3e 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/MediaSampleInspectorTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/MediaSampleInspectorTests.kt @@ -138,7 +138,7 @@ class MediaSampleInspectorTests : AbsRobolectricTest() { private fun scan(limits: SampleScanLimits = SampleScanLimits(), cancel: () -> Boolean = { false }, reader: ((ByteBuffer) -> Int)? = null, reportedSize: Long = 6, flagsOverride: Int? = null, - input: MediaMetadataInspection = metadata(), provenDuration: MediaFact? = null): MediaSampleInspection { + input: MediaMetadataInspection = metadata()): MediaSampleInspection { mockkConstructor(MediaExtractor::class) var index = 0 var released = false @@ -156,7 +156,7 @@ class MediaSampleInspectorTests : AbsRobolectricTest() { every { anyConstructed().advance() } answers { ++index < 3 } every { anyConstructed().release() } answers { released = true } return try { - MediaSampleInspector(limits).inspect(File("unused"), input, provenDuration, cancel).also { + MediaSampleInspector(limits).inspect(File("unused"), input, cancel).also { assertTrue(released) } } finally { unmockkConstructor(MediaExtractor::class) } @@ -202,7 +202,7 @@ class MediaSampleInspectorTests : AbsRobolectricTest() { assertEquals(SampleScanStatus.LimitExceeded, scan(SampleScanLimits(maximumSampleBytes = 16), reader = { read = true; 6 }, reportedSize = 17).status) assertFalse(read) - assertEquals(SampleScanStatus.LimitExceeded, scan(reportedSize = 7).status) + assertEquals(SampleScanStatus.Unreadable, scan(reportedSize = 7).status) assertEquals(SampleScanStatus.Complete, scan().status) assertEquals(SampleScanStatus.Unreadable, scan(reader = { throw IllegalArgumentException() }).status) } @@ -239,14 +239,43 @@ class MediaSampleInspectorTests : AbsRobolectricTest() { assertEquals(MediaFact.Unknown, SampleRandomAccessReader.read(VideoCodec.H264, data, data.size)) } finally { unmockkStatic(NalUnitUtil::class) } } - @Test fun provenOutputDurationSuppliesBitrateAndFinalGopDenominator() { - val source = scan() - assertEquals(MediaFact.Known(48L), source.facts.averageBitrate) - val output = scan(provenDuration = MediaFact.Known(2.5)) - assertEquals(SampleScanStatus.Complete, output.status) - assertEquals(MediaFact.Known(58L), output.facts.averageBitrate) - assertEquals(MediaFact.Known(58L), output.facts.maximumGopBitrate) - assertEquals(MediaFact.Known(2.5), output.facts.maximumKeyframeIntervalSeconds) + private fun editedTimeline(times: List, end: Double, presented: List): IsoTrackTimeline = + IsoTrackTimeline(times.toDoubleArray(), 0.0, end, presented.map { maxOf(0.0, times[it]) }.sorted(), + true, 1000, 1.0, presented) + + @Test fun effectiveEditExcludesTrimmedBytesButRetainsThePrerollIdrProof() { + val samples = listOf(sample(-1_000_000, true, 1000), sample(0, bytes = 100), sample(1_000_000, bytes = 100)) + val time = editedTimeline(listOf(-1.0, 0.0, 1.0), 2.0, listOf(1, 2)) + val facts = VideoSampleFactsReader.read(samples, MediaFact.Unknown, timeline = time) + assertEquals(known(800L), facts.averageBitrate) + assertEquals(known(800L), facts.maximumGopBitrate) + assertEquals(known(200L), facts.maximumGopByteSize) + assertEquals(known(2.0), facts.maximumKeyframeIntervalSeconds) + assertEquals(idr, facts.gopStructure) + assertEquals(known(TimestampFacts(0.0, 1.0, 2, true)), facts.timestamps) + val unproven = samples.toMutableList().apply { this[0] = this[0].copy(randomAccess = MediaFact.Unknown) } + val rejected = VideoSampleFactsReader.read(unproven, MediaFact.Unknown, timeline = time) + assertEquals(MediaFact.Unknown, rejected.gopStructure) + assertEquals(MediaFact.Unknown, rejected.maximumGopBitrate) + } + + @Test fun editedVfrFinalGopEndsAtTheProvenEndpointInsteadOfRawFirstPtsPlusDuration() { + val samples = (0..4).map { sample((it - 1) * 1_000_000L, it == 0 || it == 2) } + val time = editedTimeline(listOf(-1.0, 0.0, 1.0, 2.0, 3.0), 4.5, listOf(1, 2, 3, 4)) + val facts = VideoSampleFactsReader.read(samples, MediaFact.Unknown, timeline = time) + assertEquals(known(3.5), facts.maximumKeyframeIntervalSeconds) + assertEquals(known(711L), facts.averageBitrate) + assertEquals(known(800L), facts.maximumGopBitrate) } + @Test fun samplesWhollyPastTheEditDoNotChangePresentedCadenceOrGops() { + val samples = listOf(sample(0, true), sample(1_000_000), + sample(2_000_000, true, 10000, known(GopStructure.Open)), sample(4_000_000)) + val time = editedTimeline(listOf(0.0,1.0,2.0,4.0), 2.0, listOf(0,1)) + val facts = VideoSampleFactsReader.read(samples, MediaFact.Unknown, timeline = time) + assertEquals(known(Cadence.Constant), facts.cadence) + assertEquals(idr, facts.gopStructure) + assertEquals(known(800L), facts.averageBitrate) + assertEquals(known(2.0), facts.maximumKeyframeIntervalSeconds) + } } diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputOutputValidatorTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputOutputValidatorTests.kt index 5cc5ba00..a728320b 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputOutputValidatorTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputOutputValidatorTests.kt @@ -11,7 +11,7 @@ class StandardInputOutputValidatorTests { options(), fullCapabilities()).action as StandardInputAction.Convert).conversion private val output = source.copy(encodedDimensions = known(plan.outputDimensions), rotationDegrees = known(0)) private fun timeline(duration: Double = 3.0, offset: AudioVideoStartOffset = AudioVideoStartOffset.Seconds(0.0)) = - StandardInputTimelineFacts(known(duration), known(offset), known(listOf(0.0, 1.0, 2.0)), known(duration), known(duration)) + StandardInputTimelineFacts(known(duration), known(offset), known(listOf(0.0, 1.0, 2.0)), known(duration), known(duration), known(90000L)) private val validator = StandardInputOutputValidator() private fun validate(facts: MediaFacts = output, input: MediaFacts = source, sourceTime: StandardInputTimelineFacts = timeline(), outputTime: StandardInputTimelineFacts = timeline(), @@ -124,11 +124,10 @@ class StandardInputOutputValidatorTests { file.writeBytes(byteArrayOf(1)) for (status in SampleScanStatus.entries.filter { it != SampleScanStatus.Complete }) { val v = StandardInputOutputValidator(inspectMetadata = { MetadataInspectionResult.Success(metadata) }, - inspectSamples = { _, _, _, _ -> MediaSampleInspection(output, status, 1, 1, 1, 0) }, - inspectTimeline = { _, _, _ -> TimelineInspection(timeline(), SampleScanStatus.Complete) }) + inspectTimeline = { _, _, _ -> TimelineInspection(timeline(), status, output) }) val result = v.validateGeneratedOutput(file, source, timeline(), plan) if (status == SampleScanStatus.Cancelled) assertEquals(StandardInputOutputValidation.Cancelled, result) - else assertEquals(OutputRejection.SampleInspection(status), reason(result)) + else assertEquals(OutputRejection.TimelineInspection(status), reason(result)) } var cancelled = false val v = StandardInputOutputValidator(inspectMetadata = { cancelled = true; MetadataInspectionResult.Failure(MetadataFailure.Malformed) }) @@ -140,7 +139,6 @@ class StandardInputOutputValidatorTests { val metadata = MediaMetadataInspection(known(ContainerKind.IsoBaseMedia), emptyList(), MediaFact.Unknown, output, 0) fun validator(timelineRead: () -> TimelineInspection) = StandardInputOutputValidator( inspectMetadata = { MetadataInspectionResult.Success(metadata) }, - inspectSamples = { _, _, _, _ -> MediaSampleInspection(output, SampleScanStatus.Complete, 3, 3, 1, 0) }, inspectTimeline = { _, _, _ -> timelineRead() }) try { file.writeBytes(byteArrayOf(1)) @@ -149,9 +147,9 @@ class StandardInputOutputValidatorTests { if (status == SampleScanStatus.Cancelled) assertEquals(StandardInputOutputValidation.Cancelled, result) else assertEquals(OutputRejection.TimelineInspection(status), reason(result)) } - val changed = validator { file.appendBytes(byteArrayOf(2)); TimelineInspection(timeline(), SampleScanStatus.Complete) } + val changed = validator { file.appendBytes(byteArrayOf(2)); TimelineInspection(timeline(), SampleScanStatus.Complete, output) } assertEquals(OutputRejection.ChangedDuringInspection, reason(changed.validateGeneratedOutput(file, source, timeline(), plan))) - val accepted = validator { TimelineInspection(timeline(), SampleScanStatus.Complete) }.validateGeneratedOutput(file, source, timeline(), plan) + val accepted = validator { TimelineInspection(timeline(), SampleScanStatus.Complete, output) }.validateGeneratedOutput(file, source, timeline(), plan) assertTrue(accepted is StandardInputOutputValidation.Accepted) } finally { file.delete() } } @@ -164,4 +162,34 @@ class StandardInputOutputValidatorTests { assertTrue(reason(validate(sourceTime = timeline().copy(firstAudioDurationSeconds = MediaFact.Unknown))) is OutputRejection.InsufficientPlanEvidence) } + + @Test fun timestampPreservationAllowsOnlyProvenOutputTickRounding() { + val rate = 24000.0 / 1001 + val times = List(72) { it * 1001.0 / 24000 } + val rounded = times.map { kotlin.math.round(kotlin.math.floor(it * 1e6) * 0.09) / 90000 } + val inputTime = timeline().copy(videoPresentationSeconds = known(times)) + val outputTime = timeline().copy(videoPresentationSeconds = known(rounded)) + val conversion = plan.copy(outputFrameRate = rate) + val facts = output.copy(frameRate = known(rate)) + assertTrue(validate(facts, sourceTime = inputTime, outputTime = outputTime, conversion = conversion) + is StandardInputOutputValidation.Accepted) + val drifted = rounded.toMutableList().apply { this[10] += 0.000020 } + mismatch(OutputExpectation.Timestamps, validate(facts, sourceTime = inputTime, + outputTime = outputTime.copy(videoPresentationSeconds = known(drifted)), conversion = conversion)) + for (scale in listOf(MediaFact.Unknown, known(0L), known(-1L))) + assertTrue(reason(validate(facts, sourceTime = inputTime, + outputTime = outputTime.copy(videoTimescale = scale), conversion = conversion)) is OutputRejection.InsufficientPlanEvidence) + } + + @Test fun completedTimelineWithoutSampleProofCannotAcceptOutput() { + val file = File.createTempFile("validation", ".mp4") + try { + file.writeBytes(byteArrayOf(1)) + val metadata = MediaMetadataInspection(known(ContainerKind.IsoBaseMedia), emptyList(), MediaFact.Unknown, output, 0) + val validator = StandardInputOutputValidator(inspectMetadata = { MetadataInspectionResult.Success(metadata) }, + inspectTimeline = { _, _, _ -> TimelineInspection(timeline(), SampleScanStatus.Complete) }) + assertEquals(OutputRejection.TimelineInspection(SampleScanStatus.Unreadable), + reason(validator.validateGeneratedOutput(file, source, timeline(), plan))) + } finally { file.delete() } + } } diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspectorTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspectorTests.kt index bcfba4c8..c0da537f 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspectorTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspectorTests.kt @@ -1,6 +1,7 @@ package com.mux.video.upload.internal.standardization import android.media.MediaExtractor +import android.os.Build import com.mux.exoplayeradapter.AbsRobolectricTest import io.mockk.* import org.junit.Assert.* @@ -35,7 +36,7 @@ class StandardInputTimelineInspectorTests : AbsRobolectricTest() { return try { IsoTimelineReader.read(file, maximum).getValue(0) } finally { file.delete() } } private fun invalid(tracks: ByteArray, scale: Int = 1000, maximum: Int = 250000) { - try { read(tracks, scale, maximum); fail("Unproven timeline was published") } catch (_: IllegalArgumentException) { } catch (stop: TimelineStop) { assertEquals(SampleScanStatus.LimitExceeded, stop.status) } + try { read(tracks, scale, maximum); fail("Unproven timeline was published") } catch (_: IllegalArgumentException) { } catch (stop: SampleScanStop) { assertEquals(SampleScanStatus.LimitExceeded, stop.status) } } @Test fun provesFinalVfrSampleDurationWithoutUsingReportedHeaderDuration() { val time = read(track(deltas = listOf(100, 200, 700))) @@ -79,7 +80,7 @@ class StandardInputTimelineInspectorTests : AbsRobolectricTest() { invalid(track(), maximum = 2) val file = file(extra = box("moof", byteArrayOf())) try { - try { IsoTimelineReader.read(file, 250000); fail("Fragment accepted") } catch (_: IllegalArgumentException) { } catch (stop: TimelineStop) { assertEquals(SampleScanStatus.LimitExceeded, stop.status) } + try { IsoTimelineReader.read(file, 250000); fail("Fragment accepted") } catch (_: IllegalArgumentException) { } catch (stop: SampleScanStop) { assertEquals(SampleScanStatus.LimitExceeded, stop.status) } } finally { file.delete() } } @@ -92,25 +93,53 @@ class StandardInputTimelineInspectorTests : AbsRobolectricTest() { } private fun inspect(audio: Boolean = false, audioOffset: Int = 0, sampleLimit: Int = 250000, missingTail: Boolean = false, partial: Boolean = false, cancelAfterRead: Boolean = false, - legacyBufferFailure: Boolean = false): TimelineInspection { - val file = file(track() + if (audio) track(kind = "soun", offsets = List(3) { audioOffset }) else byteArrayOf()) + legacyBufferFailure: Boolean = false, apiLevel: Int = 23, expectedSize: Long = 6, readSize: Int = 6, + videoEdit: Long? = null, audioEdit: Long? = null, platformMode: String = "shifted", + mixedConvention: Boolean = false, byteLimit: Long = 1024L * 1024 * 1024): TimelineInspection { + val file = file(track(edits = videoEdit?.let { edit(2000, it) } ?: byteArrayOf()) + + if (audio) track(kind = "soun", offsets = List(3) { audioOffset }, + edits = audioEdit?.let { edit(2000, it) } ?: byteArrayOf()) else byteArrayOf()) mockkConstructor(MediaExtractor::class) - var selected = 0; var index = 0; var cancelled = false; var releases = 0 - every { anyConstructed().setDataSource(any()) } just Runs - every { anyConstructed().selectTrack(any()) } answers { selected = firstArg(); index = 0 } - every { anyConstructed().sampleTrackIndex } answers { if (index < if (missingTail) 2 else 3) selected else -1 } - every { anyConstructed().sampleTime } answers { index * 1000000L + if (selected == 1) audioOffset * 1000L else 0L } - every { anyConstructed().sampleFlags } returns if (partial) 4 else 0 + var position = 0; var cancelled = false; var releases = 0; var sources = 0; var reads = 0 + val selected = mutableSetOf() + val count = if (missingTail) 2 else 3 + // Interleaved tracks preserve decode order within each track. + val sequence = (0 until count).flatMap { index -> (0..if (audio) 1 else 0).map { it to index } } + every { anyConstructed().setDataSource(any()) } answers { sources++ } + every { anyConstructed().selectTrack(any()) } answers { selected.add(firstArg()) } + every { anyConstructed().sampleTrackIndex } answers { sequence.getOrNull(position)?.first ?: -1 } + every { anyConstructed().sampleTime } answers { + val (track, index) = sequence[position] + val start = if (track == 0) videoEdit ?: 0L else audioEdit ?: 0L + val raw = index * 1000000L + if (track == 1) audioOffset * 1000L else 0L + if (platformMode == "raw" || (mixedConvention && index == 1)) raw + else if (platformMode == "clamped") maxOf(0L, raw - start * 1000L) else raw - start * 1000L + } + if (Build.VERSION.SDK_INT >= 28) every { anyConstructed().sampleSize } returns expectedSize + every { anyConstructed().sampleFlags } answers { if (partial) 4 else if (sequence[position].second == 0) 1 else 0 } every { anyConstructed().readSampleData(any(), 0) } answers { + reads++ if (legacyBufferFailure) throw IllegalArgumentException("Capped buffer") if (cancelAfterRead) cancelled = true - 4 + if (readSize == 6) { + if (firstArg().capacity() < 6) throw IllegalArgumentException("Capped buffer") + firstArg().put(byteArrayOf(0,0,0,1,0x65,0x80.toByte())) + } + readSize } - every { anyConstructed().advance() } answers { ++index < 3 } + every { anyConstructed().advance() } answers { ++position < sequence.size } every { anyConstructed().release() } answers { releases++ } return try { - StandardInputTimelineInspector(SampleScanLimits(maximumSamples = sampleLimit), 23).inspect(file, metadata(audio)) { cancelled } - .also { if (sampleLimit >= 3) assertTrue(releases > 0) } + StandardInputTimelineInspector(SampleScanLimits(maximumSamples = sampleLimit, maximumReadBytes = byteLimit), apiLevel) + .inspect(file, metadata(audio).copy(facts = metadata(audio).facts.copy(videoCodec = known(VideoCodec.H264)))) { cancelled } + .also { + if (it.status == SampleScanStatus.Complete) { + assertEquals(1, sources); assertEquals(1, releases) + assertEquals(if (audio) setOf(0,1) else setOf(0), selected) + assertEquals(sequence.size, reads) + assertNotNull(it.sampleFacts) + } + } } finally { unmockkConstructor(MediaExtractor::class); file.delete() } } @Test fun noAudioHasExplicitNotApplicableOffsetAndEffectiveEndpoint() { @@ -139,4 +168,52 @@ class StandardInputTimelineInspectorTests : AbsRobolectricTest() { metadata(false).copy(container = known(ContainerKind.Matroska))) { true } assertEquals(SampleScanStatus.Cancelled, result.status) } + + @Test @Config(sdk = [28]) fun rawShiftedAndClampedEditTimesUseTheSameEffectiveTimeline() { + for (api in listOf(23,28,29,36)) for (mode in listOf("raw","shifted","clamped")) { + val result = inspect(audio = true, videoEdit = 1000, audioEdit = 1000, apiLevel = api, platformMode = mode) + assertEquals("$api $mode", SampleScanStatus.Complete, result.status) + assertEquals(known(2.0), result.timeline.durationSeconds) + assertEquals(known(listOf(0.0,1.0)), result.timeline.videoPresentationSeconds) + assertEquals(known(48L), result.sampleFacts!!.averageBitrate) + assertEquals(known(2.0), result.sampleFacts.maximumKeyframeIntervalSeconds) + } + assertEquals(SampleScanStatus.Unsupported, inspect(videoEdit = 1000, mixedConvention = true).status) + } + + @Test fun extractorConventionMustAgreeForEverySampleIncludingBFrameReordering() { + val time = read(track(offsets = listOf(500,1500,-500), edits = edit(2000,1000))) + val shifted = PlatformTimestampMatcher(time) + assertTrue(time.presentationSeconds.indices.all { shifted.matches(it, time.presentationSeconds[it]) }) + val capped = PlatformTimestampMatcher(time) + val raw = time.presentationSeconds.map { it + time.mediaStartSeconds } + assertTrue(raw.indices.all { capped.matches(it, maxOf(0.0, raw[it] - 0.5)) }) + val wrong = PlatformTimestampMatcher(time) + assertTrue(wrong.matches(0, raw[0])) + assertFalse(wrong.matches(1, raw[1] + 0.001)) + } + + @Test @Config(sdk = [28]) fun modernSizeChecksHaveTheSameBoundsAsLegacyReads() { + assertEquals(SampleScanStatus.Complete, inspect(apiLevel = 28).status) + assertEquals(SampleScanStatus.Unreadable, inspect(apiLevel = 28, expectedSize = 7).status) + assertEquals(SampleScanStatus.LimitExceeded, inspect(apiLevel = 28, expectedSize = 0).status) + assertEquals(SampleScanStatus.LimitExceeded, inspect(apiLevel = 28, expectedSize = 4194305).status) + assertEquals(SampleScanStatus.LimitExceeded, inspect(apiLevel = 23, readSize = 4194305).status) + assertEquals(SampleScanStatus.Unreadable, inspect(apiLevel = 23, readSize = -1).status) + } + + @Test fun combinedTracksShareOneBudgetAndExactEofLimitsAreAllowed() { + assertEquals(SampleScanStatus.Complete, inspect(audio = true, sampleLimit = 6, byteLimit = 36).status) + assertEquals(SampleScanStatus.LimitExceeded, inspect(audio = true, sampleLimit = 5).status) + assertEquals(SampleScanStatus.LimitExceeded, inspect(audio = true, byteLimit = 35).status) + } + + @Test fun editedVfrEndpointAndTimescaleAreCarriedToSampleFacts() { + val time = read(track(deltas = listOf(1000,1000,1000,1000,2000), edits = edit(4500,1000))) + assertEquals(1000L, time.timescale) + assertEquals(4.5, time.endSeconds, 0.0) + val samples = List(5) { CompressedVideoSample((it - 1) * 1000000L, 100, it == 0 || it == 2, + known(GopStructure.ClosedWithIdr)) } + assertEquals(known(3.5), VideoSampleFactsReader.read(samples, MediaFact.Unknown, timeline = time).maximumKeyframeIntervalSeconds) + } } From a24ebedc5e7eb98443a985ebebb8b83ddc6e6fbc Mon Sep 17 00:00:00 2001 From: Eugene Morozov Date: Tue, 6 Oct 2026 13:28:14 -0700 Subject: [PATCH 16/49] Unify ISO inspection and include trimmed video payload --- .../standardization/MediaSampleInspector.kt | 17 +++- .../StandardInputTimelineInspector.kt | 26 +++--- .../MediaSampleInspectorTests.kt | 27 ++++-- .../StandardInputTimelineInspectorTests.kt | 82 +++++++++++++++++-- 4 files changed, 126 insertions(+), 26 deletions(-) diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/MediaSampleInspector.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/MediaSampleInspector.kt index 6b8c946a..a817e571 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/MediaSampleInspector.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/MediaSampleInspector.kt @@ -28,6 +28,8 @@ internal data class MediaSampleInspection( val bytesRead: Long, val largestSampleBytes: Int, val elapsedNanos: Long, + /** Proven ISO timeline from this scan; unknown for other containers or incomplete reads. */ + val timeline: StandardInputTimelineFacts = StandardInputTimelineFacts(), ) internal data class CompressedVideoSample( @@ -37,11 +39,17 @@ internal data class CompressedVideoSample( val randomAccess: MediaFact = MediaFact.Unknown, ) -/** Separate from metadata inspection. No decoding, exporting, or effective timeline calculation. */ +/** ISO source facts and timeline share the output scan; other containers retain bounded raw observations. */ internal class MediaSampleInspector(private val limits: SampleScanLimits = SampleScanLimits(), private val apiLevel: Int = Build.VERSION.SDK_INT) { fun inspect(file: File, metadata: MediaMetadataInspection, isCancelled: () -> Boolean = { Thread.currentThread().isInterrupted }): MediaSampleInspection { + if (metadata.container == MediaFact.Known(ContainerKind.IsoBaseMedia)) { + val inspection = StandardInputTimelineInspector(limits, apiLevel).inspect(file, metadata, isCancelled) + return MediaSampleInspection(inspection.sampleFacts ?: metadata.facts, inspection.status, + inspection.sampleCount, inspection.bytesRead, inspection.largestSampleBytes, inspection.elapsedNanos, + inspection.timeline) + } val reader = BoundedSampleReader(limits, apiLevel, isCancelled) val samples = ArrayList() fun result(status: SampleScanStatus, facts: MediaFacts = metadata.facts) = @@ -184,7 +192,8 @@ internal object VideoSampleFactsReader { val duration = timeline?.let { it.endSeconds - it.startSeconds } ?: reportedDuration.valueOrNull?.takeIf { it.isFinite() && it > span && it - span <= intervals.max() / 1e6 * 2 } - if (duration != null) facts = facts.copy(averageBitrate = bitrate(measured.sumOf { it.byteSize.toLong() }, duration)) + // Whole-track payload includes preroll and trimmed tails, even when cadence uses the effective edit. + if (duration != null) facts = facts.copy(averageBitrate = bitrate(samples.sumOf { it.byteSize.toLong() }, duration)) if (!samples.first().isSync) return facts val boundaries = samples.indices.filter { samples[it].isSync } val presentedSet = presented?.toHashSet() @@ -214,7 +223,9 @@ internal object VideoSampleFactsReader { val seconds = endTime - startTime if (!seconds.isFinite() || seconds <= 0) return facts maxInterval = maxOf(maxInterval, seconds) - val bytes = (start until end).sumOf { if (isPresented(it)) samples[it].byteSize.toLong() else 0L } + // Count the complete intersecting GOP, including its IDR and possible decode dependencies + // outside the edit. Without a reference graph, this is a conservative payload bound. + val bytes = (start until end).sumOf { samples[it].byteSize.toLong() } maxBytes = maxOf(maxBytes, bytes) maxRate = maxOf(maxRate, bitrate(bytes, seconds).valueOrNull ?: return facts) } diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspector.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspector.kt index 5a3d12b3..46335505 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspector.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspector.kt @@ -28,6 +28,10 @@ internal data class TimelineInspection( val status: SampleScanStatus, /** Facts proven by the same payload scan; never published after a partial read. */ val sampleFacts: MediaFacts? = null, + val sampleCount: Int = 0, + val bytesRead: Long = 0, + val largestSampleBytes: Int = 0, + val elapsedNanos: Long = 0, ) /** @@ -42,21 +46,23 @@ internal class StandardInputTimelineInspector( fun inspect(file: File, metadata: MediaMetadataInspection, isCancelled: () -> Boolean = { Thread.currentThread().isInterrupted }): TimelineInspection { val reader = BoundedSampleReader(limits, apiLevel, isCancelled) - fun failure(status: SampleScanStatus) = TimelineInspection(status = status) + fun result(status: SampleScanStatus, timeline: StandardInputTimelineFacts = StandardInputTimelineFacts(), + facts: MediaFacts? = null) = TimelineInspection(timeline, status, facts, reader.sampleCount, + reader.bytesRead, reader.largestSampleBytes, reader.elapsedNanos) return try { reader.checkBudget() if (metadata.container != MediaFact.Known(ContainerKind.IsoBaseMedia) || metadata.facts.videoTrackCount != MediaFact.Known(1) || metadata.facts.audioTracks == MediaFact.Unknown) - return failure(SampleScanStatus.Unsupported) + return result(SampleScanStatus.Unsupported) val tracks = IsoTimelineReader.read(file, limits.maximumSamples) { reader.checkBudget() } val media = metadata.tracks.filter { it.kind == TrackKind.Video || it.kind == TrackKind.Audio } if (media.size != tracks.size || media.any { it.containerIndex.valueOrNull !in tracks.keys }) - return failure(SampleScanStatus.Unsupported) + return result(SampleScanStatus.Unsupported) val timingByExtractor = media.associate { it.extractorIndex to tracks.getValue(it.containerIndex.valueOrNull!!) } for (track in media) { if (track.kind == TrackKind.Audio && !timingByExtractor.getValue(track.extractorIndex).hasEdit && track.audio?.let { (it.encoderDelaySamples.valueOrNull ?: 0) > 0 || (it.encoderPaddingSamples.valueOrNull ?: 0) > 0 - } == true) return failure(SampleScanStatus.Unsupported) + } == true) return result(SampleScanStatus.Unsupported) } val video = media.single { it.kind == TrackKind.Video } val videoTime = timingByExtractor.getValue(video.extractorIndex) @@ -95,15 +101,15 @@ internal class StandardInputTimelineInspector( val facts = VideoSampleFactsReader.read(samples, MediaFact.Unknown, metadata.facts, videoTime) .copy(editList = videoEditList(metadata, video)) reader.checkBudget() - TimelineInspection(StandardInputTimelineFacts(MediaFact.Known(tracks.values.maxOf { it.endSeconds }), + result(SampleScanStatus.Complete, StandardInputTimelineFacts(MediaFact.Known(tracks.values.maxOf { it.endSeconds }), MediaFact.Known(offset), MediaFact.Known(videoTime.effectivePresentationSeconds), MediaFact.Known(videoTime.endSeconds - videoTime.startSeconds), audioTime?.let { MediaFact.Known(it.endSeconds - it.startSeconds) } ?: MediaFact.Unknown, - MediaFact.Known(videoTime.timescale)), SampleScanStatus.Complete, facts) - } catch (stop: SampleScanStop) { failure(stop.status) } - catch (_: CancellationException) { failure(SampleScanStatus.Cancelled) } - catch (_: LinkageError) { failure(SampleScanStatus.Unsupported) } - catch (_: Exception) { failure(if (isCancelled()) SampleScanStatus.Cancelled else SampleScanStatus.Unreadable) } + MediaFact.Known(videoTime.timescale)), facts) + } catch (stop: SampleScanStop) { result(stop.status) } + catch (_: CancellationException) { result(SampleScanStatus.Cancelled) } + catch (_: LinkageError) { result(SampleScanStatus.Unsupported) } + catch (_: Exception) { result(if (isCancelled()) SampleScanStatus.Cancelled else SampleScanStatus.Unreadable) } } } diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/MediaSampleInspectorTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/MediaSampleInspectorTests.kt index 659d6d3e..76ce80af 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/MediaSampleInspectorTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/MediaSampleInspectorTests.kt @@ -243,13 +243,13 @@ class MediaSampleInspectorTests : AbsRobolectricTest() { IsoTrackTimeline(times.toDoubleArray(), 0.0, end, presented.map { maxOf(0.0, times[it]) }.sorted(), true, 1000, 1.0, presented) - @Test fun effectiveEditExcludesTrimmedBytesButRetainsThePrerollIdrProof() { + @Test fun effectiveEditCountsPrerollBytesAndRetainsTheIdrProof() { val samples = listOf(sample(-1_000_000, true, 1000), sample(0, bytes = 100), sample(1_000_000, bytes = 100)) val time = editedTimeline(listOf(-1.0, 0.0, 1.0), 2.0, listOf(1, 2)) val facts = VideoSampleFactsReader.read(samples, MediaFact.Unknown, timeline = time) - assertEquals(known(800L), facts.averageBitrate) - assertEquals(known(800L), facts.maximumGopBitrate) - assertEquals(known(200L), facts.maximumGopByteSize) + assertEquals(known(4800L), facts.averageBitrate) + assertEquals(known(4800L), facts.maximumGopBitrate) + assertEquals(known(1200L), facts.maximumGopByteSize) assertEquals(known(2.0), facts.maximumKeyframeIntervalSeconds) assertEquals(idr, facts.gopStructure) assertEquals(known(TimestampFacts(0.0, 1.0, 2, true)), facts.timestamps) @@ -264,18 +264,29 @@ class MediaSampleInspectorTests : AbsRobolectricTest() { val time = editedTimeline(listOf(-1.0, 0.0, 1.0, 2.0, 3.0), 4.5, listOf(1, 2, 3, 4)) val facts = VideoSampleFactsReader.read(samples, MediaFact.Unknown, timeline = time) assertEquals(known(3.5), facts.maximumKeyframeIntervalSeconds) - assertEquals(known(711L), facts.averageBitrate) - assertEquals(known(800L), facts.maximumGopBitrate) + assertEquals(known(889L), facts.averageBitrate) + assertEquals(known(1600L), facts.maximumGopBitrate) } - @Test fun samplesWhollyPastTheEditDoNotChangePresentedCadenceOrGops() { + @Test fun samplesWhollyPastTheEditCountTowardWholeTrackBitrateButNotPresentedCadence() { val samples = listOf(sample(0, true), sample(1_000_000), sample(2_000_000, true, 10000, known(GopStructure.Open)), sample(4_000_000)) val time = editedTimeline(listOf(0.0,1.0,2.0,4.0), 2.0, listOf(0,1)) val facts = VideoSampleFactsReader.read(samples, MediaFact.Unknown, timeline = time) assertEquals(known(Cadence.Constant), facts.cadence) assertEquals(idr, facts.gopStructure) - assertEquals(known(800L), facts.averageBitrate) + assertEquals(known(41200L), facts.averageBitrate) assertEquals(known(2.0), facts.maximumKeyframeIntervalSeconds) } + + @Test fun trimmingCannotHideOversizedDecodePrerollOrTailFromBitratePolicy() { + val samples = listOf(sample(-1_000_000, true, 2_000_000), sample(0), sample(100_000), + sample(200_000, bytes = 2_000_000)) + val time = editedTimeline(listOf(-1.0, 0.0, 0.1, 0.2), 0.2, listOf(1,2)) + val facts = VideoSampleFactsReader.read(samples, MediaFact.Unknown, compliantFacts(), time) + val evaluation = StandardInputPolicyEvaluator().evaluate(facts, selection(), MediaRole.GeneratedOutput) + assertTrue(evaluation.nonCompliantRequirements.contains(PolicyRequirement.AverageBitrate)) + assertTrue(evaluation.nonCompliantRequirements.contains(PolicyRequirement.MaximumGopBitrate)) + assertEquals(known(4_000_200L), facts.maximumGopByteSize) + } } diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspectorTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspectorTests.kt index c0da537f..c97e3098 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspectorTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspectorTests.kt @@ -95,7 +95,8 @@ class StandardInputTimelineInspectorTests : AbsRobolectricTest() { missingTail: Boolean = false, partial: Boolean = false, cancelAfterRead: Boolean = false, legacyBufferFailure: Boolean = false, apiLevel: Int = 23, expectedSize: Long = 6, readSize: Int = 6, videoEdit: Long? = null, audioEdit: Long? = null, platformMode: String = "shifted", - mixedConvention: Boolean = false, byteLimit: Long = 1024L * 1024 * 1024): TimelineInspection { + mixedConvention: Boolean = false, byteLimit: Long = 1024L * 1024 * 1024, + sourcePath: Boolean = false): TimelineInspection { val file = file(track(edits = videoEdit?.let { edit(2000, it) } ?: byteArrayOf()) + if (audio) track(kind = "soun", offsets = List(3) { audioOffset }, edits = audioEdit?.let { edit(2000, it) } ?: byteArrayOf()) else byteArrayOf()) @@ -130,9 +131,15 @@ class StandardInputTimelineInspectorTests : AbsRobolectricTest() { every { anyConstructed().advance() } answers { ++position < sequence.size } every { anyConstructed().release() } answers { releases++ } return try { - StandardInputTimelineInspector(SampleScanLimits(maximumSamples = sampleLimit, maximumReadBytes = byteLimit), apiLevel) - .inspect(file, metadata(audio).copy(facts = metadata(audio).facts.copy(videoCodec = known(VideoCodec.H264)))) { cancelled } - .also { + val limits = SampleScanLimits(maximumSamples = sampleLimit, maximumReadBytes = byteLimit) + val input = metadata(audio).copy(facts = metadata(audio).facts.copy(videoCodec = known(VideoCodec.H264))) + val result = if (sourcePath) { + val source = MediaSampleInspector(limits, apiLevel).inspect(file, input) { cancelled } + if (source.status != SampleScanStatus.Complete) assertEquals(input.facts, source.facts) + TimelineInspection(source.timeline, source.status, source.facts.takeIf { source.status == SampleScanStatus.Complete }, + source.sampleCount, source.bytesRead, source.largestSampleBytes, source.elapsedNanos) + } else StandardInputTimelineInspector(limits, apiLevel).inspect(file, input) { cancelled } + result.also { if (it.status == SampleScanStatus.Complete) { assertEquals(1, sources); assertEquals(1, releases) assertEquals(if (audio) setOf(0,1) else setOf(0), selected) @@ -175,7 +182,7 @@ class StandardInputTimelineInspectorTests : AbsRobolectricTest() { assertEquals("$api $mode", SampleScanStatus.Complete, result.status) assertEquals(known(2.0), result.timeline.durationSeconds) assertEquals(known(listOf(0.0,1.0)), result.timeline.videoPresentationSeconds) - assertEquals(known(48L), result.sampleFacts!!.averageBitrate) + assertEquals(known(72L), result.sampleFacts!!.averageBitrate) assertEquals(known(2.0), result.sampleFacts.maximumKeyframeIntervalSeconds) } assertEquals(SampleScanStatus.Unsupported, inspect(videoEdit = 1000, mixedConvention = true).status) @@ -216,4 +223,69 @@ class StandardInputTimelineInspectorTests : AbsRobolectricTest() { known(GopStructure.ClosedWithIdr)) } assertEquals(known(3.5), VideoSampleFactsReader.read(samples, MediaFact.Unknown, timeline = time).maximumKeyframeIntervalSeconds) } + + @Test fun editedIsoSourceCadencePlansAndValidatesOnlyThePresentedWindow() { + val file = file(track(deltas = listOf(25,200,200,200), edits = edit(600,25))) + val input = metadata(false).copy( + isoTracks = known(listOf(IsoTrackMetadata("vide", known(1), listOf("avc1"), false, + known(0), true, known(EditList.Simple)))), + facts = compliantFacts(dimensions = Dimensions(2560,1440)).copy(audioTracks = known(emptyList()))) + val rawTimes = listOf(0L,25000L,225000L,425000L) + mockkConstructor(MediaExtractor::class) + var index = 0 + every { anyConstructed().setDataSource(any()) } just Runs + every { anyConstructed().selectTrack(0) } just Runs + every { anyConstructed().sampleTrackIndex } answers { if (index < rawTimes.size) 0 else -1 } + every { anyConstructed().sampleTime } answers { rawTimes[index] } + every { anyConstructed().sampleFlags } answers { if (index == 0) 1 else 0 } + every { anyConstructed().readSampleData(any(), 0) } answers { + firstArg().put(byteArrayOf(0,0,0,1,0x65,0x80.toByte())); 6 + } + every { anyConstructed().advance() } answers { ++index < rawTimes.size } + every { anyConstructed().release() } just Runs + try { + val source = MediaSampleInspector().inspect(file, input) + assertEquals(SampleScanStatus.Complete, source.status) + assertEquals(known(5.0), source.facts.frameRate) + assertEquals(known(Cadence.Constant), source.facts.cadence) + val conversion = (StandardInputPlanner().plan(source.facts, options(), fullCapabilities()) + .action as StandardInputAction.Convert).conversion + assertEquals(5.0, conversion.outputFrameRate, 0.0) + assertEquals(OutputCadence.PreserveSourceTimestamps, conversion.outputCadence) + val time = StandardInputTimelineFacts(known(0.6), known(AudioVideoStartOffset.NotApplicable), + known(listOf(0.0,0.2,0.4)), known(0.6), videoTimescale = known(1000L)) + assertEquals(time.copy(videoPresentationSeconds = source.timeline.videoPresentationSeconds), source.timeline) + time.videoPresentationSeconds.valueOrNull!!.zip(source.timeline.videoPresentationSeconds.valueOrNull!!).forEach { + (expected, actual) -> assertEquals(expected, actual, 1e-12) + } + assertEquals(4, source.sampleCount) + assertEquals(24L, source.bytesRead) + assertEquals(6, source.largestSampleBytes) + assertTrue(source.elapsedNanos > 0) + val output = source.facts.copy(displayDimensions = known(conversion.outputDimensions), + encodedDimensions = known(conversion.outputDimensions), rotationDegrees = known(0)) + val result = StandardInputOutputValidator().validateFacts(output, source.facts, source.timeline, time, conversion) + assertTrue("Correctly trimmed output rejected: $result", result is StandardInputOutputValidation.Accepted) + verify(exactly = 1) { anyConstructed().setDataSource(file.absolutePath) } + verify(exactly = 4) { anyConstructed().readSampleData(any(), 0) } + verify(exactly = 1) { anyConstructed().release() } + } finally { unmockkConstructor(MediaExtractor::class); file.delete() } + } + + @Test fun isoSourceSharesTheCombinedBudgetAndNeverFallsBackToRawPartialFacts() { + val complete = inspect(audio = true, sourcePath = true, sampleLimit = 6, byteLimit = 36) + assertEquals(SampleScanStatus.Complete, complete.status) + assertEquals(6, complete.sampleCount) + assertEquals(36L, complete.bytesRead) + for ((result, status) in listOf( + inspect(audio = true, sourcePath = true, sampleLimit = 5) to SampleScanStatus.LimitExceeded, + inspect(sourcePath = true, cancelAfterRead = true) to SampleScanStatus.Cancelled, + inspect(sourcePath = true, missingTail = true) to SampleScanStatus.Unreadable, + inspect(sourcePath = true, videoEdit = 1000, mixedConvention = true) to SampleScanStatus.Unsupported)) { + assertEquals(status, result.status) + assertNull(result.sampleFacts) + assertEquals(StandardInputTimelineFacts(), result.timeline) + assertTrue(result.elapsedNanos > 0) + } + } } From e17ae3cfa0c91d706a99f971a35834b60df74533 Mon Sep 17 00:00:00 2001 From: Eugene Morozov Date: Tue, 6 Oct 2026 15:07:25 -0700 Subject: [PATCH 17/49] Simplify inspection results and regression setup --- .../standardization/BoundedSampleReader.kt | 4 + .../standardization/MediaSampleInspector.kt | 38 ++++--- .../StandardInputOutputValidator.kt | 8 +- .../StandardInputTimelineInspector.kt | 16 +-- .../StandardInputOutputValidatorTests.kt | 21 ++-- .../StandardInputTimelineInspectorTests.kt | 102 +++++++----------- 6 files changed, 74 insertions(+), 115 deletions(-) diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/BoundedSampleReader.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/BoundedSampleReader.kt index e2bfae48..177d5b91 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/BoundedSampleReader.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/BoundedSampleReader.kt @@ -19,6 +19,10 @@ internal class BoundedSampleReader( var largestSampleBytes = 0; private set val elapsedNanos get() = System.nanoTime() - started + fun result(facts: MediaFacts, status: SampleScanStatus, + timeline: StandardInputTimelineFacts = StandardInputTimelineFacts()) = + MediaSampleInspection(facts, status, sampleCount, bytesRead, largestSampleBytes, elapsedNanos, timeline) + fun checkBudget(beforeRead: Boolean = false) { if (isCancelled()) throw SampleScanStop(SampleScanStatus.Cancelled) if (elapsedNanos >= limits.maximumElapsedNanos || diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/MediaSampleInspector.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/MediaSampleInspector.kt index a817e571..10fe7f80 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/MediaSampleInspector.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/MediaSampleInspector.kt @@ -24,10 +24,10 @@ internal data class SampleScanLimits( internal data class MediaSampleInspection( val facts: MediaFacts, val status: SampleScanStatus, - val sampleCount: Int, - val bytesRead: Long, - val largestSampleBytes: Int, - val elapsedNanos: Long, + val sampleCount: Int = 0, + val bytesRead: Long = 0, + val largestSampleBytes: Int = 0, + val elapsedNanos: Long = 0, /** Proven ISO timeline from this scan; unknown for other containers or incomplete reads. */ val timeline: StandardInputTimelineFacts = StandardInputTimelineFacts(), ) @@ -44,16 +44,12 @@ internal class MediaSampleInspector(private val limits: SampleScanLimits = Sampl private val apiLevel: Int = Build.VERSION.SDK_INT) { fun inspect(file: File, metadata: MediaMetadataInspection, isCancelled: () -> Boolean = { Thread.currentThread().isInterrupted }): MediaSampleInspection { - if (metadata.container == MediaFact.Known(ContainerKind.IsoBaseMedia)) { - val inspection = StandardInputTimelineInspector(limits, apiLevel).inspect(file, metadata, isCancelled) - return MediaSampleInspection(inspection.sampleFacts ?: metadata.facts, inspection.status, - inspection.sampleCount, inspection.bytesRead, inspection.largestSampleBytes, inspection.elapsedNanos, - inspection.timeline) - } + if (metadata.container == MediaFact.Known(ContainerKind.IsoBaseMedia)) + return StandardInputTimelineInspector(limits, apiLevel).inspect(file, metadata, isCancelled) val reader = BoundedSampleReader(limits, apiLevel, isCancelled) val samples = ArrayList() fun result(status: SampleScanStatus, facts: MediaFacts = metadata.facts) = - MediaSampleInspection(facts, status, reader.sampleCount, reader.bytesRead, reader.largestSampleBytes, reader.elapsedNanos) + reader.result(facts, status) if (isCancelled()) return result(SampleScanStatus.Cancelled) val track = metadata.tracks.filter { it.kind == TrackKind.Video }.singleOrNull() ?: return result(SampleScanStatus.Unsupported) @@ -168,24 +164,26 @@ internal object VideoSampleFactsReader { if (samples.size < 2 || samples.any { it.byteSize <= 0 }) return base if (timeline != null && timeline.presentationSeconds.size != samples.size) return base val presented = timeline?.effectiveSampleIndices - if (presented != null && presented.size < 2) return base fun time(index: Int) = timeline?.presentationSeconds?.get(index) ?: (samples[index].presentationTimeUs / 1e6) - val measured = if (timeline == null) samples else presented!!.map { samples[it].copy(presentationTimeUs = - (maxOf(timeline?.startSeconds ?: Double.NEGATIVE_INFINITY, time(it)) * 1e6).roundToLong()) } - val ordered = LongArray(measured.size) { measured[it].presentationTimeUs }.also { it.sort() } - val intervals = LongArray(measured.size - 1) { ordered[it + 1] - ordered[it] } + val ordered = LongArray(presented?.size ?: samples.size) { + val index = presented?.get(it) ?: it + if (timeline == null) samples[index].presentationTimeUs + else (maxOf(timeline.startSeconds, time(index)) * 1e6).roundToLong() + } + if (ordered.size < 2) return base + val matchesDecodeOrder = (1 until ordered.size).all { ordered[it - 1] < ordered[it] } + ordered.sort() + val intervals = LongArray(ordered.size - 1) { ordered[it + 1] - ordered[it] } if (intervals.any { it <= 0 }) return base val span = (ordered.last() - ordered.first()) / 1e6 if (!span.isFinite() || span <= 0) return base - val frameInterval = span / (measured.size - 1) + val frameInterval = span / (ordered.size - 1) val constant = intervals.all { abs(it / 1e6 - frameInterval) <= maxOf(2e-6, frameInterval * 0.001) } var facts = base.copy( frameRate = MediaFact.Known(1 / frameInterval), cadence = MediaFact.Known(if (constant) Cadence.Constant else Cadence.Variable), timestamps = MediaFact.Known(TimestampFacts(ordered.first() / 1e6, ordered.last() / 1e6, - measured.size.toLong(), (1 until measured.size).all { - measured[it - 1].presentationTimeUs < measured[it].presentationTimeUs - })), + ordered.size.toLong(), matchesDecodeOrder)), ) // Only a proven effective window replaces the raw source-duration observation. // Unproven metadata must not extrapolate the final VFR interval. diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputOutputValidator.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputOutputValidator.kt index 0576c4d7..9a67dfb9 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputOutputValidator.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputOutputValidator.kt @@ -31,7 +31,7 @@ internal sealed interface StandardInputOutputValidation { internal class StandardInputOutputValidator( private val evaluator: StandardInputPolicyEvaluator = StandardInputPolicyEvaluator(), private val inspectMetadata: (File) -> MetadataInspectionResult = MediaMetadataInspector()::inspect, - private val inspectTimeline: (File, MediaMetadataInspection, () -> Boolean) -> TimelineInspection = + private val inspectTimeline: (File, MediaMetadataInspection, () -> Boolean) -> MediaSampleInspection = { file, metadata, cancelled -> StandardInputTimelineInspector().inspect(file, metadata, cancelled) }, ) { fun validateGeneratedOutput(file: File, source: MediaFacts, sourceTimeline: StandardInputTimelineFacts, @@ -53,13 +53,9 @@ internal class StandardInputOutputValidator( val timeline = inspectTimeline(file, metadata, isCancelled) if (isCancelled() || timeline.status == SampleScanStatus.Cancelled) return StandardInputOutputValidation.Cancelled if (timeline.status != SampleScanStatus.Complete) return rejected(OutputRejection.TimelineInspection(timeline.status)) - val duration = timeline.timeline.videoDurationSeconds.valueOrNull - if (duration == null || !duration.isFinite() || duration <= 0) - return rejected(OutputRejection.InsufficientPlanEvidence(setOf(OutputExpectation.Duration))) - val facts = timeline.sampleFacts ?: return rejected(OutputRejection.TimelineInspection(SampleScanStatus.Unreadable)) if (file.length() != initialSize || file.lastModified() != initialModified) return rejected(OutputRejection.ChangedDuringInspection) - val result = validateFacts(facts, source, sourceTimeline, timeline.timeline, conversion) + val result = validateFacts(timeline.facts, source, sourceTimeline, timeline.timeline, conversion) if (isCancelled()) StandardInputOutputValidation.Cancelled else result } catch (_: CancellationException) { StandardInputOutputValidation.Cancelled } catch (_: LinkageError) { failedInspection() } diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspector.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspector.kt index 46335505..bf106563 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspector.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspector.kt @@ -23,17 +23,6 @@ internal data class StandardInputTimelineFacts( val videoTimescale: MediaFact = MediaFact.Unknown, ) -internal data class TimelineInspection( - val timeline: StandardInputTimelineFacts = StandardInputTimelineFacts(), - val status: SampleScanStatus, - /** Facts proven by the same payload scan; never published after a partial read. */ - val sampleFacts: MediaFacts? = null, - val sampleCount: Int = 0, - val bytesRead: Long = 0, - val largestSampleBytes: Int = 0, - val elapsedNanos: Long = 0, -) - /** * Effective ISO presentation time, shared by source inspection and generated-output validation. * Timing tables prove the final sample's duration; full platform reads prove those samples exist. @@ -44,11 +33,10 @@ internal class StandardInputTimelineInspector( private val apiLevel: Int = Build.VERSION.SDK_INT, ) { fun inspect(file: File, metadata: MediaMetadataInspection, - isCancelled: () -> Boolean = { Thread.currentThread().isInterrupted }): TimelineInspection { + isCancelled: () -> Boolean = { Thread.currentThread().isInterrupted }): MediaSampleInspection { val reader = BoundedSampleReader(limits, apiLevel, isCancelled) fun result(status: SampleScanStatus, timeline: StandardInputTimelineFacts = StandardInputTimelineFacts(), - facts: MediaFacts? = null) = TimelineInspection(timeline, status, facts, reader.sampleCount, - reader.bytesRead, reader.largestSampleBytes, reader.elapsedNanos) + facts: MediaFacts = metadata.facts) = reader.result(facts, status, timeline) return try { reader.checkBudget() if (metadata.container != MediaFact.Known(ContainerKind.IsoBaseMedia) || diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputOutputValidatorTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputOutputValidatorTests.kt index a728320b..542bfe7c 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputOutputValidatorTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputOutputValidatorTests.kt @@ -124,7 +124,7 @@ class StandardInputOutputValidatorTests { file.writeBytes(byteArrayOf(1)) for (status in SampleScanStatus.entries.filter { it != SampleScanStatus.Complete }) { val v = StandardInputOutputValidator(inspectMetadata = { MetadataInspectionResult.Success(metadata) }, - inspectTimeline = { _, _, _ -> TimelineInspection(timeline(), status, output) }) + inspectTimeline = { _, _, _ -> MediaSampleInspection(output, status, timeline = timeline()) }) val result = v.validateGeneratedOutput(file, source, timeline(), plan) if (status == SampleScanStatus.Cancelled) assertEquals(StandardInputOutputValidation.Cancelled, result) else assertEquals(OutputRejection.TimelineInspection(status), reason(result)) @@ -134,22 +134,17 @@ class StandardInputOutputValidatorTests { assertEquals(StandardInputOutputValidation.Cancelled, v.validateGeneratedOutput(file, source, timeline(), plan) { cancelled }) } finally { file.delete() } } - @Test fun timelineFailureCancellationAndFileChangesCannotSelectGeneratedBytes() { + @Test fun changedFileCannotSelectGeneratedBytesAfterCompleteInspection() { val file = File.createTempFile("validation", ".mp4") val metadata = MediaMetadataInspection(known(ContainerKind.IsoBaseMedia), emptyList(), MediaFact.Unknown, output, 0) - fun validator(timelineRead: () -> TimelineInspection) = StandardInputOutputValidator( + fun validator(timelineRead: () -> MediaSampleInspection) = StandardInputOutputValidator( inspectMetadata = { MetadataInspectionResult.Success(metadata) }, inspectTimeline = { _, _, _ -> timelineRead() }) try { file.writeBytes(byteArrayOf(1)) - for (status in SampleScanStatus.entries.filter { it != SampleScanStatus.Complete }) { - val result = validator { TimelineInspection(status = status) }.validateGeneratedOutput(file, source, timeline(), plan) - if (status == SampleScanStatus.Cancelled) assertEquals(StandardInputOutputValidation.Cancelled, result) - else assertEquals(OutputRejection.TimelineInspection(status), reason(result)) - } - val changed = validator { file.appendBytes(byteArrayOf(2)); TimelineInspection(timeline(), SampleScanStatus.Complete, output) } + val changed = validator { file.appendBytes(byteArrayOf(2)); MediaSampleInspection(output, SampleScanStatus.Complete, timeline = timeline()) } assertEquals(OutputRejection.ChangedDuringInspection, reason(changed.validateGeneratedOutput(file, source, timeline(), plan))) - val accepted = validator { TimelineInspection(timeline(), SampleScanStatus.Complete, output) }.validateGeneratedOutput(file, source, timeline(), plan) + val accepted = validator { MediaSampleInspection(output, SampleScanStatus.Complete, timeline = timeline()) }.validateGeneratedOutput(file, source, timeline(), plan) assertTrue(accepted is StandardInputOutputValidation.Accepted) } finally { file.delete() } } @@ -187,9 +182,9 @@ class StandardInputOutputValidatorTests { file.writeBytes(byteArrayOf(1)) val metadata = MediaMetadataInspection(known(ContainerKind.IsoBaseMedia), emptyList(), MediaFact.Unknown, output, 0) val validator = StandardInputOutputValidator(inspectMetadata = { MetadataInspectionResult.Success(metadata) }, - inspectTimeline = { _, _, _ -> TimelineInspection(timeline(), SampleScanStatus.Complete) }) - assertEquals(OutputRejection.TimelineInspection(SampleScanStatus.Unreadable), - reason(validator.validateGeneratedOutput(file, source, timeline(), plan))) + inspectTimeline = { _, _, _ -> MediaSampleInspection(MediaFacts(), SampleScanStatus.Complete, timeline = timeline()) }) + assertTrue(reason(validator.validateGeneratedOutput(file, source, timeline(), plan)) + is OutputRejection.InsufficientPolicyEvidence) } finally { file.delete() } } } diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspectorTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspectorTests.kt index c97e3098..24302f6f 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspectorTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspectorTests.kt @@ -96,14 +96,17 @@ class StandardInputTimelineInspectorTests : AbsRobolectricTest() { legacyBufferFailure: Boolean = false, apiLevel: Int = 23, expectedSize: Long = 6, readSize: Int = 6, videoEdit: Long? = null, audioEdit: Long? = null, platformMode: String = "shifted", mixedConvention: Boolean = false, byteLimit: Long = 1024L * 1024 * 1024, - sourcePath: Boolean = false): TimelineInspection { - val file = file(track(edits = videoEdit?.let { edit(2000, it) } ?: byteArrayOf()) + + sourcePath: Boolean = false, videoDeltas: List = listOf(1000,1000,1000), editDuration: Long = 2000, + input: MediaMetadataInspection = metadata(audio).copy(facts = metadata(audio).facts.copy(videoCodec = known(VideoCodec.H264))) + ): MediaSampleInspection { + val file = file(track(deltas = videoDeltas, edits = videoEdit?.let { edit(editDuration, it) } ?: byteArrayOf()) + if (audio) track(kind = "soun", offsets = List(3) { audioOffset }, edits = audioEdit?.let { edit(2000, it) } ?: byteArrayOf()) else byteArrayOf()) mockkConstructor(MediaExtractor::class) var position = 0; var cancelled = false; var releases = 0; var sources = 0; var reads = 0 val selected = mutableSetOf() - val count = if (missingTail) 2 else 3 + val rawTimes = videoDeltas.runningFold(0L) { time, delta -> time + delta * 1000L }.dropLast(1) + val count = rawTimes.size - if (missingTail) 1 else 0 // Interleaved tracks preserve decode order within each track. val sequence = (0 until count).flatMap { index -> (0..if (audio) 1 else 0).map { it to index } } every { anyConstructed().setDataSource(any()) } answers { sources++ } @@ -112,7 +115,7 @@ class StandardInputTimelineInspectorTests : AbsRobolectricTest() { every { anyConstructed().sampleTime } answers { val (track, index) = sequence[position] val start = if (track == 0) videoEdit ?: 0L else audioEdit ?: 0L - val raw = index * 1000000L + if (track == 1) audioOffset * 1000L else 0L + val raw = rawTimes[index] + if (track == 1) audioOffset * 1000L else 0L if (platformMode == "raw" || (mixedConvention && index == 1)) raw else if (platformMode == "clamped") maxOf(0L, raw - start * 1000L) else raw - start * 1000L } @@ -132,21 +135,15 @@ class StandardInputTimelineInspectorTests : AbsRobolectricTest() { every { anyConstructed().release() } answers { releases++ } return try { val limits = SampleScanLimits(maximumSamples = sampleLimit, maximumReadBytes = byteLimit) - val input = metadata(audio).copy(facts = metadata(audio).facts.copy(videoCodec = known(VideoCodec.H264))) - val result = if (sourcePath) { - val source = MediaSampleInspector(limits, apiLevel).inspect(file, input) { cancelled } - if (source.status != SampleScanStatus.Complete) assertEquals(input.facts, source.facts) - TimelineInspection(source.timeline, source.status, source.facts.takeIf { source.status == SampleScanStatus.Complete }, - source.sampleCount, source.bytesRead, source.largestSampleBytes, source.elapsedNanos) - } else StandardInputTimelineInspector(limits, apiLevel).inspect(file, input) { cancelled } - result.also { - if (it.status == SampleScanStatus.Complete) { - assertEquals(1, sources); assertEquals(1, releases) - assertEquals(if (audio) setOf(0,1) else setOf(0), selected) - assertEquals(sequence.size, reads) - assertNotNull(it.sampleFacts) - } - } + val inspector = if (sourcePath) MediaSampleInspector(limits, apiLevel)::inspect + else StandardInputTimelineInspector(limits, apiLevel)::inspect + inspector(file, input) { cancelled }.also { + if (it.status == SampleScanStatus.Complete) { + assertEquals(1, sources); assertEquals(1, releases) + assertEquals(if (audio) setOf(0,1) else setOf(0), selected) + assertEquals(sequence.size, reads) + } else assertEquals(input.facts, it.facts) + } } finally { unmockkConstructor(MediaExtractor::class); file.delete() } } @Test fun noAudioHasExplicitNotApplicableOffsetAndEffectiveEndpoint() { @@ -182,8 +179,8 @@ class StandardInputTimelineInspectorTests : AbsRobolectricTest() { assertEquals("$api $mode", SampleScanStatus.Complete, result.status) assertEquals(known(2.0), result.timeline.durationSeconds) assertEquals(known(listOf(0.0,1.0)), result.timeline.videoPresentationSeconds) - assertEquals(known(72L), result.sampleFacts!!.averageBitrate) - assertEquals(known(2.0), result.sampleFacts.maximumKeyframeIntervalSeconds) + assertEquals(known(72L), result.facts.averageBitrate) + assertEquals(known(2.0), result.facts.maximumKeyframeIntervalSeconds) } assertEquals(SampleScanStatus.Unsupported, inspect(videoEdit = 1000, mixedConvention = true).status) } @@ -225,51 +222,33 @@ class StandardInputTimelineInspectorTests : AbsRobolectricTest() { } @Test fun editedIsoSourceCadencePlansAndValidatesOnlyThePresentedWindow() { - val file = file(track(deltas = listOf(25,200,200,200), edits = edit(600,25))) val input = metadata(false).copy( isoTracks = known(listOf(IsoTrackMetadata("vide", known(1), listOf("avc1"), false, known(0), true, known(EditList.Simple)))), facts = compliantFacts(dimensions = Dimensions(2560,1440)).copy(audioTracks = known(emptyList()))) - val rawTimes = listOf(0L,25000L,225000L,425000L) - mockkConstructor(MediaExtractor::class) - var index = 0 - every { anyConstructed().setDataSource(any()) } just Runs - every { anyConstructed().selectTrack(0) } just Runs - every { anyConstructed().sampleTrackIndex } answers { if (index < rawTimes.size) 0 else -1 } - every { anyConstructed().sampleTime } answers { rawTimes[index] } - every { anyConstructed().sampleFlags } answers { if (index == 0) 1 else 0 } - every { anyConstructed().readSampleData(any(), 0) } answers { - firstArg().put(byteArrayOf(0,0,0,1,0x65,0x80.toByte())); 6 + val source = inspect(sourcePath = true, videoDeltas = listOf(25,200,200,200), videoEdit = 25, + editDuration = 600, platformMode = "raw", input = input) + assertEquals(SampleScanStatus.Complete, source.status) + assertEquals(known(5.0), source.facts.frameRate) + assertEquals(known(Cadence.Constant), source.facts.cadence) + val conversion = (StandardInputPlanner().plan(source.facts, options(), fullCapabilities()) + .action as StandardInputAction.Convert).conversion + assertEquals(5.0, conversion.outputFrameRate, 0.0) + assertEquals(OutputCadence.PreserveSourceTimestamps, conversion.outputCadence) + val time = StandardInputTimelineFacts(known(0.6), known(AudioVideoStartOffset.NotApplicable), + known(listOf(0.0,0.2,0.4)), known(0.6), videoTimescale = known(1000L)) + assertEquals(time.copy(videoPresentationSeconds = source.timeline.videoPresentationSeconds), source.timeline) + time.videoPresentationSeconds.valueOrNull!!.zip(source.timeline.videoPresentationSeconds.valueOrNull!!).forEach { + (expected, actual) -> assertEquals(expected, actual, 1e-12) } - every { anyConstructed().advance() } answers { ++index < rawTimes.size } - every { anyConstructed().release() } just Runs - try { - val source = MediaSampleInspector().inspect(file, input) - assertEquals(SampleScanStatus.Complete, source.status) - assertEquals(known(5.0), source.facts.frameRate) - assertEquals(known(Cadence.Constant), source.facts.cadence) - val conversion = (StandardInputPlanner().plan(source.facts, options(), fullCapabilities()) - .action as StandardInputAction.Convert).conversion - assertEquals(5.0, conversion.outputFrameRate, 0.0) - assertEquals(OutputCadence.PreserveSourceTimestamps, conversion.outputCadence) - val time = StandardInputTimelineFacts(known(0.6), known(AudioVideoStartOffset.NotApplicable), - known(listOf(0.0,0.2,0.4)), known(0.6), videoTimescale = known(1000L)) - assertEquals(time.copy(videoPresentationSeconds = source.timeline.videoPresentationSeconds), source.timeline) - time.videoPresentationSeconds.valueOrNull!!.zip(source.timeline.videoPresentationSeconds.valueOrNull!!).forEach { - (expected, actual) -> assertEquals(expected, actual, 1e-12) - } - assertEquals(4, source.sampleCount) - assertEquals(24L, source.bytesRead) - assertEquals(6, source.largestSampleBytes) - assertTrue(source.elapsedNanos > 0) - val output = source.facts.copy(displayDimensions = known(conversion.outputDimensions), - encodedDimensions = known(conversion.outputDimensions), rotationDegrees = known(0)) - val result = StandardInputOutputValidator().validateFacts(output, source.facts, source.timeline, time, conversion) - assertTrue("Correctly trimmed output rejected: $result", result is StandardInputOutputValidation.Accepted) - verify(exactly = 1) { anyConstructed().setDataSource(file.absolutePath) } - verify(exactly = 4) { anyConstructed().readSampleData(any(), 0) } - verify(exactly = 1) { anyConstructed().release() } - } finally { unmockkConstructor(MediaExtractor::class); file.delete() } + assertEquals(4, source.sampleCount) + assertEquals(24L, source.bytesRead) + assertEquals(6, source.largestSampleBytes) + assertTrue(source.elapsedNanos > 0) + val output = source.facts.copy(displayDimensions = known(conversion.outputDimensions), + encodedDimensions = known(conversion.outputDimensions), rotationDegrees = known(0)) + val result = StandardInputOutputValidator().validateFacts(output, source.facts, source.timeline, time, conversion) + assertTrue("Correctly trimmed output rejected: $result", result is StandardInputOutputValidation.Accepted) } @Test fun isoSourceSharesTheCombinedBudgetAndNeverFallsBackToRawPartialFacts() { @@ -283,7 +262,6 @@ class StandardInputTimelineInspectorTests : AbsRobolectricTest() { inspect(sourcePath = true, missingTail = true) to SampleScanStatus.Unreadable, inspect(sourcePath = true, videoEdit = 1000, mixedConvention = true) to SampleScanStatus.Unsupported)) { assertEquals(status, result.status) - assertNull(result.sampleFacts) assertEquals(StandardInputTimelineFacts(), result.timeline) assertTrue(result.elapsedNanos > 0) } From b1a5125c3f35cba7969fd99db6dc6c16fa2f843f Mon Sep 17 00:00:00 2001 From: Eugene Morozov Date: Tue, 6 Oct 2026 15:34:26 -0700 Subject: [PATCH 18/49] Correct edited bitrate spans and require source timeline proof --- .../standardization/MediaSampleInspector.kt | 9 +++-- .../standardization/StandardInputPlanner.kt | 4 +++ .../StandardInputTimelineInspector.kt | 4 ++- .../MediaSampleInspectorTests.kt | 36 +++++++++++++------ .../StandardInputPlannerTests.kt | 4 +++ .../standardization/StandardInputTestFacts.kt | 1 + .../StandardInputTimelineInspectorTests.kt | 3 +- 7 files changed, 43 insertions(+), 18 deletions(-) diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/MediaSampleInspector.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/MediaSampleInspector.kt index 10fe7f80..4fd63def 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/MediaSampleInspector.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/MediaSampleInspector.kt @@ -185,9 +185,8 @@ internal object VideoSampleFactsReader { timestamps = MediaFact.Known(TimestampFacts(ordered.first() / 1e6, ordered.last() / 1e6, ordered.size.toLong(), matchesDecodeOrder)), ) - // Only a proven effective window replaces the raw source-duration observation. - // Unproven metadata must not extrapolate the final VFR interval. - val duration = timeline?.let { it.endSeconds - it.startSeconds } ?: reportedDuration.valueOrNull?.takeIf { + // Whole-track bytes use the full sample span; metadata must not extrapolate the final VFR interval. + val duration = timeline?.let { it.fullEndSeconds - it.presentationSeconds.min() } ?: reportedDuration.valueOrNull?.takeIf { it.isFinite() && it > span && it - span <= intervals.max() / 1e6 * 2 } // Whole-track payload includes preroll and trimmed tails, even when cadence uses the effective edit. @@ -216,7 +215,7 @@ internal object VideoSampleFactsReader { val end = boundaries.getOrNull(i + 1) ?: samples.size val startTime = maxOf(timeline?.startSeconds ?: Double.NEGATIVE_INFINITY, time(start)) val rawEnd = if (end < samples.size) time(end) - else timeline?.endSeconds ?: duration?.plus(ordered.first() / 1e6) ?: return facts + else timeline?.fullEndSeconds ?: duration?.plus(ordered.first() / 1e6) ?: return facts val endTime = minOf(timeline?.endSeconds ?: Double.POSITIVE_INFINITY, rawEnd) val seconds = endTime - startTime if (!seconds.isFinite() || seconds <= 0) return facts @@ -225,7 +224,7 @@ internal object VideoSampleFactsReader { // outside the edit. Without a reference graph, this is a conservative payload bound. val bytes = (start until end).sumOf { samples[it].byteSize.toLong() } maxBytes = maxOf(maxBytes, bytes) - maxRate = maxOf(maxRate, bitrate(bytes, seconds).valueOrNull ?: return facts) + maxRate = maxOf(maxRate, bitrate(bytes, rawEnd - time(start)).valueOrNull ?: return facts) } return facts.copy(maximumKeyframeIntervalSeconds = MediaFact.Known(maxInterval), // Leading pictures in open/unproven GOPs cannot be assigned reliably to a byte window. diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputPlanner.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputPlanner.kt index 2ec0ab8d..bd71c0bb 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputPlanner.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputPlanner.kt @@ -6,6 +6,8 @@ import com.mux.video.upload.internal.InputStandardization /** Proof for this source and conversion path, supplied by the caller, never inferred from a codec list. */ internal data class PlanningCapabilities( val sourceIsDecodable: Boolean = false, + /** Effective source duration, A/V offset, and video timestamps are proven for output validation. */ + val sourceTimelineIsProven: Boolean = false, val encodableVideoCodecs: Set = emptySet(), /** A FrameRate entry must prove cadence conversion while preserving the effective source timeline. */ val remediableRequirements: Set = emptySet(), @@ -21,6 +23,7 @@ internal data class PlanningCapabilities( ) { fun missingCapabilities(conversion: StandardInputConversion): Set = buildSet { if (!sourceIsDecodable) add(ConversionCapabilityFailure.SourceDecode) + if (!sourceTimelineIsProven) add(ConversionCapabilityFailure.SourceTimeline) if (conversion.outputCodec !in encodableVideoCodecs) add(ConversionCapabilityFailure.VideoEncode) for (requirement in conversion.requirementsToRemediate - remediableRequirements) { add(ConversionCapabilityFailure.Remediation(requirement)) @@ -37,6 +40,7 @@ internal data class PlanningCapabilities( /** Missing proof for the codec, range, and requirements carried by the associated conversion. */ internal sealed interface ConversionCapabilityFailure { data object SourceDecode : ConversionCapabilityFailure + data object SourceTimeline : ConversionCapabilityFailure data object VideoEncode : ConversionCapabilityFailure data class Remediation(val requirement: PolicyRequirement) : ConversionCapabilityFailure data object AacAudioPreparation : ConversionCapabilityFailure diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspector.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspector.kt index bf106563..d19bd4c9 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspector.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspector.kt @@ -114,6 +114,8 @@ internal data class IsoTrackTimeline( val presentationSeconds: DoubleArray, val startSeconds: Double, val endSeconds: Double, + /** End of all sample presentation intervals, including trimmed tails, in the shifted timebase. */ + val fullEndSeconds: Double, val effectivePresentationSeconds: List, val hasEdit: Boolean, val timescale: Long, @@ -262,7 +264,7 @@ internal object IsoTimelineReader { val effectiveIndices = starts.indices.filter { ends[it].toDouble() / scale > mediaStart && platformTimes[it] < end } val effectiveTimes = effectiveIndices.map { maxOf(start, platformTimes[it]) }.sorted() require(effectiveTimes.isNotEmpty()) - index to IsoTrackTimeline(platformTimes, start, end, effectiveTimes, editDuration != null, scale, mediaStart, effectiveIndices) + index to IsoTrackTimeline(platformTimes, start, end, rawEnd - mediaStart, effectiveTimes, editDuration != null, scale, mediaStart, effectiveIndices) }.toMap() } } diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/MediaSampleInspectorTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/MediaSampleInspectorTests.kt index 76ce80af..8a6d2fa7 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/MediaSampleInspectorTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/MediaSampleInspectorTests.kt @@ -239,16 +239,16 @@ class MediaSampleInspectorTests : AbsRobolectricTest() { assertEquals(MediaFact.Unknown, SampleRandomAccessReader.read(VideoCodec.H264, data, data.size)) } finally { unmockkStatic(NalUnitUtil::class) } } - private fun editedTimeline(times: List, end: Double, presented: List): IsoTrackTimeline = - IsoTrackTimeline(times.toDoubleArray(), 0.0, end, presented.map { maxOf(0.0, times[it]) }.sorted(), + private fun editedTimeline(times: List, end: Double, presented: List, fullEnd: Double = end): IsoTrackTimeline = + IsoTrackTimeline(times.toDoubleArray(), 0.0, end, fullEnd, presented.map { maxOf(0.0, times[it]) }.sorted(), true, 1000, 1.0, presented) @Test fun effectiveEditCountsPrerollBytesAndRetainsTheIdrProof() { val samples = listOf(sample(-1_000_000, true, 1000), sample(0, bytes = 100), sample(1_000_000, bytes = 100)) val time = editedTimeline(listOf(-1.0, 0.0, 1.0), 2.0, listOf(1, 2)) val facts = VideoSampleFactsReader.read(samples, MediaFact.Unknown, timeline = time) - assertEquals(known(4800L), facts.averageBitrate) - assertEquals(known(4800L), facts.maximumGopBitrate) + assertEquals(known(3200L), facts.averageBitrate) + assertEquals(known(3200L), facts.maximumGopBitrate) assertEquals(known(1200L), facts.maximumGopByteSize) assertEquals(known(2.0), facts.maximumKeyframeIntervalSeconds) assertEquals(idr, facts.gopStructure) @@ -259,30 +259,44 @@ class MediaSampleInspectorTests : AbsRobolectricTest() { assertEquals(MediaFact.Unknown, rejected.maximumGopBitrate) } - @Test fun editedVfrFinalGopEndsAtTheProvenEndpointInsteadOfRawFirstPtsPlusDuration() { + @Test fun editedVfrUsesFullSpanForBitrateAndEffectiveSpanForKeyframeInterval() { val samples = (0..4).map { sample((it - 1) * 1_000_000L, it == 0 || it == 2) } - val time = editedTimeline(listOf(-1.0, 0.0, 1.0, 2.0, 3.0), 4.5, listOf(1, 2, 3, 4)) + val time = editedTimeline(listOf(-1.0, 0.0, 1.0, 2.0, 3.0), 4.5, listOf(1, 2, 3, 4), fullEnd = 5.0) val facts = VideoSampleFactsReader.read(samples, MediaFact.Unknown, timeline = time) assertEquals(known(3.5), facts.maximumKeyframeIntervalSeconds) - assertEquals(known(889L), facts.averageBitrate) - assertEquals(known(1600L), facts.maximumGopBitrate) + assertEquals(known(667L), facts.averageBitrate) + assertEquals(known(800L), facts.maximumGopBitrate) } @Test fun samplesWhollyPastTheEditCountTowardWholeTrackBitrateButNotPresentedCadence() { val samples = listOf(sample(0, true), sample(1_000_000), sample(2_000_000, true, 10000, known(GopStructure.Open)), sample(4_000_000)) - val time = editedTimeline(listOf(0.0,1.0,2.0,4.0), 2.0, listOf(0,1)) + val time = editedTimeline(listOf(0.0,1.0,2.0,4.0), 2.0, listOf(0,1), fullEnd = 5.0) val facts = VideoSampleFactsReader.read(samples, MediaFact.Unknown, timeline = time) assertEquals(known(Cadence.Constant), facts.cadence) assertEquals(idr, facts.gopStructure) - assertEquals(known(41200L), facts.averageBitrate) + assertEquals(known(16480L), facts.averageBitrate) assertEquals(known(2.0), facts.maximumKeyframeIntervalSeconds) } + @Test fun compliantBitrateDoesNotIncreaseWhenAnEditCutsThroughAGop() { + val samples = List(120) { sample(it * 1_000_000L / 30, it % 60 == 0, 16667) } + for ((start, end) in listOf(59.0 / 30 to 4.0, 2.5 to 4.0, 0.0 to 61.0 / 30)) { + val times = samples.indices.map { it / 30.0 - start } + val presented = times.indices.filter { times[it] >= 0 && times[it] < end - start } + val timeline = editedTimeline(times, end - start, presented, fullEnd = 4.0 - start) + val facts = VideoSampleFactsReader.read(samples, MediaFact.Unknown, compliantFacts(), timeline) + assertEquals(known(4_000_080L), facts.averageBitrate) + assertEquals(known(4_000_080L), facts.maximumGopBitrate) + assertTrue(facts.maximumKeyframeIntervalSeconds.valueOrNull!! <= 2.0) + assertTrue(StandardInputPlanner().plan(facts, options(), fullCapabilities()).action is StandardInputAction.UploadOriginal) + } + } + @Test fun trimmingCannotHideOversizedDecodePrerollOrTailFromBitratePolicy() { val samples = listOf(sample(-1_000_000, true, 2_000_000), sample(0), sample(100_000), sample(200_000, bytes = 2_000_000)) - val time = editedTimeline(listOf(-1.0, 0.0, 0.1, 0.2), 0.2, listOf(1,2)) + val time = editedTimeline(listOf(-1.0, 0.0, 0.1, 0.2), 0.2, listOf(1,2), fullEnd = 0.3) val facts = VideoSampleFactsReader.read(samples, MediaFact.Unknown, compliantFacts(), time) val evaluation = StandardInputPolicyEvaluator().evaluate(facts, selection(), MediaRole.GeneratedOutput) assertTrue(evaluation.nonCompliantRequirements.contains(PolicyRequirement.AverageBitrate)) diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputPlannerTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputPlannerTests.kt index 506d0c43..a4ca9414 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputPlannerTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputPlannerTests.kt @@ -98,6 +98,7 @@ class StandardInputPlannerTests { @Test fun unknownAudioRequiresIndependentAacPreparationProof() { val videoCapabilities = PlanningCapabilities( sourceIsDecodable = true, + sourceTimelineIsProven = true, encodableVideoCodecs = setOf(VideoCodec.H264), remediableRequirements = setOf(PolicyRequirement.FrameRate), ) @@ -295,6 +296,7 @@ class StandardInputPlannerTests { val full = fullCapabilities() val cases = listOf( full.copy(sourceIsDecodable = false) to ConversionCapabilityFailure.SourceDecode, + full.copy(sourceTimelineIsProven = false) to ConversionCapabilityFailure.SourceTimeline, full.copy(encodableVideoCodecs = emptySet()) to ConversionCapabilityFailure.VideoEncode, full.copy(remediableRequirements = emptySet()) to ConversionCapabilityFailure.Remediation(PolicyRequirement.GopStructure), @@ -307,6 +309,8 @@ class StandardInputPlannerTests { assertEquals(VideoCodec.Hevc, reason.conversion.outputCodec) assertEquals(DynamicRange.Hlg, reason.conversion.sourceDynamicRange) } + assertTrue(planner.plan(compliantFacts(), capabilities = full.copy(sourceTimelineIsProven = false)) + .action is StandardInputAction.UploadOriginal) val allMissing = fallback(facts, PlanningCapabilities(), toneMapOptions()) as FallbackReason.UnsupportedConversion assertEquals(cases.map { it.second }.toSet(), allMissing.missingCapabilities) } diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputTestFacts.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputTestFacts.kt index f917f832..2e660b11 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputTestFacts.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputTestFacts.kt @@ -30,6 +30,7 @@ internal fun hdrFacts(range: DynamicRange = DynamicRange.Hlg): MediaFacts = comp internal fun fullCapabilities() = PlanningCapabilities( sourceIsDecodable = true, + sourceTimelineIsProven = true, encodableVideoCodecs = setOf(VideoCodec.H264, VideoCodec.Hevc), remediableRequirements = PolicyRequirement.entries.toSet(), toneMappableDynamicRanges = setOf(DynamicRange.Hlg, DynamicRange.Pq), diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspectorTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspectorTests.kt index 24302f6f..17b1ea4c 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspectorTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspectorTests.kt @@ -179,7 +179,7 @@ class StandardInputTimelineInspectorTests : AbsRobolectricTest() { assertEquals("$api $mode", SampleScanStatus.Complete, result.status) assertEquals(known(2.0), result.timeline.durationSeconds) assertEquals(known(listOf(0.0,1.0)), result.timeline.videoPresentationSeconds) - assertEquals(known(72L), result.facts.averageBitrate) + assertEquals(known(48L), result.facts.averageBitrate) assertEquals(known(2.0), result.facts.maximumKeyframeIntervalSeconds) } assertEquals(SampleScanStatus.Unsupported, inspect(videoEdit = 1000, mixedConvention = true).status) @@ -216,6 +216,7 @@ class StandardInputTimelineInspectorTests : AbsRobolectricTest() { val time = read(track(deltas = listOf(1000,1000,1000,1000,2000), edits = edit(4500,1000))) assertEquals(1000L, time.timescale) assertEquals(4.5, time.endSeconds, 0.0) + assertEquals(5.0, time.fullEndSeconds, 0.0) val samples = List(5) { CompressedVideoSample((it - 1) * 1000000L, 100, it == 0 || it == 2, known(GopStructure.ClosedWithIdr)) } assertEquals(known(3.5), VideoSampleFactsReader.read(samples, MediaFact.Unknown, timeline = time).maximumKeyframeIntervalSeconds) From 74d725006cdb6dafbd338197fb616f6b7281a712 Mon Sep 17 00:00:00 2001 From: Eugene Morozov Date: Tue, 6 Oct 2026 15:48:23 -0700 Subject: [PATCH 19/49] Remove non-ISO sample scanning and raw duration guesses --- .../standardization/MediaSampleInspector.kt | 93 +++--------------- .../StandardInputTimelineInspector.kt | 8 +- ...ectorTests.kt => MediaSampleFactsTests.kt} | 94 +++++++------------ .../StandardInputPlannerTests.kt | 6 ++ .../StandardInputTimelineInspectorTests.kt | 42 +++++---- 5 files changed, 84 insertions(+), 159 deletions(-) rename library/src/test/java/com/mux/video/upload/internal/standardization/{MediaSampleInspectorTests.kt => MediaSampleFactsTests.kt} (76%) diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/MediaSampleInspector.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/MediaSampleInspector.kt index 4fd63def..1ba860d4 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/MediaSampleInspector.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/MediaSampleInspector.kt @@ -1,10 +1,6 @@ package com.mux.video.upload.internal.standardization -import android.media.MediaExtractor -import android.os.Build import androidx.media3.container.NalUnitUtil -import java.io.File -import java.util.concurrent.CancellationException import kotlin.math.abs import kotlin.math.roundToLong @@ -33,64 +29,11 @@ internal data class MediaSampleInspection( ) internal data class CompressedVideoSample( - val presentationTimeUs: Long, val byteSize: Int, val isSync: Boolean, val randomAccess: MediaFact = MediaFact.Unknown, ) -/** ISO source facts and timeline share the output scan; other containers retain bounded raw observations. */ -internal class MediaSampleInspector(private val limits: SampleScanLimits = SampleScanLimits(), - private val apiLevel: Int = Build.VERSION.SDK_INT) { - fun inspect(file: File, metadata: MediaMetadataInspection, - isCancelled: () -> Boolean = { Thread.currentThread().isInterrupted }): MediaSampleInspection { - if (metadata.container == MediaFact.Known(ContainerKind.IsoBaseMedia)) - return StandardInputTimelineInspector(limits, apiLevel).inspect(file, metadata, isCancelled) - val reader = BoundedSampleReader(limits, apiLevel, isCancelled) - val samples = ArrayList() - fun result(status: SampleScanStatus, facts: MediaFacts = metadata.facts) = - reader.result(facts, status) - if (isCancelled()) return result(SampleScanStatus.Cancelled) - val track = metadata.tracks.filter { it.kind == TrackKind.Video }.singleOrNull() - ?: return result(SampleScanStatus.Unsupported) - if (metadata.facts.videoTrackCount != MediaFact.Known(1) || - metadata.facts.videoCodec.valueOrNull !in listOf(VideoCodec.H264, VideoCodec.Hevc)) - return result(SampleScanStatus.Unsupported) - val codec = metadata.facts.videoCodec.valueOrNull!! - val extractor = MediaExtractor() - return try { - extractor.setDataSource(file.absolutePath) - extractor.selectTrack(track.extractorIndex) - while (extractor.sampleTrackIndex >= 0) { - reader.read(extractor) { time, size, flags, data -> - val sync = flags and MediaExtractor.SAMPLE_FLAG_SYNC != 0 - samples.add(CompressedVideoSample(time, size, sync, - if (sync) SampleRandomAccessReader.read(codec, data, size) else MediaFact.Unknown)) - } - extractor.advance() - } - reader.checkBudget() - val facts = VideoSampleFactsReader.read(samples, track.reportedDurationSeconds, metadata.facts) - .copy(editList = videoEditList(metadata, track)) - reader.checkBudget() - result(SampleScanStatus.Complete, facts) - } catch (stop: SampleScanStop) { result(stop.status) } - catch (_: CancellationException) { result(SampleScanStatus.Cancelled) } - catch (_: Exception) { result(if (isCancelled()) SampleScanStatus.Cancelled else SampleScanStatus.Unreadable) } - finally { extractor.release() } - } - -} - -/** Match the shared video-edit policy; audio edits remain separate track observations. */ -internal fun videoEditList(metadata: MediaMetadataInspection, track: TrackMetadata): MediaFact = - when (metadata.container.valueOrNull) { - ContainerKind.IsoBaseMedia -> metadata.isoTracks.valueOrNull - ?.getOrNull(track.containerIndex.valueOrNull ?: -1)?.editList ?: MediaFact.Unknown - ContainerKind.Matroska -> MediaFact.Known(EditList.None) - else -> MediaFact.Unknown - } - /** Only sync samples need relevant NAL inspection. Unrecognized framing or syntax stays unknown. */ internal object SampleRandomAccessReader { fun read(codec: VideoCodec, data: ByteArray, size: Int): MediaFact = try { @@ -159,17 +102,12 @@ internal object SampleRandomAccessReader { } internal object VideoSampleFactsReader { - fun read(samples: List, reportedDuration: MediaFact, - base: MediaFacts = MediaFacts(), timeline: IsoTrackTimeline? = null): MediaFacts { + fun read(samples: List, timeline: IsoTrackTimeline, base: MediaFacts = MediaFacts()): MediaFacts { if (samples.size < 2 || samples.any { it.byteSize <= 0 }) return base - if (timeline != null && timeline.presentationSeconds.size != samples.size) return base - val presented = timeline?.effectiveSampleIndices - fun time(index: Int) = timeline?.presentationSeconds?.get(index) ?: (samples[index].presentationTimeUs / 1e6) - val ordered = LongArray(presented?.size ?: samples.size) { - val index = presented?.get(it) ?: it - if (timeline == null) samples[index].presentationTimeUs - else (maxOf(timeline.startSeconds, time(index)) * 1e6).roundToLong() - } + if (timeline.presentationSeconds.size != samples.size) return base + val presented = timeline.effectiveSampleIndices + fun time(index: Int) = timeline.presentationSeconds[index] + val ordered = LongArray(presented.size) { (maxOf(timeline.startSeconds, time(presented[it])) * 1e6).roundToLong() } if (ordered.size < 2) return base val matchesDecodeOrder = (1 until ordered.size).all { ordered[it - 1] < ordered[it] } ordered.sort() @@ -185,18 +123,14 @@ internal object VideoSampleFactsReader { timestamps = MediaFact.Known(TimestampFacts(ordered.first() / 1e6, ordered.last() / 1e6, ordered.size.toLong(), matchesDecodeOrder)), ) - // Whole-track bytes use the full sample span; metadata must not extrapolate the final VFR interval. - val duration = timeline?.let { it.fullEndSeconds - it.presentationSeconds.min() } ?: reportedDuration.valueOrNull?.takeIf { - it.isFinite() && it > span && it - span <= intervals.max() / 1e6 * 2 - } - // Whole-track payload includes preroll and trimmed tails, even when cadence uses the effective edit. - if (duration != null) facts = facts.copy(averageBitrate = bitrate(samples.sumOf { it.byteSize.toLong() }, duration)) + // Whole-track bytes include preroll and trimmed tails, over their full proven sample span. + val duration = timeline.fullEndSeconds - timeline.presentationSeconds.min() + facts = facts.copy(averageBitrate = bitrate(samples.sumOf { it.byteSize.toLong() }, duration)) if (!samples.first().isSync) return facts val boundaries = samples.indices.filter { samples[it].isSync } - val presentedSet = presented?.toHashSet() - fun isPresented(index: Int) = presentedSet == null || index in presentedSet + val presentedSet = presented.toHashSet() val relevant = boundaries.indices.filter { i -> - presentedSet == null || (boundaries[i] until (boundaries.getOrNull(i + 1) ?: samples.size)).any { isPresented(it) } + (boundaries[i] until (boundaries.getOrNull(i + 1) ?: samples.size)).any { it in presentedSet } } // Keep the original IDR boundary, including decode preroll outside the effective edit. val kinds = relevant.map { samples[boundaries[it]].randomAccess } @@ -213,10 +147,9 @@ internal object VideoSampleFactsReader { for (i in relevant) { val start = boundaries[i] val end = boundaries.getOrNull(i + 1) ?: samples.size - val startTime = maxOf(timeline?.startSeconds ?: Double.NEGATIVE_INFINITY, time(start)) - val rawEnd = if (end < samples.size) time(end) - else timeline?.fullEndSeconds ?: duration?.plus(ordered.first() / 1e6) ?: return facts - val endTime = minOf(timeline?.endSeconds ?: Double.POSITIVE_INFINITY, rawEnd) + val startTime = maxOf(timeline.startSeconds, time(start)) + val rawEnd = if (end < samples.size) time(end) else timeline.fullEndSeconds + val endTime = minOf(timeline.endSeconds, rawEnd) val seconds = endTime - startTime if (!seconds.isFinite() || seconds <= 0) return facts maxInterval = maxOf(maxInterval, seconds) diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspector.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspector.kt index d19bd4c9..b10871ee 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspector.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspector.kt @@ -6,7 +6,6 @@ import java.io.File import java.io.RandomAccessFile import java.util.concurrent.CancellationException import kotlin.math.abs -import kotlin.math.roundToLong internal sealed interface AudioVideoStartOffset { data object NotApplicable : AudioVideoStartOffset @@ -71,7 +70,7 @@ internal class StandardInputTimelineInspector( if (trackIndex == video.extractorIndex) { val sync = flags and MediaExtractor.SAMPLE_FLAG_SYNC != 0 val codec = metadata.facts.videoCodec.valueOrNull - samples.add(CompressedVideoSample((timing.presentationSeconds[index] * 1e6).roundToLong(), size, sync, + samples.add(CompressedVideoSample(size, sync, if (sync && codec != null) SampleRandomAccessReader.read(codec, data, size) else MediaFact.Unknown)) } } @@ -86,8 +85,9 @@ internal class StandardInputTimelineInspector( val audioTime = firstAudio?.let { timingByExtractor.getValue(it.extractorIndex) } val offset = audioTime?.let { AudioVideoStartOffset.Seconds(it.startSeconds - videoTime.startSeconds) } ?: AudioVideoStartOffset.NotApplicable - val facts = VideoSampleFactsReader.read(samples, MediaFact.Unknown, metadata.facts, videoTime) - .copy(editList = videoEditList(metadata, video)) + val facts = VideoSampleFactsReader.read(samples, videoTime, metadata.facts) + .copy(editList = metadata.isoTracks.valueOrNull + ?.getOrNull(video.containerIndex.valueOrNull ?: -1)?.editList ?: MediaFact.Unknown) reader.checkBudget() result(SampleScanStatus.Complete, StandardInputTimelineFacts(MediaFact.Known(tracks.values.maxOf { it.endSeconds }), MediaFact.Known(offset), MediaFact.Known(videoTime.effectivePresentationSeconds), diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/MediaSampleInspectorTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/MediaSampleFactsTests.kt similarity index 76% rename from library/src/test/java/com/mux/video/upload/internal/standardization/MediaSampleInspectorTests.kt rename to library/src/test/java/com/mux/video/upload/internal/standardization/MediaSampleFactsTests.kt index 8a6d2fa7..836cc879 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/MediaSampleInspectorTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/MediaSampleFactsTests.kt @@ -12,12 +12,14 @@ import java.io.File import java.nio.ByteBuffer @Config(sdk = [23]) -class MediaSampleInspectorTests : AbsRobolectricTest() { +class MediaSampleFactsTests : AbsRobolectricTest() { private val idr = MediaFact.Known(GopStructure.ClosedWithIdr) private fun sample(time: Long, sync: Boolean = false, bytes: Int = 100, - kind: MediaFact = idr) = CompressedVideoSample(time, bytes, sync, kind) - private fun facts(samples: List, duration: Double = 3.0) = - VideoSampleFactsReader.read(samples, MediaFact.Known(duration)) + kind: MediaFact = idr) = time to CompressedVideoSample(bytes, sync, kind) + private fun facts(samples: List>, duration: Double = 3.0): MediaFacts { + val times = samples.map { it.first / 1e6 } + return VideoSampleFactsReader.read(samples.map { it.second }, editedTimeline(times, duration, times.indices.toList())) + } private fun nal(vararg bytes: Int) = byteArrayOf(0, 0, 0, 1) + bytes.map { it.toByte() }.toByteArray() @Test fun measuresWholeVideoSeparatelyFromMaximumGopAndIncludesFinalTail() { @@ -42,11 +44,11 @@ class MediaSampleInspectorTests : AbsRobolectricTest() { assertFalse(facts.timestamps.valueOrNull!!.presentationOrderMatchesDecodeOrder) } - @Test fun variableIntervalsRemainVariableAndNegativePrerollIsObserved() { - val facts = facts(listOf(sample(-100_000, true), sample(100_000), sample(400_000)), 0.8) + @Test fun variablePresentedIntervalsRemainVariable() { + val facts = facts(listOf(sample(0, true), sample(200_000), sample(500_000)), 0.8) assertEquals(MediaFact.Known(Cadence.Variable), facts.cadence) assertEquals(MediaFact.Known(4.0), facts.frameRate) - assertEquals(-0.1, facts.timestamps.valueOrNull!!.firstPresentationSeconds, 0.0) + assertEquals(0.0, facts.timestamps.valueOrNull!!.firstPresentationSeconds, 0.0) } @Test fun noSyncAtStartCannotProveGops() { @@ -70,16 +72,6 @@ class MediaSampleInspectorTests : AbsRobolectricTest() { assertEquals(MediaFact.Known(2.0), facts.maximumKeyframeIntervalSeconds) } - @Test fun contradictoryOrMissingDurationDoesNotExtrapolateVfrTail() { - val samples = listOf(sample(0, true), sample(1_000_000), sample(2_000_000)) - for (duration in listOf(MediaFact.Unknown, MediaFact.Known(1.0), MediaFact.Known(100.0))) { - val facts = VideoSampleFactsReader.read(samples, duration) - assertEquals(MediaFact.Known(1.0), facts.frameRate) - assertEquals(MediaFact.Unknown, facts.averageBitrate) - assertEquals(MediaFact.Unknown, facts.maximumKeyframeIntervalSeconds) - } - } - @Test fun duplicateTimesEmptyOrInvalidBytesCannotInventRates() { for (samples in listOf(emptyList(), listOf(sample(0, true)), listOf(sample(0, true), sample(0)), listOf(sample(0, true), sample(1_000_000, bytes = 0)))) @@ -137,37 +129,39 @@ class MediaSampleInspectorTests : AbsRobolectricTest() { MediaFacts(videoCodec = MediaFact.Known(VideoCodec.H264), videoTrackCount = MediaFact.Known(1)), 0) private fun scan(limits: SampleScanLimits = SampleScanLimits(), cancel: () -> Boolean = { false }, - reader: ((ByteBuffer) -> Int)? = null, reportedSize: Long = 6, flagsOverride: Int? = null, - input: MediaMetadataInspection = metadata()): MediaSampleInspection { - mockkConstructor(MediaExtractor::class) + reader: ((ByteBuffer) -> Int)? = null, reportedSize: Long = 6, flagsOverride: Int? = null): MediaSampleInspection { + val extractor = mockk() + val bounded = BoundedSampleReader(limits, Build.VERSION.SDK_INT, cancel) var index = 0 - var released = false val bytes = nal(0x65, 0x80) - every { anyConstructed().setDataSource(any()) } just Runs - every { anyConstructed().selectTrack(0) } just Runs - every { anyConstructed().sampleTrackIndex } answers { if (index < 3) 0 else -1 } - every { anyConstructed().sampleFlags } answers { flagsOverride ?: if (index == 0) 1 else 0 } - if (Build.VERSION.SDK_INT >= 28) every { anyConstructed().sampleSize } returns reportedSize - every { anyConstructed().sampleTime } answers { index * 1_000_000L } - every { anyConstructed().readSampleData(any(), 0) } answers { + every { extractor.sampleTrackIndex } answers { if (index < 3) 0 else -1 } + every { extractor.sampleFlags } answers { flagsOverride ?: if (index == 0) 1 else 0 } + if (Build.VERSION.SDK_INT >= 28) every { extractor.sampleSize } returns reportedSize + every { extractor.sampleTime } answers { index * 1_000_000L } + every { extractor.readSampleData(any(), 0) } answers { reader?.invoke(firstArg()) ?: if (firstArg().capacity() < bytes.size) -1 else bytes.size.also { firstArg().put(bytes) } } - every { anyConstructed().advance() } answers { ++index < 3 } - every { anyConstructed().release() } answers { released = true } + every { extractor.advance() } answers { ++index < 3 } return try { - MediaSampleInspector(limits).inspect(File("unused"), input, cancel).also { - assertTrue(released) + while (extractor.sampleTrackIndex >= 0) { + bounded.read(extractor) { _, size, flags, data -> + if (flags and MediaExtractor.SAMPLE_FLAG_SYNC != 0) + assertEquals(idr, SampleRandomAccessReader.read(VideoCodec.H264, data, size)) + } + extractor.advance() } - } finally { unmockkConstructor(MediaExtractor::class) } + bounded.result(metadata().facts, SampleScanStatus.Complete) + } catch (stop: SampleScanStop) { bounded.result(metadata().facts, stop.status) } + catch (_: Exception) { bounded.result(metadata().facts, SampleScanStatus.Unreadable) } } - @Test fun api23UsesBoundedReadsAndReleasesExtractor() { + @Test fun api23UsesBoundedPayloadForNalInspection() { val result = scan(SampleScanLimits(maximumSampleBytes = 16)) assertEquals(SampleScanStatus.Complete, result.status) assertEquals(18L, result.bytesRead) assertEquals(3, result.sampleCount) - assertEquals(idr, result.facts.gopStructure) + assertEquals(6, result.largestSampleBytes) } @Test fun limitsAndCancellationDiscardPartialProof() { @@ -207,16 +201,6 @@ class MediaSampleInspectorTests : AbsRobolectricTest() { assertEquals(SampleScanStatus.Unreadable, scan(reader = { throw IllegalArgumentException() }).status) } - @Test fun matroskaHasNoIsoEditListWhileOtherContainersStayUnknown() { - val result = scan(input = metadata().copy(container = MediaFact.Known(ContainerKind.Matroska))) - assertEquals(SampleScanStatus.Complete, result.status) - assertEquals(MediaFact.Known(EditList.None), result.facts.editList) - assertEquals(MediaFact.Unknown, result.facts.durationSeconds) - assertEquals(MediaFact.Unknown, result.facts.audioVideoStartOffsetSeconds) - for (container in listOf(MediaFact.Unknown, MediaFact.Known(ContainerKind.Other))) - assertEquals(MediaFact.Unknown, scan(input = metadata().copy(container = container)).facts.editList) - } - @Test fun encryptedAndPartialFramesCannotSupplySampleFacts() { for (flags in listOf(2, 4)) { val result = scan(flagsOverride = flags) @@ -225,12 +209,6 @@ class MediaSampleInspectorTests : AbsRobolectricTest() { } } - @Test fun cancellationWinsBeforeUnsupportedInputFallback() { - val result = MediaSampleInspector().inspect(File("unused"), metadata().copy(tracks = emptyList())) { true } - assertEquals(SampleScanStatus.Cancelled, result.status) - assertEquals(metadata().facts, result.facts) - } - @Test fun incompatibleMedia3HelperLeavesRandomAccessUnknown() { mockkStatic(NalUnitUtil::class) try { @@ -246,15 +224,15 @@ class MediaSampleInspectorTests : AbsRobolectricTest() { @Test fun effectiveEditCountsPrerollBytesAndRetainsTheIdrProof() { val samples = listOf(sample(-1_000_000, true, 1000), sample(0, bytes = 100), sample(1_000_000, bytes = 100)) val time = editedTimeline(listOf(-1.0, 0.0, 1.0), 2.0, listOf(1, 2)) - val facts = VideoSampleFactsReader.read(samples, MediaFact.Unknown, timeline = time) + val facts = VideoSampleFactsReader.read(samples.map { it.second }, time) assertEquals(known(3200L), facts.averageBitrate) assertEquals(known(3200L), facts.maximumGopBitrate) assertEquals(known(1200L), facts.maximumGopByteSize) assertEquals(known(2.0), facts.maximumKeyframeIntervalSeconds) assertEquals(idr, facts.gopStructure) assertEquals(known(TimestampFacts(0.0, 1.0, 2, true)), facts.timestamps) - val unproven = samples.toMutableList().apply { this[0] = this[0].copy(randomAccess = MediaFact.Unknown) } - val rejected = VideoSampleFactsReader.read(unproven, MediaFact.Unknown, timeline = time) + val unproven = samples.toMutableList().apply { this[0] = this[0].first to this[0].second.copy(randomAccess = MediaFact.Unknown) } + val rejected = VideoSampleFactsReader.read(unproven.map { it.second }, time) assertEquals(MediaFact.Unknown, rejected.gopStructure) assertEquals(MediaFact.Unknown, rejected.maximumGopBitrate) } @@ -262,7 +240,7 @@ class MediaSampleInspectorTests : AbsRobolectricTest() { @Test fun editedVfrUsesFullSpanForBitrateAndEffectiveSpanForKeyframeInterval() { val samples = (0..4).map { sample((it - 1) * 1_000_000L, it == 0 || it == 2) } val time = editedTimeline(listOf(-1.0, 0.0, 1.0, 2.0, 3.0), 4.5, listOf(1, 2, 3, 4), fullEnd = 5.0) - val facts = VideoSampleFactsReader.read(samples, MediaFact.Unknown, timeline = time) + val facts = VideoSampleFactsReader.read(samples.map { it.second }, time) assertEquals(known(3.5), facts.maximumKeyframeIntervalSeconds) assertEquals(known(667L), facts.averageBitrate) assertEquals(known(800L), facts.maximumGopBitrate) @@ -272,7 +250,7 @@ class MediaSampleInspectorTests : AbsRobolectricTest() { val samples = listOf(sample(0, true), sample(1_000_000), sample(2_000_000, true, 10000, known(GopStructure.Open)), sample(4_000_000)) val time = editedTimeline(listOf(0.0,1.0,2.0,4.0), 2.0, listOf(0,1), fullEnd = 5.0) - val facts = VideoSampleFactsReader.read(samples, MediaFact.Unknown, timeline = time) + val facts = VideoSampleFactsReader.read(samples.map { it.second }, time) assertEquals(known(Cadence.Constant), facts.cadence) assertEquals(idr, facts.gopStructure) assertEquals(known(16480L), facts.averageBitrate) @@ -285,7 +263,7 @@ class MediaSampleInspectorTests : AbsRobolectricTest() { val times = samples.indices.map { it / 30.0 - start } val presented = times.indices.filter { times[it] >= 0 && times[it] < end - start } val timeline = editedTimeline(times, end - start, presented, fullEnd = 4.0 - start) - val facts = VideoSampleFactsReader.read(samples, MediaFact.Unknown, compliantFacts(), timeline) + val facts = VideoSampleFactsReader.read(samples.map { it.second }, timeline, compliantFacts()) assertEquals(known(4_000_080L), facts.averageBitrate) assertEquals(known(4_000_080L), facts.maximumGopBitrate) assertTrue(facts.maximumKeyframeIntervalSeconds.valueOrNull!! <= 2.0) @@ -297,7 +275,7 @@ class MediaSampleInspectorTests : AbsRobolectricTest() { val samples = listOf(sample(-1_000_000, true, 2_000_000), sample(0), sample(100_000), sample(200_000, bytes = 2_000_000)) val time = editedTimeline(listOf(-1.0, 0.0, 0.1, 0.2), 0.2, listOf(1,2), fullEnd = 0.3) - val facts = VideoSampleFactsReader.read(samples, MediaFact.Unknown, compliantFacts(), time) + val facts = VideoSampleFactsReader.read(samples.map { it.second }, time, compliantFacts()) val evaluation = StandardInputPolicyEvaluator().evaluate(facts, selection(), MediaRole.GeneratedOutput) assertTrue(evaluation.nonCompliantRequirements.contains(PolicyRequirement.AverageBitrate)) assertTrue(evaluation.nonCompliantRequirements.contains(PolicyRequirement.MaximumGopBitrate)) diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputPlannerTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputPlannerTests.kt index a4ca9414..55e97c73 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputPlannerTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputPlannerTests.kt @@ -39,6 +39,12 @@ class StandardInputPlannerTests { } } + @Test fun resolutionViolationWithoutScannedCadenceFallsBackBeforeConversion() { + val facts = compliantFacts(dimensions = Dimensions(2560, 1440)).copy(frameRate = MediaFact.Unknown) + assertEquals(StandardInputAction.Fallback(FallbackReason.InsufficientEvidence), + planner.plan(facts, options(), fullCapabilities().copy(sourceTimelineIsProven = false)).action) + } + @Test fun knownViolationPreservesH264AndHevcFamilies() { for (codec in listOf(VideoCodec.H264, VideoCodec.Hevc)) { val conversion = conversion(compliantFacts(codec).copy(frameRate = known(121.0))) diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspectorTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspectorTests.kt index 17b1ea4c..63a04ee4 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspectorTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspectorTests.kt @@ -96,7 +96,7 @@ class StandardInputTimelineInspectorTests : AbsRobolectricTest() { legacyBufferFailure: Boolean = false, apiLevel: Int = 23, expectedSize: Long = 6, readSize: Int = 6, videoEdit: Long? = null, audioEdit: Long? = null, platformMode: String = "shifted", mixedConvention: Boolean = false, byteLimit: Long = 1024L * 1024 * 1024, - sourcePath: Boolean = false, videoDeltas: List = listOf(1000,1000,1000), editDuration: Long = 2000, + videoDeltas: List = listOf(1000,1000,1000), editDuration: Long = 2000, input: MediaMetadataInspection = metadata(audio).copy(facts = metadata(audio).facts.copy(videoCodec = known(VideoCodec.H264))) ): MediaSampleInspection { val file = file(track(deltas = videoDeltas, edits = videoEdit?.let { edit(editDuration, it) } ?: byteArrayOf()) + @@ -135,9 +135,7 @@ class StandardInputTimelineInspectorTests : AbsRobolectricTest() { every { anyConstructed().release() } answers { releases++ } return try { val limits = SampleScanLimits(maximumSamples = sampleLimit, maximumReadBytes = byteLimit) - val inspector = if (sourcePath) MediaSampleInspector(limits, apiLevel)::inspect - else StandardInputTimelineInspector(limits, apiLevel)::inspect - inspector(file, input) { cancelled }.also { + StandardInputTimelineInspector(limits, apiLevel).inspect(file, input) { cancelled }.also { if (it.status == SampleScanStatus.Complete) { assertEquals(1, sources); assertEquals(1, releases) assertEquals(if (audio) setOf(0,1) else setOf(0), selected) @@ -167,10 +165,20 @@ class StandardInputTimelineInspectorTests : AbsRobolectricTest() { assertEquals(StandardInputTimelineFacts(), result.timeline) } } - @Test fun cancellationWinsBeforeUnsupportedMetadata() { - val result = StandardInputTimelineInspector(apiLevel = 23).inspect(File("unused"), - metadata(false).copy(container = known(ContainerKind.Matroska))) { true } - assertEquals(SampleScanStatus.Cancelled, result.status) + @Test fun nonIsoScansReturnMetadataWithoutOpeningAnExtractorAndCancellationWins() { + mockkConstructor(MediaExtractor::class) + try { + for (container in listOf(known(ContainerKind.Matroska), known(ContainerKind.Other), MediaFact.Unknown)) { + val input = metadata(false).copy(container = container) + val result = StandardInputTimelineInspector().inspect(File("unused"), input) + assertEquals(SampleScanStatus.Unsupported, result.status) + assertEquals(input.facts, result.facts) + assertEquals(StandardInputTimelineFacts(), result.timeline) + assertEquals(0, result.sampleCount) + assertEquals(SampleScanStatus.Cancelled, StandardInputTimelineInspector().inspect(File("unused"), input) { true }.status) + } + verify(exactly = 0) { anyConstructed().setDataSource(any()) } + } finally { unmockkConstructor(MediaExtractor::class) } } @Test @Config(sdk = [28]) fun rawShiftedAndClampedEditTimesUseTheSameEffectiveTimeline() { @@ -217,9 +225,9 @@ class StandardInputTimelineInspectorTests : AbsRobolectricTest() { assertEquals(1000L, time.timescale) assertEquals(4.5, time.endSeconds, 0.0) assertEquals(5.0, time.fullEndSeconds, 0.0) - val samples = List(5) { CompressedVideoSample((it - 1) * 1000000L, 100, it == 0 || it == 2, + val samples = List(5) { CompressedVideoSample(100, it == 0 || it == 2, known(GopStructure.ClosedWithIdr)) } - assertEquals(known(3.5), VideoSampleFactsReader.read(samples, MediaFact.Unknown, timeline = time).maximumKeyframeIntervalSeconds) + assertEquals(known(3.5), VideoSampleFactsReader.read(samples, time).maximumKeyframeIntervalSeconds) } @Test fun editedIsoSourceCadencePlansAndValidatesOnlyThePresentedWindow() { @@ -227,7 +235,7 @@ class StandardInputTimelineInspectorTests : AbsRobolectricTest() { isoTracks = known(listOf(IsoTrackMetadata("vide", known(1), listOf("avc1"), false, known(0), true, known(EditList.Simple)))), facts = compliantFacts(dimensions = Dimensions(2560,1440)).copy(audioTracks = known(emptyList()))) - val source = inspect(sourcePath = true, videoDeltas = listOf(25,200,200,200), videoEdit = 25, + val source = inspect(videoDeltas = listOf(25,200,200,200), videoEdit = 25, editDuration = 600, platformMode = "raw", input = input) assertEquals(SampleScanStatus.Complete, source.status) assertEquals(known(5.0), source.facts.frameRate) @@ -252,16 +260,16 @@ class StandardInputTimelineInspectorTests : AbsRobolectricTest() { assertTrue("Correctly trimmed output rejected: $result", result is StandardInputOutputValidation.Accepted) } - @Test fun isoSourceSharesTheCombinedBudgetAndNeverFallsBackToRawPartialFacts() { - val complete = inspect(audio = true, sourcePath = true, sampleLimit = 6, byteLimit = 36) + @Test fun incompleteIsoScansNeverPublishPartialFacts() { + val complete = inspect(audio = true, sampleLimit = 6, byteLimit = 36) assertEquals(SampleScanStatus.Complete, complete.status) assertEquals(6, complete.sampleCount) assertEquals(36L, complete.bytesRead) for ((result, status) in listOf( - inspect(audio = true, sourcePath = true, sampleLimit = 5) to SampleScanStatus.LimitExceeded, - inspect(sourcePath = true, cancelAfterRead = true) to SampleScanStatus.Cancelled, - inspect(sourcePath = true, missingTail = true) to SampleScanStatus.Unreadable, - inspect(sourcePath = true, videoEdit = 1000, mixedConvention = true) to SampleScanStatus.Unsupported)) { + inspect(audio = true, sampleLimit = 5) to SampleScanStatus.LimitExceeded, + inspect(cancelAfterRead = true) to SampleScanStatus.Cancelled, + inspect(missingTail = true) to SampleScanStatus.Unreadable, + inspect(videoEdit = 1000, mixedConvention = true) to SampleScanStatus.Unsupported)) { assertEquals(status, result.status) assertEquals(StandardInputTimelineFacts(), result.timeline) assertTrue(result.elapsedNanos > 0) From c43fb88e5bdd4eb3d4a0f0869d21d3e71e5d9fa5 Mon Sep 17 00:00:00 2001 From: Eugene Morozov Date: Tue, 6 Oct 2026 16:45:59 -0700 Subject: [PATCH 20/49] Ignore dropped audio tracks in expected output duration --- .../StandardInputTimelineInspector.kt | 4 +- .../StandardInputTimelineInspectorTests.kt | 53 +++++++++++++++---- 2 files changed, 46 insertions(+), 11 deletions(-) diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspector.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspector.kt index b10871ee..7ba8c893 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspector.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspector.kt @@ -13,6 +13,7 @@ internal sealed interface AudioVideoStartOffset { } internal data class StandardInputTimelineFacts( + /** Effective endpoint of video and first audio, the tracks retained by conversion. */ val durationSeconds: MediaFact = MediaFact.Unknown, val audioVideoStartOffset: MediaFact = MediaFact.Unknown, /** Effective video PTS in presentation order, excluding encoder priming outside the edit. */ @@ -89,7 +90,8 @@ internal class StandardInputTimelineInspector( .copy(editList = metadata.isoTracks.valueOrNull ?.getOrNull(video.containerIndex.valueOrNull ?: -1)?.editList ?: MediaFact.Unknown) reader.checkBudget() - result(SampleScanStatus.Complete, StandardInputTimelineFacts(MediaFact.Known(tracks.values.maxOf { it.endSeconds }), + result(SampleScanStatus.Complete, StandardInputTimelineFacts( + MediaFact.Known(maxOf(videoTime.endSeconds, audioTime?.endSeconds ?: videoTime.endSeconds)), MediaFact.Known(offset), MediaFact.Known(videoTime.effectivePresentationSeconds), MediaFact.Known(videoTime.endSeconds - videoTime.startSeconds), audioTime?.let { MediaFact.Known(it.endSeconds - it.startSeconds) } ?: MediaFact.Unknown, diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspectorTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspectorTests.kt index 63a04ee4..2639169e 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspectorTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspectorTests.kt @@ -84,38 +84,47 @@ class StandardInputTimelineInspectorTests : AbsRobolectricTest() { } finally { file.delete() } } - private fun metadata(audio: Boolean): MediaMetadataInspection { + private fun metadata(audio: Boolean, secondaryAudio: Boolean = false): MediaMetadataInspection { fun track(index: Int, kind: TrackKind) = TrackMetadata(index, known(index), known(index + 1), kind, MediaFact.Unknown, known(999.0), MediaFact.Unknown, MediaFact.Unknown) return MediaMetadataInspection(known(ContainerKind.IsoBaseMedia), - listOf(track(0, TrackKind.Video)) + if (audio) listOf(track(1, TrackKind.Audio)) else emptyList(), - MediaFact.Unknown, MediaFacts(videoTrackCount = known(1), audioTracks = known(if (audio) listOf(AudioTrack()) else emptyList())), 0) + listOf(track(0, TrackKind.Video)) + (if (audio) listOf(track(1, TrackKind.Audio)) else emptyList()) + + (if (secondaryAudio) listOf(track(2, TrackKind.Audio)) else emptyList()), + MediaFact.Unknown, MediaFacts(videoTrackCount = known(1), + audioTracks = known(List(if (secondaryAudio) 2 else if (audio) 1 else 0) { AudioTrack() })), 0) } - private fun inspect(audio: Boolean = false, audioOffset: Int = 0, sampleLimit: Int = 250000, + private fun inspect(audio: Boolean = false, audioOffset: Int = 0, secondaryAudioDelta: Int? = null, sampleLimit: Int = 250000, missingTail: Boolean = false, partial: Boolean = false, cancelAfterRead: Boolean = false, legacyBufferFailure: Boolean = false, apiLevel: Int = 23, expectedSize: Long = 6, readSize: Int = 6, videoEdit: Long? = null, audioEdit: Long? = null, platformMode: String = "shifted", mixedConvention: Boolean = false, byteLimit: Long = 1024L * 1024 * 1024, videoDeltas: List = listOf(1000,1000,1000), editDuration: Long = 2000, - input: MediaMetadataInspection = metadata(audio).copy(facts = metadata(audio).facts.copy(videoCodec = known(VideoCodec.H264))) + input: MediaMetadataInspection = metadata(audio, secondaryAudioDelta != null).copy( + facts = metadata(audio, secondaryAudioDelta != null).facts.copy(videoCodec = known(VideoCodec.H264))) ): MediaSampleInspection { val file = file(track(deltas = videoDeltas, edits = videoEdit?.let { edit(editDuration, it) } ?: byteArrayOf()) + - if (audio) track(kind = "soun", offsets = List(3) { audioOffset }, - edits = audioEdit?.let { edit(2000, it) } ?: byteArrayOf()) else byteArrayOf()) + (if (audio) track(kind = "soun", offsets = List(3) { audioOffset }, + edits = audioEdit?.let { edit(2000, it) } ?: byteArrayOf()) else byteArrayOf()) + + (secondaryAudioDelta?.let { delta -> track(kind = "soun", deltas = List(3) { delta }) } ?: byteArrayOf())) mockkConstructor(MediaExtractor::class) var position = 0; var cancelled = false; var releases = 0; var sources = 0; var reads = 0 val selected = mutableSetOf() val rawTimes = videoDeltas.runningFold(0L) { time, delta -> time + delta * 1000L }.dropLast(1) val count = rawTimes.size - if (missingTail) 1 else 0 // Interleaved tracks preserve decode order within each track. - val sequence = (0 until count).flatMap { index -> (0..if (audio) 1 else 0).map { it to index } } + val lastTrack = if (secondaryAudioDelta != null) 2 else if (audio) 1 else 0 + val sequence = (0 until count).flatMap { index -> (0..lastTrack).map { it to index } } every { anyConstructed().setDataSource(any()) } answers { sources++ } every { anyConstructed().selectTrack(any()) } answers { selected.add(firstArg()) } every { anyConstructed().sampleTrackIndex } answers { sequence.getOrNull(position)?.first ?: -1 } every { anyConstructed().sampleTime } answers { val (track, index) = sequence[position] val start = if (track == 0) videoEdit ?: 0L else audioEdit ?: 0L - val raw = rawTimes[index] + if (track == 1) audioOffset * 1000L else 0L + val raw = when (track) { + 0 -> rawTimes[index] + 1 -> index * 1_000_000L + audioOffset * 1000L + else -> index * secondaryAudioDelta!! * 1000L + } if (platformMode == "raw" || (mixedConvention && index == 1)) raw else if (platformMode == "clamped") maxOf(0L, raw - start * 1000L) else raw - start * 1000L } @@ -138,7 +147,7 @@ class StandardInputTimelineInspectorTests : AbsRobolectricTest() { StandardInputTimelineInspector(limits, apiLevel).inspect(file, input) { cancelled }.also { if (it.status == SampleScanStatus.Complete) { assertEquals(1, sources); assertEquals(1, releases) - assertEquals(if (audio) setOf(0,1) else setOf(0), selected) + assertEquals((0..lastTrack).toSet(), selected) assertEquals(sequence.size, reads) } else assertEquals(input.facts, it.facts) } @@ -156,6 +165,30 @@ class StandardInputTimelineInspectorTests : AbsRobolectricTest() { assertEquals(known(3.1), result.timeline.durationSeconds) assertEquals(known(AudioVideoStartOffset.Seconds(0.1)), result.timeline.audioVideoStartOffset) } + @Test fun droppedSecondaryAudioDoesNotExtendTheValidatedTimeline() { + val firstAudio = compliantFacts().audioTracks.valueOrNull!!.single() + val input = metadata(true, true).copy(facts = compliantFacts().copy(audioTracks = known(listOf(firstAudio, AudioTrack()))), + isoTracks = known(listOf(IsoTrackMetadata("vide", known(1), listOf("avc1"), false, known(0), true, known(EditList.None))))) + val output = inspect(audio = true, audioOffset = 100, videoDeltas = List(3) { 200 }, + input = input.copy(tracks = input.tracks.take(2), facts = input.facts.copy(audioTracks = known(listOf(firstAudio))))) + for (secondaryDelta in listOf(500, 2000)) { + val source = inspect(audio = true, audioOffset = 100, secondaryAudioDelta = secondaryDelta, + videoDeltas = List(3) { 200 }, input = input) + assertEquals(SampleScanStatus.Complete, source.status) + assertEquals(9, source.sampleCount) + val conversion = (StandardInputPlanner().plan(source.facts.copy(averageBitrate = known(9_000_000L)), + options(), fullCapabilities()).action as StandardInputAction.Convert).conversion + val facts = output.facts.copy(rotationDegrees = known(0), encodedDimensions = known(conversion.outputDimensions)) + val validator = StandardInputOutputValidator() + val result = validator.validateFacts(facts, source.facts, source.timeline, output.timeline, conversion) + assertTrue("Secondary audio duration changed acceptance: $result", result is StandardInputOutputValidation.Accepted) + assertEquals(known(3.1), source.timeline.durationSeconds) + val shortened = validator.validateFacts(facts, source.facts, source.timeline, + output.timeline.copy(firstAudioDurationSeconds = known(2.0)), conversion) as StandardInputOutputValidation.Rejected + assertTrue(OutputExpectation.Duration in (shortened.reason as OutputRejection.DoesNotMatchPlan).expectations) + } + } + @Test fun truncatedPartialCancelledOrFailedLegacyReadsNeverPublishTimeline() { for ((result, status) in listOf(inspect(missingTail = true) to SampleScanStatus.Unreadable, inspect(partial = true) to SampleScanStatus.Unsupported, From 66eb9a5671b98629ac80b59d9293b1448d1774a3 Mon Sep 17 00:00:00 2001 From: Eugene Morozov Date: Fri, 16 Oct 2026 11:27:56 -0700 Subject: [PATCH 21/49] Match timeline evidence using platform track identity --- .../StandardInputTimelineInspector.kt | 24 ++++++++++++------- .../StandardInputOutputValidatorTests.kt | 6 ++--- .../StandardInputTimelineInspectorTests.kt | 20 +++++++++------- 3 files changed, 29 insertions(+), 21 deletions(-) diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspector.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspector.kt index 7ba8c893..b955946c 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspector.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspector.kt @@ -44,9 +44,10 @@ internal class StandardInputTimelineInspector( return result(SampleScanStatus.Unsupported) val tracks = IsoTimelineReader.read(file, limits.maximumSamples) { reader.checkBudget() } val media = metadata.tracks.filter { it.kind == TrackKind.Video || it.kind == TrackKind.Audio } - if (media.size != tracks.size || media.any { it.containerIndex.valueOrNull !in tracks.keys }) + if (media.size != tracks.size || media.any { it.sourceTrackId.valueOrNull !in tracks.keys } || + media.map { it.sourceTrackId }.distinct().size != media.size) return result(SampleScanStatus.Unsupported) - val timingByExtractor = media.associate { it.extractorIndex to tracks.getValue(it.containerIndex.valueOrNull!!) } + val timingByExtractor = media.associate { it.extractorIndex to tracks.getValue(it.sourceTrackId.valueOrNull!!) } for (track in media) { if (track.kind == TrackKind.Audio && !timingByExtractor.getValue(track.extractorIndex).hasEdit && track.audio?.let { (it.encoderDelaySamples.valueOrNull ?: 0) > 0 || (it.encoderPaddingSamples.valueOrNull ?: 0) > 0 @@ -82,13 +83,12 @@ internal class StandardInputTimelineInspector( throw SampleScanStop(SampleScanStatus.Unreadable) } finally { extractor.release() } reader.checkBudget() - val firstAudio = media.filter { it.kind == TrackKind.Audio }.minByOrNull { it.containerIndex.valueOrNull!! } + val firstAudio = media.filter { it.kind == TrackKind.Audio }.minByOrNull { it.extractorIndex } val audioTime = firstAudio?.let { timingByExtractor.getValue(it.extractorIndex) } val offset = audioTime?.let { AudioVideoStartOffset.Seconds(it.startSeconds - videoTime.startSeconds) } ?: AudioVideoStartOffset.NotApplicable val facts = VideoSampleFactsReader.read(samples, videoTime, metadata.facts) - .copy(editList = metadata.isoTracks.valueOrNull - ?.getOrNull(video.containerIndex.valueOrNull ?: -1)?.editList ?: MediaFact.Unknown) + .copy(editList = IsoContainerMetadataReader.videoEdit(file, video.sourceTrackId.valueOrNull)) reader.checkBudget() result(SampleScanStatus.Complete, StandardInputTimelineFacts( MediaFact.Known(maxOf(videoTime.endSeconds, audioTime?.endSeconds ?: videoTime.endSeconds)), @@ -171,7 +171,7 @@ internal object IsoTimelineReader { } val movieScale = timescale(children.single { it.type == "mvhd" }) var totalSamples = 0L - children.filter { it.type == "trak" }.mapIndexedNotNull { index, track -> + children.filter { it.type == "trak" }.mapNotNull { track -> checkBudget() val trackChildren = children(track) val mdia = children(trackChildren.single { it.type == "mdia" }) @@ -179,7 +179,13 @@ internal object IsoTimelineReader { require(handler.end - handler.payload >= 12) input.seek(handler.payload + 8) val kind = ByteArray(4).also { input.readFully(it) }.toString(Charsets.US_ASCII) - if (kind !in listOf("vide", "soun")) return@mapIndexedNotNull null + if (kind !in listOf("vide", "soun")) return@mapNotNull null + val trackHeader = trackChildren.single { it.type == "tkhd" } + input.seek(trackHeader.payload) + val idOffset = when (input.readUnsignedByte()) { 0 -> 12; 1 -> 20; else -> error("Unsupported track header") } + require(trackHeader.end - trackHeader.payload >= idOffset + 4) + input.seek(trackHeader.payload + idOffset) + val trackId = input.readInt().also { require(it > 0) } val scale = timescale(mdia.single { it.type == "mdhd" }) val tables = children(children(mdia.single { it.type == "minf" }).single { it.type == "stbl" }) fun runs(box: IsoMetadataBox, composition: Boolean): List> { @@ -266,7 +272,7 @@ internal object IsoTimelineReader { val effectiveIndices = starts.indices.filter { ends[it].toDouble() / scale > mediaStart && platformTimes[it] < end } val effectiveTimes = effectiveIndices.map { maxOf(start, platformTimes[it]) }.sorted() require(effectiveTimes.isNotEmpty()) - index to IsoTrackTimeline(platformTimes, start, end, rawEnd - mediaStart, effectiveTimes, editDuration != null, scale, mediaStart, effectiveIndices) - }.toMap() + trackId to IsoTrackTimeline(platformTimes, start, end, rawEnd - mediaStart, effectiveTimes, editDuration != null, scale, mediaStart, effectiveIndices) + }.also { require(it.map { track -> track.first }.distinct().size == it.size) }.toMap() } } diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputOutputValidatorTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputOutputValidatorTests.kt index 542bfe7c..c7144e06 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputOutputValidatorTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputOutputValidatorTests.kt @@ -119,7 +119,7 @@ class StandardInputOutputValidatorTests { } @Test fun partialScansNeverReachComparisonAndLateCancellationWins() { val file = File.createTempFile("validation", ".mp4") - val metadata = MediaMetadataInspection(known(ContainerKind.IsoBaseMedia), emptyList(), MediaFact.Unknown, output, 0) + val metadata = MediaMetadataInspection(known(ContainerKind.IsoBaseMedia), emptyList(), output, 0) try { file.writeBytes(byteArrayOf(1)) for (status in SampleScanStatus.entries.filter { it != SampleScanStatus.Complete }) { @@ -136,7 +136,7 @@ class StandardInputOutputValidatorTests { } @Test fun changedFileCannotSelectGeneratedBytesAfterCompleteInspection() { val file = File.createTempFile("validation", ".mp4") - val metadata = MediaMetadataInspection(known(ContainerKind.IsoBaseMedia), emptyList(), MediaFact.Unknown, output, 0) + val metadata = MediaMetadataInspection(known(ContainerKind.IsoBaseMedia), emptyList(), output, 0) fun validator(timelineRead: () -> MediaSampleInspection) = StandardInputOutputValidator( inspectMetadata = { MetadataInspectionResult.Success(metadata) }, inspectTimeline = { _, _, _ -> timelineRead() }) @@ -180,7 +180,7 @@ class StandardInputOutputValidatorTests { val file = File.createTempFile("validation", ".mp4") try { file.writeBytes(byteArrayOf(1)) - val metadata = MediaMetadataInspection(known(ContainerKind.IsoBaseMedia), emptyList(), MediaFact.Unknown, output, 0) + val metadata = MediaMetadataInspection(known(ContainerKind.IsoBaseMedia), emptyList(), output, 0) val validator = StandardInputOutputValidator(inspectMetadata = { MetadataInspectionResult.Success(metadata) }, inspectTimeline = { _, _, _ -> MediaSampleInspection(MediaFacts(), SampleScanStatus.Complete, timeline = timeline()) }) assertTrue(reason(validator.validateGeneratedOutput(file, source, timeline(), plan)) diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspectorTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspectorTests.kt index 2639169e..439fa552 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspectorTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspectorTests.kt @@ -22,18 +22,18 @@ class StandardInputTimelineInspectorTests : AbsRobolectricTest() { (if (version == 0) int(duration.toInt()) + int(start.toInt()) else long(duration) + long(start)) + int(rate))) private fun track(deltas: List = listOf(1000, 1000, 1000), offsets: List? = null, edits: ByteArray = byteArrayOf(), kind: String = "vide", scale: Int = 1000, - timingVersion: Int = 0, headerVersion: Int = 0): ByteArray { + timingVersion: Int = 0, headerVersion: Int = 0, id: Int = if (kind == "vide") 1 else 2): ByteArray { val stts = box("stts", ByteArray(4) + int(deltas.size) + deltas.flatMap { (int(1) + int(it)).toList() }.toByteArray()) val ctts = offsets?.let { box("ctts", byteArrayOf(timingVersion.toByte(), 0, 0, 0) + int(it.size) + it.flatMap { offset -> (int(1) + int(offset)).toList() }.toByteArray()) } ?: byteArrayOf() - return box("trak", edits + box("mdia", box("hdlr", ByteArray(8) + kind.toByteArray()) + + return box("trak", box("tkhd", ByteArray(12) + int(id)) + edits + box("mdia", box("hdlr", ByteArray(8) + kind.toByteArray()) + box("mdhd", header(scale, headerVersion)) + box("minf", box("stbl", stts + ctts)))) } private fun file(tracks: ByteArray = track(), scale: Int = 1000, extra: ByteArray = byteArrayOf()): File = File.createTempFile("timeline", ".mp4").apply { writeBytes(box("moov", box("mvhd", header(scale)) + tracks) + extra) } private fun read(tracks: ByteArray = track(), scale: Int = 1000, maximum: Int = 250000): IsoTrackTimeline { val file = file(tracks, scale) - return try { IsoTimelineReader.read(file, maximum).getValue(0) } finally { file.delete() } + return try { IsoTimelineReader.read(file, maximum).getValue(1) } finally { file.delete() } } private fun invalid(tracks: ByteArray, scale: Int = 1000, maximum: Int = 250000) { try { read(tracks, scale, maximum); fail("Unproven timeline was published") } catch (_: IllegalArgumentException) { } catch (stop: SampleScanStop) { assertEquals(SampleScanStatus.LimitExceeded, stop.status) } @@ -90,7 +90,7 @@ class StandardInputTimelineInspectorTests : AbsRobolectricTest() { return MediaMetadataInspection(known(ContainerKind.IsoBaseMedia), listOf(track(0, TrackKind.Video)) + (if (audio) listOf(track(1, TrackKind.Audio)) else emptyList()) + (if (secondaryAudio) listOf(track(2, TrackKind.Audio)) else emptyList()), - MediaFact.Unknown, MediaFacts(videoTrackCount = known(1), + MediaFacts(videoTrackCount = known(1), audioTracks = known(List(if (secondaryAudio) 2 else if (audio) 1 else 0) { AudioTrack() })), 0) } private fun inspect(audio: Boolean = false, audioOffset: Int = 0, secondaryAudioDelta: Int? = null, sampleLimit: Int = 250000, @@ -105,7 +105,7 @@ class StandardInputTimelineInspectorTests : AbsRobolectricTest() { val file = file(track(deltas = videoDeltas, edits = videoEdit?.let { edit(editDuration, it) } ?: byteArrayOf()) + (if (audio) track(kind = "soun", offsets = List(3) { audioOffset }, edits = audioEdit?.let { edit(2000, it) } ?: byteArrayOf()) else byteArrayOf()) + - (secondaryAudioDelta?.let { delta -> track(kind = "soun", deltas = List(3) { delta }) } ?: byteArrayOf())) + (secondaryAudioDelta?.let { delta -> track(kind = "soun", deltas = List(3) { delta }, id = 3) } ?: byteArrayOf())) mockkConstructor(MediaExtractor::class) var position = 0; var cancelled = false; var releases = 0; var sources = 0; var reads = 0 val selected = mutableSetOf() @@ -159,6 +159,11 @@ class StandardInputTimelineInspectorTests : AbsRobolectricTest() { assertEquals(known(3.0), result.timeline.durationSeconds) assertEquals(known(AudioVideoStartOffset.NotApplicable), result.timeline.audioVideoStartOffset) } + @Test fun timelineRequiresThePlatformTrackIdentityToMatch() { + val input = metadata(false) + assertEquals(SampleScanStatus.Unsupported, inspect(input = input.copy( + tracks = input.tracks.map { it.copy(sourceTrackId = known(99)) })).status) + } @Test fun offsetComesFromEffectiveAudioAndVideoStarts() { val result = inspect(audio = true, audioOffset = 100) assertEquals(SampleScanStatus.Complete, result.status) @@ -167,8 +172,7 @@ class StandardInputTimelineInspectorTests : AbsRobolectricTest() { } @Test fun droppedSecondaryAudioDoesNotExtendTheValidatedTimeline() { val firstAudio = compliantFacts().audioTracks.valueOrNull!!.single() - val input = metadata(true, true).copy(facts = compliantFacts().copy(audioTracks = known(listOf(firstAudio, AudioTrack()))), - isoTracks = known(listOf(IsoTrackMetadata("vide", known(1), listOf("avc1"), false, known(0), true, known(EditList.None))))) + val input = metadata(true, true).copy(facts = compliantFacts().copy(audioTracks = known(listOf(firstAudio, AudioTrack())))) val output = inspect(audio = true, audioOffset = 100, videoDeltas = List(3) { 200 }, input = input.copy(tracks = input.tracks.take(2), facts = input.facts.copy(audioTracks = known(listOf(firstAudio))))) for (secondaryDelta in listOf(500, 2000)) { @@ -265,8 +269,6 @@ class StandardInputTimelineInspectorTests : AbsRobolectricTest() { @Test fun editedIsoSourceCadencePlansAndValidatesOnlyThePresentedWindow() { val input = metadata(false).copy( - isoTracks = known(listOf(IsoTrackMetadata("vide", known(1), listOf("avc1"), false, - known(0), true, known(EditList.Simple)))), facts = compliantFacts(dimensions = Dimensions(2560,1440)).copy(audioTracks = known(emptyList()))) val source = inspect(videoDeltas = listOf(25,200,200,200), videoEdit = 25, editDuration = 600, platformMode = "raw", input = input) From 012b3096f5d3bdee78e0980cf067c93a39f97140 Mon Sep 17 00:00:00 2001 From: Eugene Morozov Date: Wed, 7 Oct 2026 12:34:09 -0700 Subject: [PATCH 22/49] NAT-547: define SDR targets and bounded codec preflight --- .../standardization/SdrEncodingTargets.kt | 145 ++++++++++++++++++ 1 file changed, 145 insertions(+) create mode 100644 library/src/main/java/com/mux/video/upload/internal/standardization/SdrEncodingTargets.kt diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/SdrEncodingTargets.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/SdrEncodingTargets.kt new file mode 100644 index 00000000..4853a32a --- /dev/null +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/SdrEncodingTargets.kt @@ -0,0 +1,145 @@ +package com.mux.video.upload.internal.standardization + +import android.media.MediaCodecInfo +import android.media.MediaCodecInfo.CodecProfileLevel +import android.media.MediaCodecList +import android.media.MediaFormat +import kotlin.math.abs +import kotlin.math.ceil + +internal enum class SdrConversionFailure { + UnsupportedPlan, InsufficientSourceEvidence, CapabilityUnavailable, DiskSpace, + Export, ForbiddenFallback, CodecChanged, OutputInvalid, SourceChanged, FileOwnership, +} + +/** Encoder targets are deliberately below policy ceilings; output validation still proves compliance. */ +internal data class SdrEncodingTargets( + val videoMime: String, + val profile: VideoProfile, + val platformProfile: Int, + val bitrate: Int, + val keyframeIntervalSeconds: Int, + val audioChannels: Int?, + val audioSampleRate: Int?, + val copyAac: Boolean, + val firstAudioTrackId: String?, +) { + fun estimatedOutputBytes(duration: Double): Long? { + if (!duration.isFinite() || duration <= 0) return null + val bytes = ceil(duration * (bitrate + if (audioChannels != null) 384_000 else 0) / 8 * 1.5) + 8 * 1024 * 1024 + return bytes.takeIf { it.isFinite() && it > 0 && it < Long.MAX_VALUE.toDouble() }?.toLong() + } + + companion object { + fun from(conversion: StandardInputConversion, metadata: MediaMetadataInspection, + source: MediaSampleInspection): SdrEncodingTargets? { + val facts = source.facts + if (source.status != SampleScanStatus.Complete || conversion.toneMapsToSdr || + conversion.sourceDynamicRange != DynamicRange.Sdr || facts.dynamicRange != MediaFact.Known(DynamicRange.Sdr) || + facts.videoCodec != MediaFact.Known(conversion.sourceCodec) || facts.videoTrackCount != MediaFact.Known(1) || + facts.displayDimensions != MediaFact.Known(conversion.sourceDisplayDimensions) || + source.timeline.videoPresentationSeconds.valueOrNull.isNullOrEmpty() || + source.timeline.videoDurationSeconds.valueOrNull == null || + source.timeline.audioVideoStartOffset == MediaFact.Unknown) return null + val expectedCodec = if (conversion.sourceCodec == VideoCodec.Hevc) VideoCodec.Hevc else VideoCodec.H264 + val size = conversion.outputDimensions + val rate = conversion.outputFrameRate + if (conversion.outputCodec != expectedCodec || !size.isValid || size.width % 2 != 0 || size.height % 2 != 0 || + size.width > conversion.sourceDisplayDimensions.width || size.height > conversion.sourceDisplayDimensions.height || + !size.fitsWithin(conversion.selection.generatedOutputDimensions) || + !StandardInputOutputValidator.preservesAspectRatio(conversion.sourceDisplayDimensions, size) || + !rate.isFinite() || rate !in conversion.outputPolicyLimits.frameRateRange || + conversion.outputPixelFormat != PixelFormat(8, ChromaSubsampling.Yuv420)) return null + // This SDR surface path has no proof of 10-bit SDR preservation or frame duplication. + val sourceRate = facts.frameRate.valueOrNull ?: return null + if (conversion.outputCadence == OutputCadence.ConstantFrameRate) { + val ratio = sourceRate / rate + if (facts.cadence != MediaFact.Known(Cadence.Constant) || rate != 30.0 || ratio < 1 || + // Measured rates use microsecond-rounded timestamps; permit only rounding-scale noise. + abs(ratio - kotlin.math.round(ratio)) > maxOf(1.0, ratio) * 1e-6) return null + } else if (sourceRate != rate || facts.cadence == MediaFact.Unknown) return null + val audioFacts = facts.audioTracks.valueOrNull ?: return null + val audio = metadata.tracks.filter { it.kind == TrackKind.Audio }.sortedBy { it.containerIndex.valueOrNull ?: Int.MAX_VALUE } + if (audio.size != audioFacts.size || audio.any { it.containerIndex == MediaFact.Unknown }) return null + if ((conversion.outputAudio == OutputAudio.None) != audio.isEmpty() && + conversion.outputAudio != OutputAudio.AacFromFirstTrackIfPresent) return null + val first = audio.firstOrNull() + val channels = first?.audio?.channelCount?.valueOrNull + val sampleRate = first?.audio?.sampleRate?.valueOrNull + val copy = first?.audio?.format?.valueOrNull is AudioFormat.Aac + val layout = (first?.audio?.format?.valueOrNull as? AudioFormat.Aac)?.layout + if (first != null && (first.audio?.format == MediaFact.Unknown || first.sourceTrackId.valueOrNull == null || channels == null || sampleRate == null || + sampleRate <= 0 || (if (copy) layout !in setOf(AudioChannelLayout.Mono, AudioChannelLayout.Stereo, + AudioChannelLayout.FivePointOne) else channels !in 1..2))) return null + val interval = conversion.outputPolicyLimits.maximumKeyframeIntervals[expectedCodec] ?: return null + if (!interval.isFinite() || interval < 1) return null + return SdrEncodingTargets( + if (expectedCodec == VideoCodec.Hevc) "video/hevc" else "video/avc", + if (expectedCodec == VideoCodec.Hevc) VideoProfile.HevcMain else VideoProfile.H264Baseline, + if (expectedCodec == VideoCodec.Hevc) CodecProfileLevel.HEVCProfileMain else CodecProfileLevel.AVCProfileBaseline, + minOf(6_000_000L.takeIf { size.longSide <= 2048 } ?: 15_000_000L, + conversion.outputPolicyLimits.maximumAverageBitrate * 3 / 4).toInt().takeIf { it > 0 } ?: return null, + maxOf(1, (interval * 0.9).toInt()), channels, sampleRate, copy, first?.sourceTrackId?.valueOrNull?.toString(), + ) + } + } +} + +internal data class SdrEncoderCapability(val name: String, val level: Int) + +/** Bounded advertised-format guard. This does not manufacture planner/device-path proof. */ +internal class SdrCapabilityPreflight { + fun videoEncoder(conversion: StandardInputConversion, metadata: MediaMetadataInspection, + targets: SdrEncodingTargets, isCancelled: () -> Boolean): SdrEncoderCapability? { + val started = System.nanoTime() + fun withinBudget() = !isCancelled() && System.nanoTime() - started < 500_000_000L + val infos = MediaCodecList(MediaCodecList.REGULAR_CODECS).codecInfos + if (infos.size > 128 || !withinBudget()) return null + val video = metadata.tracks.singleOrNull { it.kind == TrackKind.Video } ?: return null + val sourceMime = video.mimeType.valueOrNull ?: return null + val sourceSize = video.video?.encodedDimensions?.valueOrNull ?: return null + fun supportsDecode(mime: String, format: MediaFormat): Boolean = infos.any { info -> + withinBudget() && !info.isEncoder && info.supportedTypes.any { it.equals(mime, true) } && + runCatching { info.getCapabilitiesForType(mime).isFormatSupported(format) }.getOrDefault(false) + } + val sourceFormat = MediaFormat.createVideoFormat(sourceMime, sourceSize.width, sourceSize.height) + val sourceProfile = when (video.video.profile.valueOrNull) { + VideoProfile.H264Baseline -> CodecProfileLevel.AVCProfileBaseline + VideoProfile.H264Main -> CodecProfileLevel.AVCProfileMain + VideoProfile.H264High -> CodecProfileLevel.AVCProfileHigh + VideoProfile.HevcMain -> CodecProfileLevel.HEVCProfileMain + VideoProfile.HevcMain10 -> CodecProfileLevel.HEVCProfileMain10 + else -> null + } + if (sourceProfile != null) sourceFormat.setInteger(MediaFormat.KEY_PROFILE, sourceProfile) + if (!supportsDecode(sourceMime, sourceFormat)) return null + if (targets.audioChannels != null && !targets.copyAac) { + val audioMime = metadata.tracks.first { it.sourceTrackId.valueOrNull?.toString() == targets.firstAudioTrackId }.mimeType.valueOrNull ?: return null + if (!supportsDecode(audioMime, MediaFormat.createAudioFormat(audioMime, targets.audioSampleRate!!, targets.audioChannels))) return null + val aac = MediaFormat.createAudioFormat("audio/mp4a-latm", targets.audioSampleRate, targets.audioChannels) + aac.setInteger(MediaFormat.KEY_BIT_RATE, 160_000) + aac.setInteger(MediaFormat.KEY_AAC_PROFILE, CodecProfileLevel.AACObjectLC) + if (infos.none { withinBudget() && it.isEncoder && it.supportedTypes.contains("audio/mp4a-latm") && + runCatching { it.getCapabilitiesForType("audio/mp4a-latm").isFormatSupported(aac) }.getOrDefault(false) }) return null + } + for (info in infos) { + if (!withinBudget()) return null + if (!info.isEncoder || !info.supportedTypes.contains(targets.videoMime)) continue + val candidate = runCatching { + val caps = info.getCapabilitiesForType(targets.videoMime) + val level = caps.profileLevels.filter { it.profile == targets.platformProfile }.maxOfOrNull { it.level } ?: return@runCatching null + val size = conversion.outputDimensions + val format = MediaFormat.createVideoFormat(targets.videoMime, size.width, size.height) + format.setFloat(MediaFormat.KEY_FRAME_RATE, conversion.outputFrameRate.toFloat()) + format.setInteger(MediaFormat.KEY_BIT_RATE, targets.bitrate) + format.setInteger(MediaFormat.KEY_PROFILE, targets.platformProfile) + format.setInteger(MediaFormat.KEY_LEVEL, level) + format.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface) + format.setInteger(MediaFormat.KEY_BITRATE_MODE, MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_VBR) + if (caps.isFormatSupported(format)) SdrEncoderCapability(info.name, level) else null + }.getOrNull() + if (candidate != null && withinBudget()) return candidate + } + return null + } +} From e58a99adc3546dbb0615706cde9567e681662c4f Mon Sep 17 00:00:00 2001 From: Eugene Morozov Date: Wed, 7 Oct 2026 12:34:09 -0700 Subject: [PATCH 23/49] NAT-547: own cancellable Media3 SDR export and validated files --- .../standardization/Media3SdrExportEngine.kt | 154 +++++++++++++++++ .../standardization/SdrConversionAdapter.kt | 162 ++++++++++++++++++ 2 files changed, 316 insertions(+) create mode 100644 library/src/main/java/com/mux/video/upload/internal/standardization/Media3SdrExportEngine.kt create mode 100644 library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/Media3SdrExportEngine.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/Media3SdrExportEngine.kt new file mode 100644 index 00000000..027ffade --- /dev/null +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/Media3SdrExportEngine.kt @@ -0,0 +1,154 @@ +package com.mux.video.upload.internal.standardization + +import android.content.Context +import android.media.MediaCodecInfo +import android.media.MediaFormat +import android.media.metrics.LogSessionId +import android.net.Uri +import android.os.Build +import android.os.Looper +import android.util.Pair +import androidx.media3.common.C +import androidx.media3.common.Format +import androidx.media3.common.MediaItem +import androidx.media3.common.util.Clock +import androidx.media3.common.util.MediaFormatUtil +import androidx.media3.common.util.UnstableApi +import androidx.media3.effect.Presentation +import androidx.media3.exoplayer.trackselection.DefaultTrackSelector +import androidx.media3.exoplayer.trackselection.ExoTrackSelection +import androidx.media3.transformer.AudioEncoderSettings +import androidx.media3.transformer.Codec +import androidx.media3.transformer.Composition +import androidx.media3.transformer.DefaultCodec +import androidx.media3.transformer.DefaultDecoderFactory +import androidx.media3.transformer.DefaultEncoderFactory +import androidx.media3.transformer.EditedMediaItem +import androidx.media3.transformer.EditedMediaItemSequence +import androidx.media3.transformer.Effects +import androidx.media3.transformer.ExoPlayerAssetLoader +import androidx.media3.transformer.ExportException +import androidx.media3.transformer.ExportResult +import androidx.media3.transformer.TransformationRequest +import androidx.media3.transformer.Transformer +import java.io.File + +@UnstableApi +internal class Media3SdrExportEngine( + private val context: Context, private val looper: Looper, + private val conversion: StandardInputConversion, private val targets: SdrEncodingTargets, + private val capability: SdrEncoderCapability, +) : SdrExportEngine { + private var transformer: Transformer? = null + + override fun start(input: File, output: File, completed: (String?, String?) -> Unit, + failed: (SdrConversionFailure) -> Unit) { + check(Looper.myLooper() == looper) + val decoder = DefaultDecoderFactory.Builder(context).setEnableDecoderFallback(false).build() + val assetLoader = ExoPlayerAssetLoader.Factory(context, decoder, Clock.DEFAULT, null, + { ctx -> FirstAudioTrackSelector(ctx, targets.firstAudioTrackId) }, null, null) + val item = EditedMediaItem.Builder(MediaItem.fromUri(Uri.fromFile(input))) + .setRemoveAudio(targets.audioChannels == null) + .setEffects(Effects(emptyList(), listOf(Presentation.createForWidthAndHeight( + conversion.outputDimensions.width, conversion.outputDimensions.height, Presentation.LAYOUT_SCALE_TO_FIT)))) + if (conversion.outputCadence == OutputCadence.ConstantFrameRate) item.setFrameRate(30) + val items = listOf(item.build()) + val sequence = if (targets.audioChannels == null) EditedMediaItemSequence.withVideoFrom(items) + else EditedMediaItemSequence.withAudioAndVideoFrom(items) + val composition = Composition.Builder(sequence) + .setHdrMode(Composition.HDR_MODE_KEEP_HDR).build() + val exporter = Transformer.Builder(context) + .setLooper(looper) + .setPortraitEncodingEnabled(true) + .setVideoMimeType(targets.videoMime) + .setAudioMimeType("audio/mp4a-latm") + .setAssetLoaderFactory(assetLoader) + .setEncoderFactory(StrictSdrEncoderFactory(context, conversion, targets, capability)) + .addListener(object : Transformer.Listener { + override fun onCompleted(composition: Composition, exportResult: ExportResult) { + if (targets.audioChannels != null && (exportResult.channelCount != targets.audioChannels || + exportResult.sampleRate != targets.audioSampleRate)) failed(SdrConversionFailure.OutputInvalid) + else completed(exportResult.videoMimeType, exportResult.audioMimeType) + } + override fun onError(composition: Composition, exportResult: ExportResult, exportException: ExportException) { + failed(SdrConversionFailure.Export) + } + override fun onFallbackApplied(composition: Composition, originalTransformationRequest: TransformationRequest, + fallbackTransformationRequest: TransformationRequest) { + failed(SdrConversionFailure.ForbiddenFallback) + } + }).build() + transformer = exporter + exporter.start(composition, output.absolutePath) + } + + override fun cancel() { + check(Looper.myLooper() == looper) + transformer?.cancel() + transformer = null + } +} + +/** MP4 Format.id is the container track ID. Never choose a preferred/default secondary track. */ +@UnstableApi +internal class FirstAudioTrackSelector(context: Context, private val trackId: String?) : DefaultTrackSelector(context) { + init { + setParameters(buildUponParameters().setForceHighestSupportedBitrate(true) + .setExceedAudioConstraintsIfNecessary(true).setExceedRendererCapabilitiesIfNecessary(true)) + } + override fun selectAudioTrack(mappedTrackInfo: MappedTrackInfo, rendererFormatSupports: Array>, + rendererMixedMimeTypeAdaptationSupports: IntArray, params: Parameters): Pair? { + if (trackId == null) return null + for (renderer in 0 until mappedTrackInfo.rendererCount) { + if (mappedTrackInfo.getRendererType(renderer) != C.TRACK_TYPE_AUDIO) continue + val groups = mappedTrackInfo.getTrackGroups(renderer) + for (groupIndex in 0 until groups.length) { + val group = groups[groupIndex] + for (track in 0 until group.length) { + if (group.getFormat(track).id == trackId) return Pair(ExoTrackSelection.Definition(group, track), renderer) + } + } + } + error("First audio track identity unavailable") + } +} + +/** + * Media3's default video factory overrides H.264 profile requests. Supply its DefaultCodec with + * exact platform settings instead; Transformer still owns the entire decode/effect/encode pipeline. + * No alternative encoder, size, profile, MIME, or HDR mode is attempted on configuration failure. + */ +@UnstableApi +internal class StrictSdrEncoderFactory( + private val context: Context, private val conversion: StandardInputConversion, + private val targets: SdrEncodingTargets, private val capability: SdrEncoderCapability, +) : Codec.EncoderFactory { + private val audioDelegate = DefaultEncoderFactory.Builder(context) + .setEnableFallback(false).setEnableFormatFallback(false) + .setRequestedAudioEncoderSettings(AudioEncoderSettings.Builder().setBitrate(160_000) + .setProfile(MediaCodecInfo.CodecProfileLevel.AACObjectLC).build()).build() + + override fun videoNeedsEncoding() = true + override fun audioNeedsEncoding() = targets.audioChannels != null && !targets.copyAac + + override fun createForAudioEncoding(format: Format, logSessionId: LogSessionId?): Codec { + check(format.sampleMimeType == "audio/mp4a-latm" && format.channelCount == targets.audioChannels && + format.sampleRate == targets.audioSampleRate) + return audioDelegate.createForAudioEncoding(format, logSessionId) + } + + override fun createForVideoEncoding(format: Format, logSessionId: LogSessionId?): Codec { + check(format.sampleMimeType == targets.videoMime && format.width == conversion.outputDimensions.width && + format.height == conversion.outputDimensions.height && format.rotationDegrees == 0 && + format.colorInfo?.colorTransfer == C.COLOR_TRANSFER_SDR) + val exact = format.buildUpon().setFrameRate(conversion.outputFrameRate.toFloat()).setAverageBitrate(targets.bitrate).build() + val media = MediaFormatUtil.createMediaFormatFromFormat(exact) + media.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface) + media.setInteger(MediaFormat.KEY_BITRATE_MODE, MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_VBR) + media.setInteger(MediaFormat.KEY_PROFILE, targets.platformProfile) + media.setInteger(MediaFormat.KEY_LEVEL, capability.level) + media.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, targets.keyframeIntervalSeconds) + if (Build.VERSION.SDK_INT >= 29) media.setInteger(MediaFormat.KEY_MAX_B_FRAMES, 0) + return DefaultCodec(context, exact, media, capability.name, false, null) + } +} diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt new file mode 100644 index 00000000..81448954 --- /dev/null +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt @@ -0,0 +1,162 @@ +package com.mux.video.upload.internal.standardization + +import android.content.Context +import android.os.Handler +import android.os.HandlerThread +import android.os.Looper +import java.io.File +import java.util.concurrent.atomic.AtomicBoolean + +internal sealed interface SdrConversionResult { + /** Caller owns retention until upload/resume reaches its terminal point. */ + data class Completed(val output: SdrGeneratedFile, val facts: MediaFacts) : SdrConversionResult + data class Failed(val reason: SdrConversionFailure) : SdrConversionResult + data object Cancelled : SdrConversionResult +} + +/** The only deletion authority returned to orchestration; never accepts an arbitrary path. */ +internal class SdrGeneratedFile private constructor(val file: File) { + fun delete(): Boolean = !file.exists() || file.delete() + + companion object { + fun allocate(cacheDir: File, requiredBytes: Long): SdrGeneratedFile? { + val area = File(cacheDir.canonicalFile, "mux-upload/standard-input") + if ((!area.isDirectory && !area.mkdirs()) || area.canonicalFile != area.absoluteFile || area.usableSpace < requiredBytes) return null + return SdrGeneratedFile(File.createTempFile("sdr-", ".mp4", area)) + } + } +} + +/** Small Media3 boundary so terminal races and cleanup can be tested without a platform encoder. */ +internal interface SdrExportEngine { + fun start(input: File, output: File, completed: (String?, String?) -> Unit, + failed: (SdrConversionFailure) -> Unit) + fun cancel() +} + +/** Each attempt owns one thread, one Transformer, and one explicitly tracked output file. */ +internal class SdrConversionAdapter( + context: Context, + private val preflight: (StandardInputConversion, MediaMetadataInspection, SdrEncodingTargets, () -> Boolean) -> SdrEncoderCapability? = + SdrCapabilityPreflight()::videoEncoder, + private val validate: (File, MediaFacts, StandardInputTimelineFacts, StandardInputConversion, () -> Boolean) -> StandardInputOutputValidation = + StandardInputOutputValidator()::validateGeneratedOutput, + private val createEngine: (Context, Looper, StandardInputConversion, SdrEncodingTargets, SdrEncoderCapability) -> SdrExportEngine = + ::Media3SdrExportEngine, +) { + private val appContext = context.applicationContext + + fun start(input: File, metadata: MediaMetadataInspection, source: MediaSampleInspection, + conversion: StandardInputConversion, onResult: (SdrConversionResult) -> Unit): Attempt = + Attempt(input, metadata, source, conversion, onResult).also { it.begin() } + + inner class Attempt internal constructor( + private val input: File, private val metadata: MediaMetadataInspection, + private val source: MediaSampleInspection, private val conversion: StandardInputConversion, + private val onResult: (SdrConversionResult) -> Unit, + ) { + private val thread = HandlerThread("MuxSdrConversion").apply { start() } + internal val looper: Looper = thread.looper + private val handler = Handler(looper) + private val cancelled = AtomicBoolean(false) + private val terminalLock = Any() + private var terminal = false + private var engine: SdrExportEngine? = null + private var output: SdrGeneratedFile? = null + /** Exact deletion authority retained if the OS refuses cleanup, including cancellation. */ + @Volatile var pendingCleanup: SdrGeneratedFile? = null + private set + private val sourceLength = input.length() + private val sourceModified = input.lastModified() + + fun cancel() { + synchronized(terminalLock) { + if (terminal) return + cancelled.set(true) + } + handler.post { finish(SdrConversionResult.Cancelled) } + } + + internal fun begin() { handler.post { + guarded { + if (cancelled.get()) return@guarded finish(SdrConversionResult.Cancelled) + val targets = SdrEncodingTargets.from(conversion, metadata, source) + ?: return@guarded fail(SdrConversionFailure.UnsupportedPlan) + val capability = preflight(conversion, metadata, targets, cancelled::get) + ?: return@guarded fail(SdrConversionFailure.CapabilityUnavailable) + if (cancelled.get()) return@guarded finish(SdrConversionResult.Cancelled) + if (!sourceUnchanged()) return@guarded fail(SdrConversionFailure.SourceChanged) + val required = source.timeline.durationSeconds.valueOrNull?.let(targets::estimatedOutputBytes) + ?: return@guarded fail(SdrConversionFailure.InsufficientSourceEvidence) + val owned = SdrGeneratedFile.allocate(appContext.cacheDir, required) + ?: return@guarded fail(SdrConversionFailure.DiskSpace) + output = owned + if (cancelled.get()) return@guarded finish(SdrConversionResult.Cancelled) + val exporter = createEngine(appContext, looper, conversion, targets, capability) + engine = exporter + exporter.start(input, owned.file, { videoMime, audioMime -> + handler.post { guarded { + if (isTerminal()) return@guarded + if (videoMime != targets.videoMime || audioMime != if (targets.audioChannels != null) "audio/mp4a-latm" else null) + return@guarded fail(SdrConversionFailure.CodecChanged) + if (!sourceUnchanged()) return@guarded fail(SdrConversionFailure.SourceChanged) + when (val validation = validate(owned.file, source.facts, source.timeline, conversion, cancelled::get)) { + is StandardInputOutputValidation.Accepted -> { + if (!sourceUnchanged()) return@guarded fail(SdrConversionFailure.SourceChanged) + // The shared validator checks policy, pixels, cadence, and effective timeline. + // Add this adapter's concrete encoder-profile and non-AAC channel contract. + val audio = validation.facts.audioTracks.valueOrNull?.singleOrNull()?.format?.valueOrNull as? AudioFormat.Aac + val expectedLayout = when (targets.audioChannels) { + 1 -> AudioChannelLayout.Mono + 2 -> AudioChannelLayout.Stereo + 6 -> AudioChannelLayout.FivePointOne + else -> null + } + if (validation.facts.videoProfile != MediaFact.Known(targets.profile) || + (targets.audioChannels != null && audio?.layout != expectedLayout)) fail(SdrConversionFailure.OutputInvalid) + else finish(SdrConversionResult.Completed(owned, validation.facts)) + } + is StandardInputOutputValidation.Rejected -> fail(SdrConversionFailure.OutputInvalid) + StandardInputOutputValidation.Cancelled -> finish(SdrConversionResult.Cancelled) + } + } } + }, { reason -> handler.post { guarded { if (!isTerminal()) fail(reason) } } }) + } + } } + + private fun sourceUnchanged() = input.isFile && input.canRead() && sourceLength > 0 && + input.length() == sourceLength && input.lastModified() == sourceModified + private fun isTerminal() = synchronized(terminalLock) { terminal } + private fun fail(reason: SdrConversionFailure) = finish(SdrConversionResult.Failed(reason)) + private fun guarded(block: () -> Unit) { + if (isTerminal()) return + try { block() } + catch (_: Exception) { fail(SdrConversionFailure.Export) } + catch (_: LinkageError) { fail(SdrConversionFailure.Export) } + } + + private fun finish(proposed: SdrConversionResult) { + val result = synchronized(terminalLock) { + if (terminal) return + terminal = true + if (cancelled.get()) SdrConversionResult.Cancelled else proposed + } + try { + runCatching { engine?.cancel() } + if (result !is SdrConversionResult.Completed && output?.delete() == false) { + pendingCleanup = output + // Keep ownership at its exact path if the OS refuses deletion; never sweep a directory. + if (result !is SdrConversionResult.Cancelled) { + onResult(SdrConversionResult.Failed(SdrConversionFailure.FileOwnership)) + return + } + } + onResult(result) + } finally { + engine = null + handler.removeCallbacksAndMessages(null) + thread.quitSafely() + } + } + } +} From b2c1324dae896aefdd32d95973980c985f2d2097 Mon Sep 17 00:00:00 2001 From: Eugene Morozov Date: Thu, 8 Oct 2026 10:48:40 -0700 Subject: [PATCH 24/49] Integrate upload preparation and pause/cancel lifecycle --- .../java/com/mux/video/upload/MuxUploadSdk.kt | 2 +- .../com/mux/video/upload/api/MuxUpload.kt | 81 ++-- .../mux/video/upload/api/MuxUploadManager.kt | 58 +-- .../mux/video/upload/internal/ChunkWorker.kt | 70 ++-- .../video/upload/internal/UploadAttempt.kt | 106 +++++ .../mux/video/upload/internal/UploadInfo.kt | 23 +- .../video/upload/internal/UploadJobFactory.kt | 310 +++++--------- .../upload/internal/UploadPersistence.kt | 17 +- .../standardization/SdrConversionAdapter.kt | 31 ++ .../standardization/UploadPreparation.kt | 80 ++++ .../upload/internal/ChunkCancellationTests.kt | 64 +++ .../internal/StandardInputLifecycleTests.kt | 20 +- .../internal/StandardInputPersistenceTests.kt | 2 +- .../upload/internal/UploadPersistenceTests.kt | 4 +- .../UploadPreparationLifecycleTests.kt | 388 ++++++++++++++++++ .../standardization/UploadPreparationTests.kt | 132 ++++++ 16 files changed, 1085 insertions(+), 303 deletions(-) create mode 100644 library/src/main/java/com/mux/video/upload/internal/UploadAttempt.kt create mode 100644 library/src/main/java/com/mux/video/upload/internal/standardization/UploadPreparation.kt create mode 100644 library/src/test/java/com/mux/video/upload/internal/ChunkCancellationTests.kt create mode 100644 library/src/test/java/com/mux/video/upload/internal/UploadPreparationLifecycleTests.kt create mode 100644 library/src/test/java/com/mux/video/upload/internal/standardization/UploadPreparationTests.kt diff --git a/library/src/main/java/com/mux/video/upload/MuxUploadSdk.kt b/library/src/main/java/com/mux/video/upload/MuxUploadSdk.kt index c98d433e..47a1cacd 100644 --- a/library/src/main/java/com/mux/video/upload/MuxUploadSdk.kt +++ b/library/src/main/java/com/mux/video/upload/MuxUploadSdk.kt @@ -71,7 +71,7 @@ object MuxUploadSdk { initializeUploadPersistence(realAppContext) UploadMetrics.initialize(realAppContext) if (resumeStoppedUploads) { - MuxUploadManager.resumeAllCachedJobs() + MuxUploadManager.resumeCachedJobs(includePaused = false) } } diff --git a/library/src/main/java/com/mux/video/upload/api/MuxUpload.kt b/library/src/main/java/com/mux/video/upload/api/MuxUpload.kt index a9b5d75b..34821d8e 100644 --- a/library/src/main/java/com/mux/video/upload/api/MuxUpload.kt +++ b/library/src/main/java/com/mux/video/upload/api/MuxUpload.kt @@ -1,15 +1,15 @@ package com.mux.video.upload.api import android.net.Uri -import android.util.Log import androidx.annotation.MainThread import com.mux.video.upload.MuxUploadSdk import com.mux.video.upload.api.MuxUpload.Builder import com.mux.video.upload.internal.MaximumResolution import com.mux.video.upload.internal.UploadInfo import com.mux.video.upload.internal.update +import com.mux.video.upload.internal.writeUploadState +import com.mux.video.upload.internal.forgetUploadState import kotlinx.coroutines.* -import kotlinx.coroutines.flow.distinctUntilChangedBy import java.io.File /** @@ -86,6 +86,8 @@ class MuxUpload private constructor( private var progressListener: UploadEventListener? = null private var statusListener: UploadEventListener? = null private var observerJob: Job? = null + private var cancelled = false + private var deliveredResultStatus: UploadStatus? = null private val currentStatus: UploadStatus get() = uploadInfo.statusFlow?.value ?: lastKnownStatus ?: UploadStatus.Ready private var lastKnownStatus: UploadStatus? = null @@ -125,12 +127,27 @@ class MuxUpload private constructor( forceRestart: Boolean = false, coroutineScope: CoroutineScope = CoroutineScope(Dispatchers.Default) ) { + if (cancelled) return + if (!forceRestart && uploadInfo.uploadJob?.isActive == true && uploadInfo.attempt?.isStopped() != true) { + observeUpload(uploadInfo) + return + } + _successful = false + _error = null // Get an updated UploadInfo with a job & event channels uploadInfo = if (autoManage) { // We may or may not get a fresh worker, depends on if the upload is already going /*uploadInfo =*/ MuxUploadManager.startJob(uploadInfo, forceRestart) } else { // If we're not managing the worker, the job is purely internal to this object + if (forceRestart) { + val blocked = uploadInfo.generatedResumeBlocked || uploadInfo.attempt?.preparation?.generatedRequestStarted == true + uploadInfo.attempt?.cancel { if (!blocked) forgetUploadState(uploadInfo) } + uploadInfo.uploadJob?.cancel() + if (!blocked) forgetUploadState(uploadInfo) + uploadInfo = uploadInfo.update(attempt = null, statusFlow = null, + restoredFromOriginal = false, generatedResumeBlocked = blocked) + } /*uploadInfo =*/ MuxUploadSdk.uploadJobFactory().createUploadJob(uploadInfo, coroutineScope) } @@ -174,13 +191,11 @@ class MuxUpload private constructor( /*uploadInfo =*/ MuxUploadManager.pauseJob(uploadInfo) } else { observerJob?.cancel("user requested pause") + uploadInfo.attempt?.pause { writeUploadState(uploadInfo, it) } uploadInfo.uploadJob?.cancel() - /*uploadInfo =*/ uploadInfo.update( - uploadJob = null, - statusFlow = null, - ) + uploadInfo } - lastKnownProgress?.let { state -> progressListener?.onEvent(state) } + observeUpload(uploadInfo) } /** @@ -189,9 +204,12 @@ class MuxUpload private constructor( */ @Suppress("MemberVisibilityCanBePrivate") fun cancel() { + if (cancelled) return + cancelled = true if (autoManage) { MuxUploadManager.cancelJob(uploadInfo) } else { + uploadInfo.attempt?.cancel { forgetUploadState(uploadInfo) } uploadInfo.uploadJob?.cancel("user requested cancel") } observerJob?.cancel("user requested cancel") @@ -212,7 +230,7 @@ class MuxUpload private constructor( } progressListener = listener - lastKnownProgress?.let { listener?.onEvent(it) } + if (!cancelled && uploadInfo.attempt?.isCancelled() != true) lastKnownProgress?.let { listener?.onEvent(it) } observeUpload(uploadInfo) } @@ -230,12 +248,9 @@ class MuxUpload private constructor( observeUpload(uploadInfo) } + if (resultListener !== listener) deliveredResultStatus = null resultListener = listener - lastKnownProgress?.let { - if (it.bytesUploaded >= it.totalBytes) { - listener?.onEvent(Result.success(it)) - } - } + notifyResult(currentStatus) } /** @@ -253,7 +268,7 @@ class MuxUpload private constructor( } statusListener = listener - listener?.onEvent(currentStatus) + if (!cancelled && uploadInfo.attempt?.isCancelled() != true) listener?.onEvent(currentStatus) } /** @@ -273,10 +288,14 @@ class MuxUpload private constructor( return upload.statusFlow?.let { flow -> callbackScope.launch { flow.collect { status -> + if (cancelled || uploadInfo.attempt !== upload.attempt || upload.attempt?.isCancelled() == true) return@collect // Update the status of our upload lastKnownStatus = status statusListener?.onEvent(status) + // A listener can synchronously pause, cancel, or start a replacement attempt. + if (cancelled || uploadInfo.attempt !== upload.attempt || upload.attempt?.isCancelled() == true || + (upload.attempt?.isStopped() == true && status !is UploadStatus.UploadPaused)) return@collect // Notify the specific listeners when (status) { is UploadStatus.Uploading -> { progressListener?.onEvent(status.uploadProgress) } @@ -284,13 +303,15 @@ class MuxUpload private constructor( is UploadStatus.UploadSuccess -> { _successful = true progressListener?.onEvent(status.uploadProgress) - resultListener?.onEvent(Result.success(status.uploadProgress)) + if (!cancelled && upload.attempt?.isCancelled() != true && uploadInfo.attempt === upload.attempt) + notifyResult(status) } is UploadStatus.UploadFailed -> { progressListener?.onEvent(status.uploadProgress) // Make sure we're most up-to-date - if (status.exception !is CancellationException) { + if (status.exception !is CancellationException && upload.attempt?.isCancelled() != true && + uploadInfo.attempt === upload.attempt) { _error = status.exception - resultListener?.onEvent(Result.failure(status.exception)) + notifyResult(status) } } else -> { } // no relevant info @@ -300,6 +321,21 @@ class MuxUpload private constructor( } } + private fun notifyResult(status: UploadStatus) { + if (cancelled || uploadInfo.attempt?.isCancelled() == true || resultListener == null || deliveredResultStatus === status) return + when (status) { + is UploadStatus.UploadSuccess -> { + deliveredResultStatus = status + resultListener?.onEvent(Result.success(status.uploadProgress)) + } + is UploadStatus.UploadFailed -> if (status.exception !is CancellationException) { + deliveredResultStatus = status + resultListener?.onEvent(Result.failure(status.exception)) + } + else -> {} + } + } + private fun observeUpload(uploadInfo: UploadInfo) { observerJob?.cancel("switching observers") observerJob = newObserveProgressJob(uploadInfo) @@ -372,9 +408,8 @@ class MuxUpload private constructor( /** * If requested, the Upload SDK will try to standardize the input file in order to optimize it - * for use with Mux Video. The retained resolution is passed to the legacy transcoder. - * This requests on-device output limits; the legacy transcoder does not guarantee successful - * conversion or output within those dimensions. Configure the matching Direct Upload + * for use with Mux Video. These are requested on-device output limits; if safe local + * conversion is unavailable, the original file is uploaded. Configure the matching Direct Upload * `new_asset_settings.max_resolution_tier` separately. */ @Suppress("unused") @@ -399,8 +434,8 @@ class MuxUpload private constructor( } /** - * Store the requested HDR behavior. Defaults to [HdrHandling.Preserve]. The legacy transcoder - * does not yet apply this option; HDR handling is part of the new Standard Input pipeline. + * Store the requested HDR behavior. Defaults to [HdrHandling.Preserve]. + * Unsupported local HDR conversion falls back to uploading the original file. */ fun hdrHandling(handling: HdrHandling): Builder { uploadInfo = uploadInfo.update(inputStandardization = uploadInfo.inputStandardization.copy( @@ -449,7 +484,7 @@ class MuxUpload private constructor( /** * Creates a new [MuxUpload] with the given configuration. */ - fun build() = MuxUpload(uploadInfo) + fun build() = MuxUpload(uploadInfo, manageTask) } internal companion object { diff --git a/library/src/main/java/com/mux/video/upload/api/MuxUploadManager.kt b/library/src/main/java/com/mux/video/upload/api/MuxUploadManager.kt index fdd39166..d4e203f7 100644 --- a/library/src/main/java/com/mux/video/upload/api/MuxUploadManager.kt +++ b/library/src/main/java/com/mux/video/upload/api/MuxUploadManager.kt @@ -58,8 +58,12 @@ object MuxUploadManager { * The jobs will all be resumed where they left off. Any uploads resumed this way will be returned */ @MainThread - fun resumeAllCachedJobs(): List { - return readAllCachedUploads() + fun resumeAllCachedJobs(): List = resumeCachedJobs(includePaused = true) + + @JvmSynthetic + @MainThread + internal fun resumeCachedJobs(includePaused: Boolean): List { + return readAllCachedUploads(includePaused) .filter { uploadInfo -> val exists = uploadInfo.inputFile.exists() if (!exists) { @@ -67,7 +71,7 @@ object MuxUploadManager { } exists } - .onEach { uploadInfo -> startJob(uploadInfo, restart = false) } + .map { uploadInfo -> startJob(uploadInfo, restart = false) } .map { MuxUpload.create(it) } } @@ -110,13 +114,10 @@ object MuxUploadManager { assertMainThread() // Paused jobs stay in the manager and remain persisted uploadsByFilename[upload.inputFile.absolutePath]?.let { + it.attempt?.pause { state -> writeUploadState(it, state) } cancelJobInner(it) - val pausedUpload = upload.update( - uploadJob = null, - statusFlow = null, - ) - uploadsByFilename[pausedUpload.inputFile.absolutePath] = pausedUpload - return pausedUpload + notifyListListeners() + return it } notifyListListeners() return upload @@ -126,12 +127,12 @@ object MuxUploadManager { @MainThread internal fun cancelJob(upload: UploadInfo) { assertMainThread() - uploadsByFilename[upload.inputFile.absolutePath]?.let { - observerJobsByFilename.remove(upload.inputFile.absolutePath)?.cancel() - cancelJobInner(it) - uploadsByFilename -= it.inputFile.absolutePath - forgetUploadState(upload) - } + val current = uploadsByFilename[upload.inputFile.absolutePath] ?: upload + observerJobsByFilename.remove(current.inputFile.absolutePath)?.cancel() + current.attempt?.cancel { forgetUploadState(current) } + cancelJobInner(current) + uploadsByFilename -= current.inputFile.absolutePath + forgetUploadState(current) notifyListListeners() } @@ -139,6 +140,7 @@ object MuxUploadManager { @MainThread internal fun jobFinished(upload: UploadInfo, forgetJob: Boolean = true) { assertMainThread() + if (uploadsByFilename[upload.inputFile.absolutePath]?.attempt !== upload.attempt) return observerJobsByFilename.remove(upload.inputFile.absolutePath)?.cancel() uploadsByFilename -= upload.inputFile.absolutePath if (forgetJob) { @@ -160,19 +162,21 @@ object MuxUploadManager { private fun insertOrUpdateUpload(upload: UploadInfo, restart: Boolean): UploadInfo { val filename = upload.inputFile.absolutePath - var newUpload = uploadsByFilename[filename] - // Use the old job if possible (unless requested otherwise) - if (newUpload?.uploadJob == null) { - if (restart) { - forgetUploadState(upload) - } - newUpload = startUploadJob(upload) - } else { - if (restart) { - cancelJob(upload) - newUpload = startUploadJob(upload) - } + val previous = uploadsByFilename[filename] + val newDestination = previous != null && previous.remoteUri != upload.remoteUri + if (!restart && !newDestination && previous?.uploadJob?.isActive == true && previous.attempt?.isStopped() != true) + return previous + var source = previous ?: upload + if (restart || newDestination) { + val generatedMayExistRemotely = !newDestination && (source.generatedResumeBlocked || + source.attempt?.preparation?.generatedRequestStarted == true) + previous?.attempt?.cancel { if (!generatedMayExistRemotely) forgetUploadState(source) } + previous?.uploadJob?.cancel() + if (!generatedMayExistRemotely) forgetUploadState(upload) + source = upload.update(attempt = null, uploadJob = source.uploadJob, + statusFlow = null, restoredFromOriginal = false, generatedResumeBlocked = generatedMayExistRemotely) } + val newUpload = startUploadJob(source) uploadsByFilename += upload.inputFile.absolutePath to newUpload observerJobsByFilename[upload.inputFile.absolutePath]?.cancel() observerJobsByFilename += upload.inputFile.absolutePath to newObserveProgressJob(newUpload) diff --git a/library/src/main/java/com/mux/video/upload/internal/ChunkWorker.kt b/library/src/main/java/com/mux/video/upload/internal/ChunkWorker.kt index 5e624726..39712129 100644 --- a/library/src/main/java/com/mux/video/upload/internal/ChunkWorker.kt +++ b/library/src/main/java/com/mux/video/upload/internal/ChunkWorker.kt @@ -1,7 +1,5 @@ package com.mux.video.upload.internal -import android.net.Uri -import android.util.Log import com.mux.video.upload.MuxUploadSdk import com.mux.video.upload.api.MuxUpload import com.mux.video.upload.internal.network.asCountingRequestBody @@ -9,6 +7,9 @@ import kotlinx.coroutines.* import kotlinx.coroutines.flow.MutableSharedFlow import okhttp3.MediaType.Companion.toMediaTypeOrNull import okhttp3.Request +import okhttp3.Call +import okhttp3.Callback +import kotlin.coroutines.resumeWithException import okhttp3.Response import java.io.IOException @@ -48,24 +49,16 @@ internal class ChunkWorker private constructor( val moreRetries = { triesSoFar: Int -> triesSoFar < uploadInfo.retriesPerChunk } suspend fun tryUpload(triesSoFar: Int): Result { try { + currentCoroutineContext().ensureActive() val (finalState, httpResponse) = doUpload() - if (ACCEPTABLE_STATUS_CODES.contains(httpResponse.code)) { - // End Case: Chunk success! - return Result.success(finalState) - } else if (RETRYABLE_STATUS_CODES.contains(httpResponse.code)) { - return if (moreRetries(triesSoFar)) { - // Still have more retries so try again - tryUpload(triesSoFar + 1) - } else { - Result.failure( - IOException("Upload request failed: ${httpResponse.code}/${httpResponse.message}") - ) - } - } else { - return Result.failure( - IOException("Upload request failed: ${httpResponse.code}/${httpResponse.message}") - ) + httpResponse.use { + if (ACCEPTABLE_STATUS_CODES.contains(it.code)) return Result.success(finalState) + val failure = IOException("Upload request failed: ${it.code}/${it.message}") + if (it.code !in RETRYABLE_STATUS_CODES || !moreRetries(triesSoFar)) return Result.failure(failure) } + return tryUpload(triesSoFar + 1) + } catch (e: CancellationException) { + throw e } catch (e: Exception) { return if (moreRetries(triesSoFar)) { // Still have more retries so try again @@ -76,7 +69,8 @@ internal class ChunkWorker private constructor( } } - return tryUpload(0).getOrThrow().also { writeUploadState(uploadInfo, it) } + currentCoroutineContext().ensureActive() + return tryUpload(0).getOrThrow() } @Throws @@ -125,20 +119,32 @@ internal class ChunkWorker private constructor( .build() logger.v("MuxUpload", "Uploading with request $request") - val httpResponse = withContext(Dispatchers.IO) { - httpClient.newCall(request).execute() + val call = httpClient.newCall(request) + val httpResponse = suspendCancellableCoroutine { continuation -> + continuation.invokeOnCancellation { call.cancel() } + call.enqueue(object : Callback { + override fun onFailure(call: Call, e: IOException) { continuation.resumeWithException(e) } + override fun onResponse(call: Call, response: Response) { + continuation.resume(response) { _, value, _ -> value.close() } + } + }) + } + try { + logger.v("MuxUpload", "Chunk Response: $httpResponse") + val finalState = MuxUpload.Progress( + bytesUploaded = chunkSize, + totalBytes = chunkSize, + startTime = startTime, + updatedTime = System.currentTimeMillis() + ) + // Cancel progress updates and make sure no one is stuck listening for more + updateCallersJob?.cancel() + progressFlow.emit(finalState) + Pair(finalState, httpResponse) + } catch (e: Exception) { + httpResponse.close() + throw e } - logger.v("MuxUpload", "Chunk Response: $httpResponse") - val finalState = MuxUpload.Progress( - bytesUploaded = chunkSize, - totalBytes = chunkSize, - startTime = startTime, - updatedTime = System.currentTimeMillis() - ) - // Cancel progress updates and make sure no one is stuck listening for more - updateCallersJob?.cancel() - progressFlow.emit(finalState) - Pair(finalState, httpResponse) } // supervisorScope } // suspend fun doUpload diff --git a/library/src/main/java/com/mux/video/upload/internal/UploadAttempt.kt b/library/src/main/java/com/mux/video/upload/internal/UploadAttempt.kt new file mode 100644 index 00000000..c665a633 --- /dev/null +++ b/library/src/main/java/com/mux/video/upload/internal/UploadAttempt.kt @@ -0,0 +1,106 @@ +package com.mux.video.upload.internal + +import com.mux.video.upload.api.MuxUpload +import com.mux.video.upload.api.UploadStatus +import com.mux.video.upload.internal.standardization.SdrGeneratedFile +import kotlinx.coroutines.flow.MutableStateFlow + +/** Shared across in-process pause/resume, never reconstructed as a generated payload from disk. */ +internal class UploadPreparationState { + @Volatile var originalSelected = false + @Volatile var verified: SdrGeneratedFile? = null + @Volatile var generatedRequestStarted = false + private val pendingCleanup = mutableSetOf() + + @Synchronized fun trackCleanup(output: SdrGeneratedFile) { pendingCleanup += output } + + @Synchronized fun discardLateOutput(output: SdrGeneratedFile) { + if (verified !== output && !output.delete()) pendingCleanup += output + } + + @Synchronized fun deleteOwnedFile() { + if (verified?.delete() != false) verified = null + pendingCleanup.removeAll { it.delete() } + } +} + +/** Serializes terminal transitions with transport acknowledgements and persistence. */ +internal class UploadAttempt( + val preparation: UploadPreparationState, + initialProgress: MuxUpload.Progress, +) { + val status = MutableStateFlow(UploadStatus.Started) + private var stopped = false + private var cancelled = false + private var terminal = false + private var superseded = false + private var confirmed = initialProgress + + @Synchronized fun isStopped() = stopped + @Synchronized fun isCancelled() = cancelled + @Synchronized fun confirmedProgress() = confirmed + @Synchronized fun supersede() { superseded = true } + + @Synchronized fun retainGenerated(output: SdrGeneratedFile) { + if (superseded) preparation.discardLateOutput(output) + else if (cancelled) { + preparation.trackCleanup(output) + preparation.deleteOwnedFile() + } else preparation.verified = output + } + + @Synchronized fun trackCleanup(output: SdrGeneratedFile) { + if (superseded) { preparation.discardLateOutput(output); return } + preparation.trackCleanup(output) + if (cancelled || terminal) preparation.deleteOwnedFile() + } + + @Synchronized fun selectPayload(totalBytes: Long) { + confirmed = confirmed.copy(totalBytes = totalBytes) + } + + @Synchronized fun beginTransport(generated: Boolean, persist: () -> Unit) { + if (stopped) throw kotlinx.coroutines.CancellationException("Upload stopped") + if (generated) { + preparation.generatedRequestStarted = true + persist() + } + } + + @Synchronized fun publish(value: UploadStatus) { + if (!stopped && !terminal && !superseded) status.value = value + } + + @Synchronized fun acknowledge(progress: MuxUpload.Progress, persist: () -> Unit) { + if (stopped || terminal) return + confirmed = progress + persist() + status.value = UploadStatus.Uploading(progress) + } + + @Synchronized fun pause(persist: (MuxUpload.Progress) -> Unit) { + if (stopped || terminal) return + stopped = true + status.value = UploadStatus.UploadPaused(confirmed) + persist(confirmed) + } + + @Synchronized fun cancel(forget: () -> Unit) { + if (cancelled) return + stopped = true + cancelled = true + forget() + preparation.deleteOwnedFile() + } + + @Synchronized fun finish(value: UploadStatus): Boolean { + if (stopped || terminal) return false + terminal = true + status.value = value + return true + } +} + +internal class GeneratedResumeBlockedException : IllegalStateException( + "Generated payload identity or server offset is unverified. Create a new Direct Upload." +) diff --git a/library/src/main/java/com/mux/video/upload/internal/UploadInfo.kt b/library/src/main/java/com/mux/video/upload/internal/UploadInfo.kt index 23b939bb..592e181e 100644 --- a/library/src/main/java/com/mux/video/upload/internal/UploadInfo.kt +++ b/library/src/main/java/com/mux/video/upload/internal/UploadInfo.kt @@ -34,16 +34,14 @@ enum class MaximumResolution(val width: Int, val height: Int) { /** * Requested maximum generated dimensions of 3840x2160 (2160p/4K), without upscaling. - * The legacy transcoder uses a broader source-size limit for this preset and does not - * guarantee output within these dimensions. Exact output limits remain pending in the new - * Standard Input pipeline. + * If safe local conversion is unavailable, the original file is uploaded. + * Configure the matching Direct Upload asset tier separately. */ Preset3840x2160(3840, 2160), // 2160p /** * Requested maximum generated dimensions of 2560x1440 (1440p), without upscaling. - * The legacy transcoder uses these size limits with its existing media rules. Reliable resizing - * and the updated tier policy remain pending in the new Standard Input pipeline. + * If safe local conversion is unavailable, the original file is uploaded. * Configure the matching Direct Upload asset tier separately. */ Preset2560x1440(2560, 1440) // 1440p @@ -90,11 +88,14 @@ internal data class UploadInfo( @JvmSynthetic internal val optOut: Boolean, @JvmSynthetic internal val uploadJob: Deferred>?, @JvmSynthetic internal val statusFlow: StateFlow?, + @JvmSynthetic internal val attempt: UploadAttempt? = null, + @JvmSynthetic internal val restoredFromOriginal: Boolean = false, + @JvmSynthetic internal val generatedResumeBlocked: Boolean = false, ) { - fun isRunning(): Boolean = - statusFlow?.value?.let { + fun isRunning(): Boolean = attempt?.isStopped() != true && + (statusFlow?.value?.let { it is UploadStatus.Uploading || it is UploadStatus.Started || it is UploadStatus.Preparing - } ?: false + } ?: false) fun isStandardizationRequested(): Boolean = inputStandardization.standardizationRequested } @@ -113,6 +114,9 @@ internal fun UploadInfo.update( optOut: Boolean = this.optOut, uploadJob: Deferred>? = this.uploadJob, statusFlow: StateFlow? = this.statusFlow, + attempt: UploadAttempt? = this.attempt, + restoredFromOriginal: Boolean = this.restoredFromOriginal, + generatedResumeBlocked: Boolean = this.generatedResumeBlocked, ) = UploadInfo( inputStandardization, remoteUri, @@ -123,4 +127,7 @@ internal fun UploadInfo.update( optOut, uploadJob, statusFlow, + attempt, + restoredFromOriginal, + generatedResumeBlocked, ) diff --git a/library/src/main/java/com/mux/video/upload/internal/UploadJobFactory.kt b/library/src/main/java/com/mux/video/upload/internal/UploadJobFactory.kt index 6b2c761a..9e996aa9 100644 --- a/library/src/main/java/com/mux/video/upload/internal/UploadJobFactory.kt +++ b/library/src/main/java/com/mux/video/upload/internal/UploadJobFactory.kt @@ -1,234 +1,152 @@ package com.mux.video.upload.internal -import android.os.Build -import android.util.Log -import com.mux.video.upload.BuildConfig import com.mux.video.upload.MuxUploadSdk import com.mux.video.upload.api.MuxUpload import com.mux.video.upload.api.MuxUploadManager import com.mux.video.upload.api.UploadStatus +import com.mux.video.upload.internal.standardization.PreparedUpload +import com.mux.video.upload.internal.standardization.UploadPreparation import kotlinx.coroutines.* import kotlinx.coroutines.channels.BufferOverflow import kotlinx.coroutines.flow.MutableSharedFlow -import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.asStateFlow import java.io.BufferedInputStream import java.io.FileInputStream -import java.io.InputStream -import java.util.* +import java.util.UUID -/** - * Creates a new Upload Job for the given [UploadInfo]. The job is started as soon as it is created. - * To pause a job, save its last-known state in UploadPersistence and cancel the [Job] in the - * [UploadInfo] returned by this function - */ @JvmSynthetic -internal fun startUploadJob(upload: UploadInfo): UploadInfo { - return MuxUploadSdk.uploadJobFactory() - .createUploadJob(upload, CoroutineScope(Dispatchers.Default)) -} - -/** - * Creates upload coroutine jobs, which handle uploading a single file and reporting/delegating - * the state of the upload. To cancel, just call [Deferred.cancel] - * This class is not intended to be used from outside the SDK - */ -internal class UploadJobFactory private constructor( +internal fun startUploadJob(upload: UploadInfo): UploadInfo = MuxUploadSdk.uploadJobFactory() + .createUploadJob(upload, CoroutineScope(Dispatchers.Default)) + +/** Owns payload selection and the transition from cancellable preparation to transport. */ +internal class UploadJobFactory internal constructor( + private val prepare: suspend (UploadInfo) -> PreparedUpload = { + UploadPreparation().prepare(it, checkNotNull(MuxUploadManager.appContext)) + }, val createWorker: (ChunkWorker.Chunk, UploadInfo, MutableSharedFlow) -> ChunkWorker = - this::createWorkerForSlice + ::createWorkerForSlice, ) { - private val logger get() = MuxUploadSdk.logger - companion object { - private const val MIME_TYPE_GENERIC_VIDEO = "video/*" - - @JvmSynthetic - internal fun create() = UploadJobFactory() - - @Suppress("unused") // It's used by method-reference, which the linter doesn't see - @JvmSynthetic - private fun createWorkerForSlice( - chunk: ChunkWorker.Chunk, - uploadInfo: UploadInfo, - progressFlow: MutableSharedFlow - ): ChunkWorker = ChunkWorker.create( - chunk = chunk, - uploadInfo = uploadInfo, - videoMimeType = MIME_TYPE_GENERIC_VIDEO, - progressFlow = progressFlow, - ) + @JvmSynthetic internal fun create() = UploadJobFactory() + private fun createWorkerForSlice(chunk: ChunkWorker.Chunk, uploadInfo: UploadInfo, + progressFlow: MutableSharedFlow): ChunkWorker = + ChunkWorker.create(chunk, uploadInfo, "video/*", progressFlow) } fun createUploadJob(uploadInfo: UploadInfo, outerScope: CoroutineScope): UploadInfo { - logger - val statusFlow = MutableStateFlow(UploadStatus.Ready) - - var fileStream: InputStream = BufferedInputStream(FileInputStream(uploadInfo.inputFile)) - var fileSize = uploadInfo.inputFile.length() - val metrics = UploadMetrics.create() - - val uploadJob = outerScope.async { - // Inside the async { }, we have officially started - statusFlow.value = UploadStatus.Started + uploadInfo.attempt?.supersede() + val preparation = uploadInfo.attempt?.preparation ?: UploadPreparationState().apply { + originalSelected = uploadInfo.restoredFromOriginal || hasOriginalResumeState(uploadInfo) + } + val startTime = System.currentTimeMillis() + val attempt = UploadAttempt(preparation, MuxUpload.Progress( + bytesUploaded = readLastByteForFile(uploadInfo), totalBytes = uploadInfo.inputFile.length(), + startTime = startTime, updatedTime = startTime)) + var runningInfo = uploadInfo.update(attempt = attempt, statusFlow = attempt.status.asStateFlow()) + // Lazy startup lets the returned identity be installed before any completion callback. + val job = outerScope.async(start = CoroutineStart.LAZY) { val sessionId = UUID.randomUUID().toString() - // This UploadInfo never gets sent outside this coroutine. It contains info related to - // standardizing the the client doesn't need to know/can't know synchronously - var innerUploadInfo = uploadInfo - - val startTime = System.currentTimeMillis() + val metrics = UploadMetrics.create() try { - // See if the file need to be converted to a standard input. - if (uploadInfo.isStandardizationRequested() - && Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP - ) { - statusFlow.value = UploadStatus.Preparing - val tcx = TranscoderContext.create(innerUploadInfo, MuxUploadManager.appContext!!, sessionId) - innerUploadInfo = tcx.process() - if (tcx.fileTranscoded) { - fileStream = withContext(Dispatchers.IO) { - BufferedInputStream(FileInputStream(innerUploadInfo.standardizedFile)) + // An old export or request must finish cancellation before another attempt uses its files. + uploadInfo.uploadJob?.join() + ensureActive() + if (uploadInfo.generatedResumeBlocked || preparation.generatedRequestStarted || uploadInfo.standardizedFile != null) + throw GeneratedResumeBlockedException() + if (!preparation.originalSelected && preparation.verified == null && uploadInfo.isStandardizationRequested()) { + attempt.publish(UploadStatus.Preparing) + when (val result = prepare(runningInfo)) { + is PreparedUpload.Original -> { + preparation.originalSelected = true + MuxUploadSdk.logger.d("MuxUploadPreparation", result.diagnostic.toString()) } - // This !! is safe by contract: process() will set the standardizedFile if it transcoded - fileSize = innerUploadInfo.standardizedFile!!.length() + is PreparedUpload.Generated -> attempt.retainGenerated(result.output) } } - - // Now that we've standardized (or not), start the upload - - var totalBytesSent: Long = getAlreadyTransferredBytes(innerUploadInfo) - Log.d("UploadJobFactory", "totalBytesSent: $totalBytesSent") - val chunkBuffer = ByteArray(innerUploadInfo.chunkSize) - - // If we're resuming, we must skip to the current file pos - if (totalBytesSent != 0L) { - withContext(Dispatchers.IO) { fileStream.skip(totalBytesSent) } + ensureActive() + // A retained validated file can be replaced or modified while paused, before transport. + val verified = preparation.verified?.takeIf { it.matchesValidation() } + if (preparation.verified != null && verified == null) { + preparation.deleteOwnedFile() + preparation.originalSelected = true } - - statusFlow.value = UploadStatus.Uploading( - MuxUpload.Progress( - bytesUploaded = totalBytesSent, - totalBytes = fileSize, - startTime = startTime, - updatedTime = System.currentTimeMillis() - ) - ) - - // Upload each chunk starting from the current head of the stream - do { - // The last chunk will almost definitely be smaller than a whole chunk - val bytesLeft = fileSize - totalBytesSent - val thisChunkSize = if (innerUploadInfo.chunkSize > bytesLeft) { - bytesLeft.toInt() - } else { - innerUploadInfo.chunkSize - } - logger.i("UploadJob", "Trying to read $thisChunkSize bytes") - //read-in a chunk - val fileReadSize = withContext(Dispatchers.IO) { - fileStream.read(chunkBuffer, 0, thisChunkSize) - } - if (fileReadSize != thisChunkSize) { // Guaranteed unless the file was changed under us or sth - throw IllegalStateException("expected to read $thisChunkSize bytes, but read $fileReadSize") + val selected = verified?.file ?: uploadInfo.inputFile + val fileSize = selected.length() + check(selected.isFile && selected.canRead() && fileSize > 0) { "Upload payload is unreadable or empty" } + attempt.selectPayload(fileSize) + var totalBytesSent = attempt.confirmedProgress().bytesUploaded + check(totalBytesSent in 0..fileSize) { "Saved original offset is outside the payload" } + withContext(Dispatchers.IO) { BufferedInputStream(FileInputStream(selected)) }.use { stream -> + withContext(Dispatchers.IO) { + var remaining = totalBytesSent + while (remaining > 0) { + val skipped = stream.skip(remaining) + check(skipped > 0) { "Unable to seek to saved original offset" } + remaining -= skipped + } } - - val chunk = ChunkWorker.Chunk( - contentLength = thisChunkSize, - startByte = totalBytesSent, - endByte = totalBytesSent + thisChunkSize - 1, - totalFileSize = fileSize, - sliceData = chunkBuffer, - ) - - val chunkProgressFlow = callbackFlow() - var updateProgressJob: Job? = null - try { - // Bounce progress updates to callers - updateProgressJob = launch { - chunkProgressFlow.collect { chunkProgress -> - statusFlow.value = UploadStatus.Uploading( - MuxUpload.Progress( - bytesUploaded = chunkProgress.bytesUploaded + totalBytesSent, - totalBytes = fileSize, - startTime = startTime, - updatedTime = chunkProgress.updatedTime, - ) - ) // statusFlow.value = ... ( - } // chunkProgressChannel.collect { + attempt.publish(UploadStatus.Uploading(attempt.confirmedProgress())) + val buffer = ByteArray(uploadInfo.chunkSize) + while (totalBytesSent < fileSize) { + ensureActive() + check(verified?.matchesValidation() != false) { "Validated payload changed. Create a new Direct Upload." } + val count = minOf(buffer.size.toLong(), fileSize - totalBytesSent).toInt() + withContext(Dispatchers.IO) { + var read = 0 + while (read < count) { + val size = stream.read(buffer, read, count - read) + check(size > 0) { "Upload payload changed while reading" } + read += size + } } - - val chunkFinalState = createWorker(chunk, innerUploadInfo, chunkProgressFlow).upload() - - // Done with a chunk, so update the state again to capture all progress before looping - totalBytesSent += chunkFinalState.bytesUploaded - val intermediateProgress = MuxUpload.Progress( - bytesUploaded = totalBytesSent, - totalBytes = fileSize, - updatedTime = chunkFinalState.updatedTime, - startTime = startTime, - ) - statusFlow.value = UploadStatus.Uploading(intermediateProgress) - } finally { - updateProgressJob?.cancel() + val chunk = ChunkWorker.Chunk(totalBytesSent, totalBytesSent + count - 1, + fileSize, count, buffer) + val progress = MutableSharedFlow(replay = 1, + extraBufferCapacity = 2, onBufferOverflow = BufferOverflow.DROP_OLDEST) + val offset = totalBytesSent + val observer = launch { + progress.collect { value -> attempt.publish(UploadStatus.Uploading(value.copy( + bytesUploaded = value.bytesUploaded + offset, totalBytes = fileSize, startTime = startTime))) } + } + try { + // Save the generated request marker before transport, even if no response arrives. + attempt.beginTransport(verified != null) { + writeUploadState(runningInfo, attempt.confirmedProgress()) + } + val final = createWorker(chunk, runningInfo, progress).upload() + ensureActive() + totalBytesSent += final.bytesUploaded + val acknowledged = final.copy(bytesUploaded = totalBytesSent, totalBytes = fileSize, startTime = startTime) + attempt.acknowledge(acknowledged) { writeUploadState(runningInfo, acknowledged) } + } finally { observer.cancelAndJoin() } } - } while (totalBytesSent < fileSize) - - // We made it! - val finalProgress = createFinalState(fileSize, startTime) - // finish up - MainScope().launch { MuxUploadManager.jobFinished(innerUploadInfo) } - val success = UploadStatus.UploadSuccess(finalProgress) - if (!innerUploadInfo.optOut) { - metrics.reportUploadSucceeded( - startTime, finalProgress.updatedTime, 0, - sessionId, - uploadInfo - ) } - statusFlow.value = success + ensureActive() + val final = attempt.confirmedProgress() + val success = UploadStatus.UploadSuccess(final) + if (attempt.finish(success)) { + if (!uploadInfo.optOut) metrics.reportUploadSucceeded(startTime, final.updatedTime, 0, sessionId, uploadInfo) + withContext(Dispatchers.Main) { MuxUploadManager.jobFinished(runningInfo) } + } Result.success(success) + } catch (e: CancellationException) { + // Pause/cancel owns the public transition. Cancellation is never an upload result. + throw e } catch (e: Exception) { - MuxUploadSdk.logger.e("MuxUpload", "Upload of ${innerUploadInfo.inputFile} failed", e) - val finalState = createFinalState(fileSize, startTime) - val failStatus = UploadStatus.UploadFailed(e, finalState) - statusFlow.value = failStatus - if (!innerUploadInfo.optOut) { - metrics.reportUploadFailed( - startTime, System.currentTimeMillis(), 0, - e.message ?: "no error description", - sessionId, - uploadInfo - ) + val failure = UploadStatus.UploadFailed(e, attempt.confirmedProgress()) + if (attempt.finish(failure)) { + MuxUploadSdk.logger.e("MuxUpload", "Upload failed") + if (!uploadInfo.optOut) metrics.reportUploadFailed(startTime, System.currentTimeMillis(), 0, + "Upload failed", sessionId, uploadInfo) + withContext(Dispatchers.Main) { MuxUploadManager.jobFinished(runningInfo, false) } } - MainScope().launch { MuxUploadManager.jobFinished(innerUploadInfo, false) } Result.failure(e) } finally { - @Suppress("BlockingMethodInNonBlockingContext") // the streams we use don't block on close - fileStream.close() - innerUploadInfo.standardizedFile?.delete() + if (!attempt.isStopped() || attempt.isCancelled()) preparation.deleteOwnedFile() } - } // val uploadJob = ... - - return uploadInfo.update( - statusFlow = statusFlow.asStateFlow(), - uploadJob = uploadJob, - ) + } + runningInfo = runningInfo.update(uploadJob = job) + job.start() + return runningInfo } - - private fun createFinalState(fileSize: Long, startTime: Long): MuxUpload.Progress { - return MuxUpload.Progress( - bytesUploaded = fileSize, - totalBytes = fileSize, - startTime = startTime, - updatedTime = System.currentTimeMillis(), - ) - } - - private fun getAlreadyTransferredBytes(file: UploadInfo): Long = readLastByteForFile(file) - - private fun callbackFlow() = - MutableSharedFlow( - replay = 1, - extraBufferCapacity = 2, // Some slop for UI to miss events - onBufferOverflow = BufferOverflow.DROP_OLDEST, - ) } diff --git a/library/src/main/java/com/mux/video/upload/internal/UploadPersistence.kt b/library/src/main/java/com/mux/video/upload/internal/UploadPersistence.kt index d6cbd2d5..956fddf1 100644 --- a/library/src/main/java/com/mux/video/upload/internal/UploadPersistence.kt +++ b/library/src/main/java/com/mux/video/upload/internal/UploadPersistence.kt @@ -32,24 +32,32 @@ internal fun writeUploadState(uploadInfo: UploadInfo, state: MuxUpload.Progress) retriesPerChunk = uploadInfo.retriesPerChunk, optOut = uploadInfo.optOut, inputStandardization = uploadInfo.inputStandardization, + generatedResumeBlocked = uploadInfo.attempt?.preparation?.generatedRequestStarted == true || uploadInfo.generatedResumeBlocked, ) ) } @JvmSynthetic internal fun readLastByteForFile(upload: UploadInfo): Long { - return UploadPersistence.readEntries()[upload.inputFile.absolutePath]?.bytesSent ?: 0 + val entry = UploadPersistence.readEntries()[upload.inputFile.absolutePath] + return entry?.takeIf { it.url == upload.remoteUri.toString() }?.bytesSent ?: 0 } +internal fun hasOriginalResumeState(upload: UploadInfo): Boolean = + UploadPersistence.readEntries()[upload.inputFile.absolutePath]?.let { + it.url == upload.remoteUri.toString() && !it.generatedResumeBlocked + } == true + @JvmSynthetic internal fun forgetUploadState(uploadInfo: UploadInfo) { UploadPersistence.removeForFile(uploadInfo) } @JvmSynthetic -internal fun readAllCachedUploads(): List { +internal fun readAllCachedUploads(includePaused: Boolean = true): List { return UploadPersistence.readEntries() .map { it.value } + .filter { includePaused || it.state == UploadPersistence.WAS_RUNNING } .map { UploadInfo( inputStandardization = it.inputStandardization, @@ -60,6 +68,8 @@ internal fun readAllCachedUploads(): List { optOut = it.optOut, uploadJob = null, statusFlow = null, + restoredFromOriginal = true, + generatedResumeBlocked = it.generatedResumeBlocked, ) } } @@ -153,11 +163,13 @@ private data class UploadEntry( val state: Int, val bytesSent: Long, val inputStandardization: InputStandardization, + val generatedResumeBlocked: Boolean = false, ) { fun toJson(): JSONObject { return JSONObject().apply { put("file", file.absolutePath) put("data", JSONObject().apply { + put("generated_resume_blocked", generatedResumeBlocked) put("url", url) put("chunk_size", chunkSize) put("retries_per_chunk", retriesPerChunk) @@ -187,6 +199,7 @@ private fun JSONObject.parsePersistenceEntry(): UploadEntry { savedAtLocalMs = data.optLong("saved_at_local_ms"), state = data.optInt("state"), bytesSent = data.optLong("bytes_sent"), + generatedResumeBlocked = data.optBoolean("generated_resume_blocked", false), inputStandardization = data.optJSONObject("input_standardization").let { options -> InputStandardization( standardizationRequested = options?.optBoolean("requested", true) ?: true, diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt index aac325a9..1a91fbd9 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt @@ -6,6 +6,8 @@ import android.os.HandlerThread import android.os.Looper import androidx.media3.common.MimeTypes import java.io.File +import java.io.RandomAccessFile +import java.security.MessageDigest import java.util.concurrent.atomic.AtomicBoolean internal sealed interface SdrConversionResult { @@ -17,6 +19,32 @@ internal sealed interface SdrConversionResult { /** The only deletion authority returned to orchestration; never accepts an arbitrary path. */ internal class SdrGeneratedFile private constructor(val file: File) { + private var validatedIdentity: FileIdentity? = null + internal fun recordValidation(): Boolean { + validatedIdentity = identity() + return validatedIdentity != null + } + internal fun matchesValidation(): Boolean = validatedIdentity?.let { it == identity() } == true + + private data class FileIdentity(val size: Long, val modified: Long, val edgeHash: List) + private fun identity(): FileIdentity? = try { + val size = file.length() + val modified = file.lastModified() + if (!file.isFile || !file.canRead() || size <= 0) null else { + val digest = MessageDigest.getInstance("SHA-256") + RandomAccessFile(file, "r").use { input -> + val first = ByteArray(minOf(size, 64 * 1024L).toInt()) + input.readFully(first); digest.update(first) + if (size > first.size) { + val last = ByteArray(minOf(size - first.size, 64 * 1024L).toInt()) + input.seek(size - last.size); input.readFully(last); digest.update(last) + } + } + if (file.length() != size || file.lastModified() != modified) null + else FileIdentity(size, modified, digest.digest().toList()) + } + } catch (_: Exception) { null } + fun delete(): Boolean = !file.exists() || file.delete() companion object { @@ -44,6 +72,7 @@ internal class SdrConversionAdapter( StandardInputOutputValidator()::validateGeneratedOutput, private val createEngine: (Context, Looper, StandardInputConversion, SdrEncodingTargets, SdrEncoderCapability) -> SdrExportEngine = ::Media3SdrExportEngine, + private val onPendingCleanup: (SdrGeneratedFile) -> Unit = {}, ) { private val appContext = context.applicationContext @@ -110,6 +139,7 @@ internal class SdrConversionAdapter( val expectedLayout = aacChannelLayout(targets.audioChannels).valueOrNull if (validation.facts.videoProfile != MediaFact.Known(targets.profile) || (targets.audioChannels != null && audio?.layout != expectedLayout)) fail(SdrConversionFailure.OutputInvalid) + else if (!owned.recordValidation()) fail(SdrConversionFailure.OutputInvalid) else finish(SdrConversionResult.Completed(owned, validation.facts)) } is StandardInputOutputValidation.Rejected -> fail(SdrConversionFailure.OutputInvalid) @@ -141,6 +171,7 @@ internal class SdrConversionAdapter( runCatching { engine?.cancel() } if (result !is SdrConversionResult.Completed && output?.delete() == false) { pendingCleanup = output + output?.let(onPendingCleanup) // Keep ownership at its exact path if the OS refuses deletion; never sweep a directory. if (result !is SdrConversionResult.Cancelled) { onResult(SdrConversionResult.Failed(SdrConversionFailure.FileOwnership)) diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/UploadPreparation.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/UploadPreparation.kt new file mode 100644 index 00000000..76759c98 --- /dev/null +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/UploadPreparation.kt @@ -0,0 +1,80 @@ +package com.mux.video.upload.internal.standardization + +import android.content.Context +import com.mux.video.upload.internal.UploadInfo +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.ensureActive +import kotlinx.coroutines.suspendCancellableCoroutine +import kotlinx.coroutines.withContext +import kotlin.coroutines.coroutineContext +import kotlin.coroutines.resume + +/** Closed diagnostics contain no customer paths, URLs, or platform error text. */ +internal sealed interface PreparationDiagnostic { + data object Original : PreparationDiagnostic + data object InspectionFailed : PreparationDiagnostic + data object UnsupportedPlan : PreparationDiagnostic + data object GeneratedResumeUnavailable : PreparationDiagnostic + data class ConversionFailed(val reason: SdrConversionFailure) : PreparationDiagnostic + data object Validated : PreparationDiagnostic +} + +internal sealed interface PreparedUpload { + val diagnostic: PreparationDiagnostic + data class Original(override val diagnostic: PreparationDiagnostic = PreparationDiagnostic.Original) : PreparedUpload + data class Generated(val output: SdrGeneratedFile, + override val diagnostic: PreparationDiagnostic = PreparationDiagnostic.Validated) : PreparedUpload +} + +internal class UploadPreparation( + private val inspectMetadata: (java.io.File) -> MetadataInspectionResult = MediaMetadataInspector()::inspect, + private val inspectSamples: (java.io.File, MediaMetadataInspection, () -> Boolean) -> MediaSampleInspection = + { file, metadata, cancelled -> StandardInputTimelineInspector().inspect(file, metadata, cancelled) }, + private val convert: suspend (Context, UploadInfo, MediaMetadataInspection, MediaSampleInspection, StandardInputConversion) -> SdrConversionResult = + { context, upload, metadata, source, conversion -> + suspendCancellableCoroutine { continuation -> + val attempt = SdrConversionAdapter(context, onPendingCleanup = { upload.attempt?.trackCleanup(it) }) + .start(upload.inputFile, metadata, source, conversion) { result -> + // Transfer ownership before dispatch: completion can race pause or cancellation. + if (result is SdrConversionResult.Completed) upload.attempt?.retainGenerated(result.output) + continuation.resume(result) { _, value, _ -> + if (value is SdrConversionResult.Completed && upload.attempt == null) value.output.delete() + } + } + continuation.invokeOnCancellation { attempt.cancel() } + } + }, +) { + suspend fun prepare(upload: UploadInfo, context: Context, + generatedResumeVerified: Boolean = false): PreparedUpload = withContext(Dispatchers.IO) { + fun original(reason: PreparationDiagnostic) = PreparedUpload.Original(reason) + if (!upload.isStandardizationRequested()) return@withContext original(PreparationDiagnostic.Original) + val job = coroutineContext + try { + val metadata = (inspectMetadata(upload.inputFile) as? MetadataInspectionResult.Success)?.inspection + ?: return@withContext original(PreparationDiagnostic.InspectionFailed) + job.ensureActive() + val source = inspectSamples(upload.inputFile, metadata) { !job[kotlinx.coroutines.Job]!!.isActive } + job.ensureActive() + val action = StandardInputPlanner().plan(source.facts, upload.inputStandardization).action + if (action is StandardInputAction.UploadOriginal) return@withContext original(PreparationDiagnostic.Original) + // The adapter provides source-specific preflight and proves actual output. Planning keeps + // unsupported HDR and unsafe structure out of this SDR boundary. + val conversion = (action as? StandardInputAction.Fallback)?.reason.let { + (it as? FallbackReason.UnsupportedConversion)?.conversion + } ?: return@withContext original(PreparationDiagnostic.UnsupportedPlan) + if (SdrEncodingTargets.from(conversion, metadata, source) == null) + return@withContext original(PreparationDiagnostic.UnsupportedPlan) + // Until the Mux status/offset protocol is verified, new attempts upload original bytes. + // This gate also avoids spending conversion time on an unusable replacement payload. + if (!generatedResumeVerified) return@withContext original(PreparationDiagnostic.GeneratedResumeUnavailable) + when (val result = convert(context, upload, metadata, source, conversion)) { + is SdrConversionResult.Completed -> PreparedUpload.Generated(result.output) + is SdrConversionResult.Failed -> original(PreparationDiagnostic.ConversionFailed(result.reason)) + SdrConversionResult.Cancelled -> throw kotlinx.coroutines.CancellationException("Preparation cancelled") + } + } catch (e: kotlinx.coroutines.CancellationException) { throw e } + catch (_: LinkageError) { original(PreparationDiagnostic.InspectionFailed) } + catch (_: Exception) { original(PreparationDiagnostic.InspectionFailed) } + } +} diff --git a/library/src/test/java/com/mux/video/upload/internal/ChunkCancellationTests.kt b/library/src/test/java/com/mux/video/upload/internal/ChunkCancellationTests.kt new file mode 100644 index 00000000..9a145f5e --- /dev/null +++ b/library/src/test/java/com/mux/video/upload/internal/ChunkCancellationTests.kt @@ -0,0 +1,64 @@ +package com.mux.video.upload.internal + +import android.net.Uri +import com.mux.exoplayeradapter.AbsRobolectricTest +import com.mux.video.upload.MuxUploadSdk +import com.mux.video.upload.api.MuxUpload +import io.mockk.* +import kotlinx.coroutines.* +import kotlinx.coroutines.flow.MutableSharedFlow +import kotlinx.coroutines.test.runTest +import okhttp3.* +import okhttp3.Call +import okhttp3.MediaType.Companion.toMediaType +import okio.BufferedSource +import okio.Buffer +import org.junit.After +import org.junit.Assert.* +import org.junit.Test +import java.io.File + +class ChunkCancellationTests : AbsRobolectricTest() { + @After fun tearDown() { unmockkAll() } + + @OptIn(ExperimentalCoroutinesApi::class) + @Test fun cancelledRequestIsNotRetriedAndLateResponseIsClosed() = runTest { + val callback = slot() + val request = slot() + val call = mockk { + every { enqueue(capture(callback)) } just Runs + every { cancel() } just Runs + } + val client = mockk { every { newCall(capture(request)) } returns call } + mockkObject(MuxUploadSdk) + every { MuxUploadSdk.httpClient() } returns client + every { MuxUploadSdk.logger } returns MuxUploadSdk.noLogger() + val info = UploadInfo(remoteUri = Uri.parse("https://example.invalid/upload"), inputFile = File("unused"), + chunkSize = 4, retriesPerChunk = 3, optOut = true, uploadJob = null, statusFlow = null) + val progress = MutableSharedFlow(replay = 1) + val worker = ChunkWorker.create(ChunkWorker.Chunk(4, 7, 12, 4, byteArrayOf(1, 2, 3, 4)), info, "video/*", progress) + val job = async { worker.upload() } + testScheduler.runCurrent() + assertEquals("bytes 4-7/12", request.captured.header("Content-Range")) + val buffer = Buffer() + request.captured.body!!.writeTo(buffer) + assertArrayEquals(byteArrayOf(1, 2, 3, 4), buffer.readByteArray()) + job.cancel() + testScheduler.runCurrent() + verify(exactly = 1) { call.cancel() } + var closed = false + val body = object : ResponseBody() { + override fun contentType() = "video/*".toMediaType() + override fun contentLength() = 0L + override fun source(): BufferedSource = Buffer() + override fun close() { closed = true } + } + callback.captured.onResponse(call, Response.Builder().request(request.captured) + .protocol(Protocol.HTTP_1_1).code(200).message("OK").body(body).build()) + testScheduler.runCurrent() + assertTrue(job.isCancelled) + assertTrue(closed) + assertTrue(progress.replayCache.isEmpty()) + verify(exactly = 1) { client.newCall(any()) } + } +} diff --git a/library/src/test/java/com/mux/video/upload/internal/StandardInputLifecycleTests.kt b/library/src/test/java/com/mux/video/upload/internal/StandardInputLifecycleTests.kt index 504fb57b..fec16c2c 100644 --- a/library/src/test/java/com/mux/video/upload/internal/StandardInputLifecycleTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/StandardInputLifecycleTests.kt @@ -5,6 +5,8 @@ import com.mux.video.upload.MuxUploadSdk import com.mux.video.upload.api.HdrHandling import com.mux.video.upload.api.MuxUpload import com.mux.video.upload.api.MuxUploadManager +import com.mux.video.upload.internal.standardization.UploadPreparation +import com.mux.video.upload.internal.standardization.PreparedUpload import io.mockk.* import kotlinx.coroutines.* import kotlinx.coroutines.test.* @@ -18,8 +20,7 @@ import java.io.File /** * Configuration retention through the public lifecycle, using unchanged original bytes. - * This test does not validate media conversion or generated-payload resume safety. - * That integration coverage remains to be implemented with the new preparation and resume pipeline. + * Generated payload ownership and resume blocking are covered in UploadPreparationLifecycleTests. */ @Config(sdk = [28]) @OptIn(ExperimentalCoroutinesApi::class) @@ -47,14 +48,10 @@ class StandardInputLifecycleTests : AbsRobolectricTest() { } mockkObject(MuxUploadSdk) every { MuxUploadSdk.uploadJobFactory() } returns factory - mockkObject(TranscoderContext.Companion) - every { TranscoderContext.create(any(), any(), any()) } answers { - val upload = firstArg() - // Always select the original file. Repeating this stub never regenerates payload bytes. - mockk { - every { fileTranscoded } returns false - coEvery { process() } coAnswers { preparations++; upload } - } + mockkConstructor(UploadPreparation::class) + coEvery { anyConstructed().prepare(any(), any(), any()) } coAnswers { + preparations++ + PreparedUpload.Original() } mockkObject(UploadMetrics.Companion) every { UploadMetrics.create() } returns mockk(relaxed = true) @@ -66,7 +63,8 @@ class StandardInputLifecycleTests : AbsRobolectricTest() { coEvery { upload() } coAnswers { workerOptions += upload.inputStandardization chunkOffsets += chunk.startByte - writeUploadState(upload, MuxUpload.Progress(bytesUploaded = 3, totalBytes = 16)) + val acknowledged = MuxUpload.Progress(bytesUploaded = 3, totalBytes = 16) + upload.attempt!!.acknowledge(acknowledged) { writeUploadState(upload, acknowledged) } awaitCancellation() } } diff --git a/library/src/test/java/com/mux/video/upload/internal/StandardInputPersistenceTests.kt b/library/src/test/java/com/mux/video/upload/internal/StandardInputPersistenceTests.kt index f51f1716..e70fda61 100644 --- a/library/src/test/java/com/mux/video/upload/internal/StandardInputPersistenceTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/StandardInputPersistenceTests.kt @@ -31,7 +31,7 @@ class StandardInputPersistenceTests : AbsRobolectricTest() { val original = upload(InputStandardization(requested, resolution, handling)) writeUploadState(original, MuxUpload.Progress(bytesUploaded = 5, totalBytes = 10)) initializeUploadPersistence(context) - assertEquals(original, readAllCachedUploads().single()) + assertEquals(original.update(restoredFromOriginal = true), readAllCachedUploads().single()) assertEquals(5L, readLastByteForFile(original)) } } diff --git a/library/src/test/java/com/mux/video/upload/internal/UploadPersistenceTests.kt b/library/src/test/java/com/mux/video/upload/internal/UploadPersistenceTests.kt index 61e9606a..b6c0769d 100644 --- a/library/src/test/java/com/mux/video/upload/internal/UploadPersistenceTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/UploadPersistenceTests.kt @@ -88,7 +88,7 @@ class UploadPersistenceTests : AbsRobolectricTest() { ) assertEquals( "upload A should be read as written", - uploadInfoInA, + uploadInfoInA.update(restoredFromOriginal = true), uploadOutA ) } @@ -164,4 +164,4 @@ class UploadPersistenceTests : AbsRobolectricTest() { } } -} \ No newline at end of file +} diff --git a/library/src/test/java/com/mux/video/upload/internal/UploadPreparationLifecycleTests.kt b/library/src/test/java/com/mux/video/upload/internal/UploadPreparationLifecycleTests.kt new file mode 100644 index 00000000..42f6ba78 --- /dev/null +++ b/library/src/test/java/com/mux/video/upload/internal/UploadPreparationLifecycleTests.kt @@ -0,0 +1,388 @@ +package com.mux.video.upload.internal + +import android.net.Uri +import com.mux.exoplayeradapter.AbsRobolectricTest +import com.mux.video.upload.MuxUploadSdk +import com.mux.video.upload.api.MuxUpload +import com.mux.video.upload.api.MuxUploadManager +import com.mux.video.upload.api.UploadStatus +import com.mux.video.upload.internal.standardization.PreparedUpload +import com.mux.video.upload.internal.standardization.SdrGeneratedFile +import io.mockk.* +import kotlinx.coroutines.* +import kotlinx.coroutines.flow.MutableSharedFlow +import kotlinx.coroutines.test.* +import org.junit.After +import org.junit.Assert.* +import org.junit.Before +import org.junit.Test +import org.robolectric.RuntimeEnvironment +import org.robolectric.annotation.Config +import java.io.File + +@Config(sdk = [28]) +@OptIn(ExperimentalCoroutinesApi::class) +class UploadPreparationLifecycleTests : AbsRobolectricTest() { + private val dispatcher = StandardTestDispatcher() + private val scope = CoroutineScope(SupervisorJob() + dispatcher) + private val context get() = RuntimeEnvironment.getApplication() + private lateinit var source: File + private lateinit var sibling: File + private var preparations = 0 + private var prepare: suspend (UploadInfo) -> PreparedUpload = { PreparedUpload.Original() } + private var work: suspend (ChunkWorker.Chunk, UploadInfo, MutableSharedFlow) -> MuxUpload.Progress = + { chunk, _, _ -> MuxUpload.Progress(bytesUploaded = chunk.contentLength.toLong()) } + private val chunks = mutableListOf() + private val statuses = mutableListOf() + private val results = mutableListOf>() + private val progress = mutableListOf() + private val owned = mutableListOf() + + @Before fun setUp() { + Dispatchers.setMain(dispatcher) + context.getSharedPreferences("mux_upload", 0).edit().clear().commit() + MuxUploadSdk.initialize(context, false) + MuxUploadSdk.useLogger(MuxUploadSdk.noLogger()) + source = File.createTempFile("customer", ".mp4", context.cacheDir).apply { writeBytes(ByteArray(16) { it.toByte() }) } + sibling = File(context.cacheDir, "mux-upload/customer-copy.mp4").apply { parentFile!!.mkdirs(); writeText("customer") } + val real = UploadJobFactory(prepare = { preparations++; prepare(it) }, createWorker = { chunk, info, flow -> + // The factory reuses its buffer; snapshot the actual payload for assertions. + chunks += chunk.copy(sliceData = chunk.sliceData.copyOf(chunk.contentLength)) + mockk { coEvery { upload() } coAnswers { work(chunk, info, flow) } } + }) + val factory = mockk() + every { factory.createUploadJob(any(), any()) } answers { real.createUploadJob(firstArg(), scope) } + mockkObject(MuxUploadSdk) + every { MuxUploadSdk.uploadJobFactory() } returns factory + mockkObject(UploadMetrics.Companion) + every { UploadMetrics.create() } returns mockk(relaxed = true) + } + + @After fun tearDown() { + MuxUploadManager.allUploadJobs().forEach { it.cancel() } + scope.cancel() + dispatcher.scheduler.runCurrent() + owned.forEach { it.delete() } + source.delete(); sibling.delete() + unmockkAll() + Dispatchers.resetMain() + } + + @Test fun originalBytesCompleteWithCumulativeProgressAndOneResult() { + val upload = observe(MuxUpload.create(info())) + upload.start() + pump { upload.isSuccessful } + assertArrayEquals(source.readBytes(), chunks.flatMap { it.sliceData.toList() }.toByteArray()) + assertEquals(listOf(0L, 4L, 8L, 12L), chunks.map { it.startByte }) + assertEquals(16L, upload.currentProgress.bytesUploaded) + assertEquals(16L, upload.currentProgress.totalBytes) + assertEquals(1, results.size) + assertTrue(readAllCachedUploads().isEmpty()) + assertTrue(source.exists()); assertTrue(sibling.exists()) + var lateResults = 0 + upload.setResultListener { lateResults++ } + dispatcher.scheduler.runCurrent() + assertEquals(1, lateResults) + } + + @Test fun automaticLegacyRestorationUsesOriginalOffsetWithoutPreparation() { + writeUploadState(info(), MuxUpload.Progress(bytesUploaded = 8, totalBytes = 16)) + // Remove new fields to model a pre-feature persistence record. + val prefs = context.getSharedPreferences("mux_upload", 0) + val json = org.json.JSONArray(prefs.getString("uploads", null)) + json.getJSONObject(0).getJSONObject("data").remove("generated_resume_blocked") + json.getJSONObject(0).getJSONObject("data").remove("input_standardization") + json.getJSONObject(0).getJSONObject("data").put("state", 0) + prefs.edit().putString("uploads", json.toString()).commit() + work = { _, _, _ -> awaitCancellation() } + MuxUploadSdk.initialize(context, true) + pump { chunks.isNotEmpty() } + val restored = MuxUploadManager.resumeAllCachedJobs().single() + assertTrue(restored.isRunning) + assertTrue(restored.uploadStatus is UploadStatus.Uploading) + assertEquals(0, preparations) + assertEquals(8L, chunks.single().startByte) + assertArrayEquals(source.readBytes().sliceArray(8..11), chunks.single().sliceData) + } + + @Test fun preparationPauseStopsWorkIsSilentAndResumesOnlyOnStart() { + var stopped = false + prepare = { try { awaitCancellation() } finally { stopped = true } } + val upload = observe(MuxUpload.create(info())) + upload.start() + pump { upload.uploadStatus is UploadStatus.Preparing } + upload.pause() + pump { stopped } + assertTrue(upload.isPaused); assertFalse(upload.isRunning) + assertEquals(0L, upload.currentProgress.bytesUploaded) + assertTrue(statuses.last() is UploadStatus.UploadPaused) + assertTrue(results.isEmpty()); assertTrue(chunks.isEmpty()) + assertEquals(0L, readLastByteForFile(info())) + dispatcher.scheduler.runCurrent() + assertEquals(1, preparations) + prepare = { PreparedUpload.Original() } + upload.start() + pump { upload.isSuccessful } + assertEquals(2, preparations) + } + + @Test fun initializationDoesNotAutomaticallyResumeAnExplicitlyPausedRecord() { + work = { _, _, _ -> awaitCancellation() } + val upload = MuxUpload.create(info()) + upload.start() + pump { chunks.size == 1 } + upload.pause() + dispatcher.scheduler.runCurrent() + MuxUploadManager.jobFinished(internalInfo(upload), false) + MuxUploadSdk.initialize(context, true) + dispatcher.scheduler.runCurrent() + assertEquals(1, chunks.size) + assertEquals(1, readAllCachedUploads().size) + assertTrue(MuxUploadManager.allUploadJobs().isEmpty()) + val restored = MuxUpload.create(info()) + restored.start() + pump { chunks.size == 2 } + assertEquals(1, preparations) + } + + @Test fun uploadPauseUsesAcknowledgedProgressAndSuppressesLateEvents() { + var late: MutableSharedFlow? = null + work = { chunk, _, flow -> + if (chunk.startByte == 0L) MuxUpload.Progress(bytesUploaded = 4) + else { late = flow; flow.emit(MuxUpload.Progress(bytesUploaded = 3)); awaitCancellation() } + } + val upload = observe(MuxUpload.create(info())) + upload.start() + pump { upload.currentProgress.bytesUploaded == 7L } + val managerHandle = MuxUploadManager.findUploadByFile(source)!! + managerHandle.pause() + pump { upload.isPaused } + assertEquals(4L, upload.currentProgress.bytesUploaded) + assertEquals(4L, readLastByteForFile(info())) + val count = statuses.size + late!!.tryEmit(MuxUpload.Progress(bytesUploaded = 4)) + dispatcher.scheduler.runCurrent() + assertEquals(count, statuses.size) + assertTrue(results.isEmpty()) + upload.start() + pump { chunks.size == 3 } + assertEquals(4L, chunks.last().startByte) + assertEquals(1, preparations) + } + + @Test fun cancellationDeletesOnlyOwnedFilesAndIgnoresLatePreparationCompletion() { + val release = CompletableDeferred() + val generated = generated(8) + prepare = { withContext(NonCancellable) { release.await(); PreparedUpload.Generated(generated) } } + val upload = observe(MuxUpload.create(info())) + upload.start() + pump { preparations == 1 } + val job = internalInfo(upload).uploadJob!! + upload.cancel(); upload.cancel() + release.complete(Unit) + pump { job.isCompleted } + val count = statuses.size + upload.setStatusListener { statuses += it } + upload.setProgressListener { progress += it } + upload.setResultListener { results += it } + dispatcher.scheduler.runCurrent() + assertEquals(count, statuses.size) + assertTrue(results.isEmpty()); assertTrue(chunks.isEmpty()) + assertFalse(generated.file.exists()) + assertTrue(source.exists()); assertTrue(sibling.exists()) + assertTrue(readAllCachedUploads().isEmpty()) + assertNull(MuxUploadManager.findUploadByFile(source)) + } + + @Test fun verifiedPreparationOutputSurvivesPauseAndUsesSelectedLengthOnResume() { + val release = CompletableDeferred() + val generated = generated(8) + prepare = { withContext(NonCancellable) { release.await(); PreparedUpload.Generated(generated) } } + val upload = observe(MuxUpload.create(info())) + upload.start() + pump { preparations == 1 } + val job = internalInfo(upload).uploadJob!! + upload.pause() + release.complete(Unit) + pump { job.isCompleted } + assertTrue(generated.file.exists()) + assertTrue(chunks.isEmpty()); assertTrue(results.isEmpty()) + upload.start() + pump { upload.isSuccessful } + assertEquals(1, preparations) + assertEquals(8L, upload.currentProgress.totalBytes) + assertEquals(8L, upload.currentProgress.bytesUploaded) + assertEquals(listOf(8L, 8L), chunks.map { it.totalFileSize }) + assertFalse(generated.file.exists()) + assertTrue(source.exists()); assertTrue(sibling.exists()) + } + + @Test fun generatedRequestCannotResumeInProcessOrAfterRestoration() { + val generated = generated(8) + prepare = { PreparedUpload.Generated(generated) } + work = { _, _, _ -> awaitCancellation() } + val upload = observe(MuxUpload.create(info())) + upload.start() + pump { chunks.isNotEmpty() } + upload.pause() + dispatcher.scheduler.runCurrent() + assertTrue(generated.file.exists()) + upload.start() + pump { upload.uploadStatus is UploadStatus.UploadFailed } + assertTrue(upload.error is GeneratedResumeBlockedException) + assertEquals(1, chunks.size) + val failedJob = internalInfo(upload).uploadJob + upload.start(forceRestart = true) + pump { internalInfo(upload).uploadJob !== failedJob && upload.uploadStatus is UploadStatus.UploadFailed } + assertTrue(upload.error is GeneratedResumeBlockedException) + assertEquals(1, chunks.size) + assertTrue(readAllCachedUploads().single().generatedResumeBlocked) + val restored = observe(MuxUploadManager.resumeAllCachedJobs().single()) + pump { restored.uploadStatus is UploadStatus.UploadFailed } + assertTrue(restored.error is GeneratedResumeBlockedException) + assertEquals(1, chunks.size) + } + + @Test fun changedVerifiedOutputBeforeNetworkFallsBackToOriginalEvenWithSameSizeAndMtime() { + val release = CompletableDeferred() + val generated = generated(8) + prepare = { withContext(NonCancellable) { release.await(); PreparedUpload.Generated(generated) } } + val upload = MuxUpload.create(info()) + upload.start() + pump { preparations == 1 } + val job = internalInfo(upload).uploadJob!! + upload.pause(); release.complete(Unit) + pump { job.isCompleted } + val modified = generated.file.lastModified() + generated.file.writeBytes(ByteArray(8) { 99 }) + assertTrue(generated.file.setLastModified(modified)) + upload.start() + pump { upload.isSuccessful } + assertEquals(16L, upload.currentProgress.totalBytes) + assertArrayEquals(source.readBytes(), chunks.flatMap { it.sliceData.toList() }.toByteArray()) + assertFalse(generated.file.exists()) + } + + @Test fun latePreparationCannotReplacePayloadAfterResumeStartsANewAttempt() { + val release = CompletableDeferred() + val generated = generated(8) + prepare = { withContext(NonCancellable) { release.await(); PreparedUpload.Generated(generated) } } + val upload = MuxUpload.create(info()) + upload.start() + pump { preparations == 1 } + upload.pause() + prepare = { PreparedUpload.Original() } + upload.start() + release.complete(Unit) + pump { upload.isSuccessful } + assertEquals(2, preparations) + assertEquals(16L, upload.currentProgress.totalBytes) + assertFalse(generated.file.exists()) + } + + @Test fun cancellationFromStatusCallbackSuppressesProgressAndResult() { + val upload = MuxUpload.create(info()) + upload.setResultListener { results += it } + upload.setProgressListener { progress += it } + upload.setStatusListener { + statuses += it + if (it is UploadStatus.Uploading) upload.cancel() + } + upload.start() + pump { statuses.any { it is UploadStatus.Uploading } } + dispatcher.scheduler.runCurrent() + assertTrue(progress.isEmpty()); assertTrue(results.isEmpty()) + assertTrue(readAllCachedUploads().isEmpty()) + } + + @Test fun cancellingAFailedUploadRemovesItsRetainedResumeRecord() { + work = { chunk, _, _ -> + if (chunk.startByte == 0L) MuxUpload.Progress(bytesUploaded = 4) + else throw java.io.IOException("Test transport failure") + } + val upload = MuxUpload.create(info()) + upload.start() + pump { upload.uploadStatus is UploadStatus.UploadFailed && MuxUploadManager.findUploadByFile(source) == null } + assertEquals(4L, readLastByteForFile(info())) + upload.cancel() + assertTrue(readAllCachedUploads().isEmpty()) + assertTrue(source.exists()); assertTrue(sibling.exists()) + } + + @Test fun staleCompletionCannotRemoveReplacementManagerJob() { + work = { _, _, _ -> awaitCancellation() } + val upload = MuxUpload.create(info()) + upload.start() + pump { chunks.isNotEmpty() } + val old = internalInfo(upload) + upload.start(forceRestart = true) + pump { chunks.size == 2 } + MuxUploadManager.jobFinished(old) + assertTrue(MuxUploadManager.findUploadByFile(source)!!.isRunning) + } + + @Test fun unmanagedBuilderPauseAndCancelHaveSameListenerContract() { + work = { _, _, _ -> awaitCancellation() } + val upload = observe(MuxUpload.Builder("https://example.invalid/upload", source) + .manageUploadTask(false).standardizationRequested(false).build()) + upload.start() + pump { chunks.isNotEmpty() } + assertNull(MuxUploadManager.findUploadByFile(source)) + upload.pause() + dispatcher.scheduler.runCurrent() + assertTrue(upload.isPaused); assertTrue(results.isEmpty()) + upload.cancel() + assertTrue(readAllCachedUploads().isEmpty()) + } + + @Test fun unmanagedForceRestartStopsPreviousWorkerBeforeStartingAnother() { + work = { _, _, _ -> awaitCancellation() } + val upload = MuxUpload.Builder("https://example.invalid/upload", source) + .manageUploadTask(false).standardizationRequested(false).build() + upload.start() + pump { chunks.size == 1 } + val old = internalInfo(upload).uploadJob!! + upload.start(forceRestart = true) + pump { chunks.size == 2 } + assertTrue(old.isCancelled) + assertEquals(0L, chunks.last().startByte) + upload.cancel() + } + + @Test fun newDirectUploadUsesOriginalFallbackWithoutOldOffsetOrGeneratedPayload() { + val generated = generated(8) + prepare = { PreparedUpload.Generated(generated) } + work = { _, _, _ -> awaitCancellation() } + val upload = MuxUpload.create(info()) + upload.start() + pump { chunks.size == 1 } + upload.pause() + prepare = { PreparedUpload.Original() } + val next = MuxUpload.create(info().copy(remoteUri = Uri.parse("https://example.invalid/new-upload"))) + next.start() + pump { chunks.size == 2 } + assertEquals(0L, chunks.last().startByte) + assertEquals(16L, chunks.last().totalFileSize) + assertArrayEquals(source.readBytes().take(4).toByteArray(), chunks.last().sliceData) + assertFalse(generated.file.exists()) + } + + private fun observe(upload: MuxUpload) = upload.apply { + setStatusListener { statuses += it } + setResultListener { results += it } + setProgressListener { progress += it } + } + private fun generated(size: Int) = SdrGeneratedFile.allocate(context.cacheDir, 1)!!.also { + it.file.writeBytes(ByteArray(size) { 42 }); assertTrue(it.recordValidation()); owned += it + } + private fun info() = UploadInfo(remoteUri = Uri.parse("https://example.invalid/upload"), inputFile = source, + chunkSize = 4, retriesPerChunk = 3, optOut = true, uploadJob = null, statusFlow = null) + private fun internalInfo(upload: MuxUpload) = MuxUpload::class.java.getDeclaredField("uploadInfo") + .apply { isAccessible = true }.get(upload) as UploadInfo + private fun pump(condition: () -> Boolean) = runBlocking { + withTimeout(5000) { + while (!condition()) { dispatcher.scheduler.runCurrent(); delay(1) } + dispatcher.scheduler.runCurrent() + } + } +} diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/UploadPreparationTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/UploadPreparationTests.kt new file mode 100644 index 00000000..c9adf9af --- /dev/null +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/UploadPreparationTests.kt @@ -0,0 +1,132 @@ +package com.mux.video.upload.internal.standardization + +import android.net.Uri +import com.mux.exoplayeradapter.AbsRobolectricTest +import com.mux.video.upload.internal.InputStandardization +import com.mux.video.upload.internal.UploadInfo +import com.mux.video.upload.internal.UploadAttempt +import com.mux.video.upload.internal.UploadPreparationState +import com.mux.video.upload.api.MuxUpload +import io.mockk.* +import kotlinx.coroutines.* +import org.junit.Assert.* +import org.junit.Test +import org.robolectric.RuntimeEnvironment +import org.robolectric.annotation.Config +import java.io.File + +@Config(sdk = [28]) +class UploadPreparationTests : AbsRobolectricTest() { + private val context get() = RuntimeEnvironment.getApplication() + private val timeline = StandardInputTimelineFacts(durationSeconds = known(1.0), + audioVideoStartOffset = known(AudioVideoStartOffset.NotApplicable), + videoPresentationSeconds = known((0 until 30).map { it / 30.0 }), videoDurationSeconds = known(1.0)) + private fun source(codec: VideoCodec = VideoCodec.H264) = compliantFacts(codec).copy( + audioTracks = known(emptyList()), cadence = known(Cadence.Constant)) + private fun metadata(facts: MediaFacts) = MediaMetadataInspection(known(ContainerKind.IsoBaseMedia), + emptyList(), MediaFact.Unknown, facts, 0) + private fun upload() = UploadInfo(remoteUri = Uri.parse("https://example.invalid/upload"), + inputFile = File("unused"), chunkSize = 4, retriesPerChunk = 3, optOut = true, uploadJob = null, statusFlow = null) + + @Test fun disabledPreparationDoesNoInspectionOrExport() = runBlocking { + val preparation = UploadPreparation(inspectMetadata = { error("Must not inspect") }, + convert = { _, _, _, _, _ -> error("Must not export") }) + assertEquals(PreparedUpload.Original(), preparation.prepare(upload().copy( + inputStandardization = InputStandardization(false)), context)) + } + + @Test fun compliantH264AndHevcSelectOriginalWithoutExport() = runBlocking { + for (codec in listOf(VideoCodec.H264, VideoCodec.Hevc)) { + val facts = source(codec) + val preparation = UploadPreparation(inspectMetadata = { MetadataInspectionResult.Success(metadata(facts)) }, + inspectSamples = { _, _, _ -> MediaSampleInspection(facts, SampleScanStatus.Complete, timeline = timeline) }, + convert = { _, _, _, _, _ -> error("Compliant input must pass through") }) + assertEquals(PreparedUpload.Original(), preparation.prepare(upload(), context)) + } + } + + @Test fun unverifiedGeneratedResumeGatesConversionBeforeExportForBothFamilies() = runBlocking { + for (codec in listOf(VideoCodec.H264, VideoCodec.Hevc)) { + val facts = source(codec).copy(averageBitrate = known(9_000_000L)) + val preparation = UploadPreparation(inspectMetadata = { MetadataInspectionResult.Success(metadata(facts)) }, + inspectSamples = { _, _, _ -> MediaSampleInspection(facts, SampleScanStatus.Complete, timeline = timeline) }, + convert = { _, _, _, _, _ -> error("Unverified generated resume must select original") }) + assertEquals(PreparedUpload.Original(PreparationDiagnostic.GeneratedResumeUnavailable), + preparation.prepare(upload(), context)) + } + } + + @Test fun adapterReceivesPlannedCodecAndOnlyValidatedCompletionCanSelectGenerated() = runBlocking { + for (codec in listOf(VideoCodec.H264, VideoCodec.Hevc)) { + val facts = source(codec).copy(averageBitrate = known(9_000_000L)) + var result: SdrConversionResult = SdrConversionResult.Failed(SdrConversionFailure.OutputInvalid) + val preparation = UploadPreparation(inspectMetadata = { MetadataInspectionResult.Success(metadata(facts)) }, + inspectSamples = { _, _, _ -> MediaSampleInspection(facts, SampleScanStatus.Complete, timeline = timeline) }, + convert = { _, _, _, _, plan -> + assertEquals(codec, plan.outputCodec) + assertFalse(plan.toneMapsToSdr) + result + }) + assertEquals(PreparedUpload.Original(PreparationDiagnostic.ConversionFailed(SdrConversionFailure.OutputInvalid)), + preparation.prepare(upload(), context, generatedResumeVerified = true)) + val owned = SdrGeneratedFile.allocate(context.cacheDir, 1)!! + try { + result = SdrConversionResult.Completed(owned, source(codec)) + assertEquals(PreparedUpload.Generated(owned), preparation.prepare(upload(), context, generatedResumeVerified = true)) + } finally { owned.delete() } + } + } + + @Test fun unsupportedHdrAndUnknownColorStayOutsideSdrExport() = runBlocking { + for (range in listOf(MediaFact.Unknown, known(DynamicRange.Hlg), known(DynamicRange.Pq), known(DynamicRange.DolbyVision))) { + val facts = source(VideoCodec.Hevc).copy(dynamicRange = range, averageBitrate = known(9_000_000L)) + val preparation = UploadPreparation(inspectMetadata = { MetadataInspectionResult.Success(metadata(facts)) }, + inspectSamples = { _, _, _ -> MediaSampleInspection(facts, SampleScanStatus.Complete, timeline = timeline) }, + convert = { _, _, _, _, _ -> error("Unproven HDR must stay outside SDR export") }) + assertTrue(preparation.prepare(upload(), context, generatedResumeVerified = true) is PreparedUpload.Original) + } + } + + @Test fun inspectionFailuresAndMissingTimelineSelectOriginalWithoutRawDiagnosticText() = runBlocking { + val failed = UploadPreparation(inspectMetadata = { throw NoSuchMethodError("https://secret.invalid/path") }) + assertEquals(PreparedUpload.Original(PreparationDiagnostic.InspectionFailed), failed.prepare(upload(), context)) + val facts = source().copy(averageBitrate = known(9_000_000L)) + val incomplete = UploadPreparation(inspectMetadata = { MetadataInspectionResult.Success(metadata(facts)) }, + inspectSamples = { _, _, _ -> MediaSampleInspection(facts, SampleScanStatus.LimitExceeded) }, + convert = { _, _, _, _, _ -> error("Incomplete inspection must not export") }) + assertEquals(PreparedUpload.Original(PreparationDiagnostic.UnsupportedPlan), incomplete.prepare(upload(), context, true)) + } + + @Test fun adapterCompletionRacingPauseTransfersVerifiedOwnershipBeforeCoroutineDispatch() = bridgeRace(cancel = false) + @Test fun adapterCompletionRacingCancelDeletesOnlyItsOwnedFile() = bridgeRace(cancel = true) + + private fun bridgeRace(cancel: Boolean) = runBlocking { + val facts = source().copy(averageBitrate = known(9_000_000L)) + val callback = slot<(SdrConversionResult) -> Unit>() + val started = CompletableDeferred() + val export = mockk { every { cancel() } just Runs } + mockkConstructor(SdrConversionAdapter::class) + every { anyConstructed().start(any(), any(), any(), any(), capture(callback)) } answers { + started.complete(Unit); export + } + val owned = SdrGeneratedFile.allocate(context.cacheDir, 1)!! + owned.file.writeText("validated output"); assertTrue(owned.recordValidation()) + val state = UploadPreparationState() + val attempt = UploadAttempt(state, MuxUpload.Progress()) + try { + val preparation = UploadPreparation(inspectMetadata = { MetadataInspectionResult.Success(metadata(facts)) }, + inspectSamples = { _, _, _ -> MediaSampleInspection(facts, SampleScanStatus.Complete, timeline = timeline) }) + val job = launch(Dispatchers.Default) { preparation.prepare(upload().copy(attempt = attempt), context, true) } + withTimeout(5000) { started.await() } + if (cancel) attempt.cancel {} else attempt.pause {} + job.cancelAndJoin() + callback.captured(SdrConversionResult.Completed(owned, facts)) + verify(exactly = 1) { export.cancel() } + assertEquals(!cancel, owned.file.exists()) + if (!cancel) assertSame(owned, state.verified) + } finally { + owned.delete() + unmockkConstructor(SdrConversionAdapter::class) + } + } +} From 3de7171deee385ad1553e7bb2b5c86c7df759f45 Mon Sep 17 00:00:00 2001 From: Eugene Morozov Date: Wed, 7 Oct 2026 12:34:09 -0700 Subject: [PATCH 25/49] NAT-547: cover terminal races and document adapter evidence --- .../upload/internal/standardization/README.md | 36 ++++ .../SdrConversionAdapterTests.kt | 193 ++++++++++++++++++ 2 files changed, 229 insertions(+) create mode 100644 library/src/main/java/com/mux/video/upload/internal/standardization/README.md create mode 100644 library/src/test/java/com/mux/video/upload/internal/standardization/SdrConversionAdapterTests.kt diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/README.md b/library/src/main/java/com/mux/video/upload/internal/standardization/README.md new file mode 100644 index 00000000..5585b973 --- /dev/null +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/README.md @@ -0,0 +1,36 @@ +# SDR adapter (NAT-547) + +`SdrConversionAdapter.start` accepts the original file, its metadata and complete shared sample/timeline inspection, and the planner's `StandardInputConversion`. These must describe the same unchanged source. It exports on a dedicated `HandlerThread`, validates the generated file with `StandardInputOutputValidator`, and invokes one internal terminal callback on that thread. It does not start transport or emit public upload events. + +- `Completed` transfers an explicitly owned `SdrGeneratedFile` and validated facts to orchestration. Keep it while upload/resume needs it; call its idempotent `delete()` at the terminal lifecycle point. +- `Failed` permits original-file fallback. No partial or rejected output is selected. +- `Cancelled` stops preparation even without a Transformer callback. Orchestration must suppress public results and must not upload the original as a cancellation fallback. +- If deletion fails, `Attempt.pendingCleanup` retains the exact deletion authority for retry. Nothing sweeps `cacheDir/mux-upload`, where customer source copies can live. + +Each attempt allocates only its tracked unique `sdr-*.mp4` file under `cacheDir/mux-upload/standard-input`, after a size-based disk guard. It deletes that file on failure/cancellation, releases Transformer, and quits its thread. A successful file outlives that thread. Persisted payload identity, abandoned-file recovery, and generated resume belong to NAT-549; upload/pause orchestration belongs to NAT-548. + +## Encoding contract + +H.264 stays H.264, HEVC stays HEVC, and another proven SDR source uses H.264. Dimensions are the planner's exact display-oriented even dimensions, with orientation baked into pixels and no upscaling. Portrait encoding is explicitly enabled. Video always encodes, including an identity presentation effect, so bitrate/GOP remediation cannot silently remux the source. + +The 8-bit 4:2:0 path requests H.264 Baseline (no B frames) or HEVC Main, VBR at 6 Mbps through a 2048-pixel long side or 15 Mbps above it, additionally capped to 75% of the plan's average-bitrate ceiling. GOP targets use 90% of the planned interval ceiling, rounded down to whole seconds, following Swift's headroom convention. Output validation remains the final bitrate, IDR/closed-GOP, profile, pixel, cadence, and timeline gate. + +Media3's default video factory overrides H.264 profile requests. The adapter therefore supplies Media3 `DefaultCodec` with the exact platform profile, level, dimensions, bitrate, nominal rate, surface input, and GOP interval. Transformer owns decoding, effects, encoding, and muxing. The AAC delegate disables both encoder and format fallback. Every `onFallbackApplied`, mismatched output MIME, or invalid generated file fails the attempt; a listed encoder never proves a usable path. + +The bounded advertised-format preflight examines at most 128 codecs and checks a 500 ms budget between platform calls. Native calls cannot be preempted. It checks source decode and exact output encode support before creating a file, and pins one supported video encoder. It does not turn advertised capabilities into the planner's device-path proof. + +Preserved cadence keeps source timestamps. Constant-rate reduction is limited to proven constant cadence with an integral source/30 fps ratio; a one-part-per-million allowance covers microsecond-rounded rate measurement. Media3 drops frames at 30 fps and the shared validator checks actual output cadence and duration. Frame duplication, fractional-rate resampling, 10-bit SDR encoding, and HDR/tone mapping have no proof in this adapter and fail conservatively. NAT-550 owns tone mapping. + +## Audio decision + +[Shipped Swift v1.2.0](https://github.com/muxinc/swift-upload-sdk/blob/faa440e6ea9339315a9d192ca314c4ef5a8668c9/Sources/MuxUploadSDK/InputStandardization/UploadInputStandardizationWorker.swift#L794) copies AAC and otherwise preserves sample rate, channel count, and available layout while encoding AAC (160 kbps for mono/stereo; 384 kbps for surround). Following that behavior and the Android TDD's exclusion of new surround conversion, this adapter copies known compliant AAC mono/stereo/5.1, converts known non-AAC mono/stereo to AAC-LC at 160 kbps with the same rate/count, and rejects non-AAC surround or unknown layouts. It adds no downmix policy. Output channel layout and reported export sample rate/count must match these targets. + +The asset loader's selector matches the first audio track's container ID, rather than Media3's preferred/default track. Missing identity fails; it never substitutes a secondary track. + +## Evidence and remaining limits + +At implementation time, 210 unit tests pass in each debug/release variant; release SDK/sample builds and lint-vital pass. External API 36 emulator checks cover six accepted exports: H.264 resize, silent H.264, rotated portrait, MP3 stereo to AAC, short HEVC Main, and 240-to-30 fps H.264. Two HEVC failures exercise safe rejection: 720p exceeds the emulator's 512-pixel encoder limit; a longer small export emits CRA/open GOP and fails the existing validator. Cancellation removes generated bytes and preserves source copies. These are nine cases across three instrumentation tests, including the grouped export matrix. + +The distinguishable multi-audio source has 3s video, 3s first stereo AAC (track ID 2, non-default), and 6s secondary mono AAC (ID 3, default). Output contains the first stereo track: its copied AAC bytes have the same SHA-256 as source audio ordinal zero and differ from the default secondary. This proves local selection only; Mux ingest parity remains pending. + +Media binaries, fixture harness, output probes, and measurements stay outside product Git in `/private/tmp/nat547-sdr`. API 23-29 execution, physical hardware, long HEVC closed-GOP support, 10-bit SDR, HDR, high-resolution device coverage, visual quality/performance acceptance, and a live Mux upload are unverified. Emulator timings do not establish physical-device performance limits. NAT-548 can use the validated SDR path for the early upload journey; no predecessor merge is required to continue the stack. diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/SdrConversionAdapterTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/SdrConversionAdapterTests.kt new file mode 100644 index 00000000..c6a0d507 --- /dev/null +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/SdrConversionAdapterTests.kt @@ -0,0 +1,193 @@ +package com.mux.video.upload.internal.standardization + +import android.content.Context +import androidx.test.core.app.ApplicationProvider +import org.junit.Assert.* +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.Shadows.shadowOf +import org.robolectric.annotation.Config +import org.robolectric.annotation.LooperMode +import java.io.File + +@RunWith(RobolectricTestRunner::class) +@Config(manifest = Config.NONE, sdk = [28]) +@LooperMode(LooperMode.Mode.PAUSED) +class SdrConversionAdapterTests { + private val context = ApplicationProvider.getApplicationContext() + private val facts = compliantFacts().copy(cadence = known(Cadence.Constant), videoProfile = known(VideoProfile.H264Baseline)) + private val conversion = (StandardInputPlanner().plan(facts.copy(averageBitrate = known(9_000_000L)), + options(), fullCapabilities()).action as StandardInputAction.Convert).conversion + private val timeline = StandardInputTimelineFacts(known(3.0), known(AudioVideoStartOffset.Seconds(0.0)), + known(listOf(0.0, 1.0, 2.0)), known(3.0), known(3.0), known(90_000L)) + private fun audio(channels: Int = 2, format: MediaFact = known(AudioFormat.Aac(AudioChannelLayout.Stereo)), + id: Int = 2, index: Int = 1) = TrackMetadata(index, known(index), known(id), TrackKind.Audio, + known(if (format.valueOrNull is AudioFormat.Aac) "audio/mp4a-latm" else "audio/mpeg"), known(3.0), + MediaFact.Unknown, MediaFact.Unknown, audio = AudioTrackMetadata(format, known(channels), known(48_000), known(0), known(0))) + private fun metadata(tracks: List = listOf(audio())) = + MediaMetadataInspection(known(ContainerKind.IsoBaseMedia), tracks, MediaFact.Unknown, facts, 0) + private fun samples(input: MediaFacts = facts) = MediaSampleInspection(input, SampleScanStatus.Complete, timeline = timeline) + private fun targets(plan: StandardInputConversion = conversion, input: MediaFacts = facts, + tracks: List = listOf(audio())) = SdrEncodingTargets.from(plan, metadata(tracks), samples(input)) + + @Test fun pinsCodecFamilyAndTargetsBelowPublishedCeilings() { + assertEquals("video/avc", targets()!!.videoMime) + assertEquals(6_000_000, targets()!!.bitrate) + assertEquals(18, targets()!!.keyframeIntervalSeconds) + assertEquals(VideoProfile.H264Baseline, targets()!!.profile) + val hevc = facts.copy(videoCodec = known(VideoCodec.Hevc)) + assertEquals("video/hevc", targets(conversion.copy(sourceCodec = VideoCodec.Hevc, outputCodec = VideoCodec.Hevc), hevc)!!.videoMime) + assertNull(targets(conversion.copy(sourceCodec = VideoCodec.Hevc), hevc)) + val other = facts.copy(videoCodec = known(VideoCodec.Other)) + assertEquals("video/avc", targets(conversion.copy(sourceCodec = VideoCodec.Other), other)!!.videoMime) + } + @Test fun rejectsHdrUnknownTimelineUpscalingAndUnprovenTenBitPath() { + assertNull(targets(conversion.copy(toneMapsToSdr = true))) + assertNull(targets(input = facts.copy(dynamicRange = MediaFact.Unknown))) + assertNull(targets(conversion.copy(outputDimensions = Dimensions(1922, 1080)))) + assertNull(targets(conversion.copy(outputPixelFormat = PixelFormat(10, ChromaSubsampling.Yuv420)))) + assertNull(SdrEncodingTargets.from(conversion, metadata(), samples().copy(timeline = StandardInputTimelineFacts()))) + } + @Test fun preservesMonoStereoForNonAacWithoutInventingSurroundDownmix() { + for (channels in listOf(1, 2)) { + val nonAac = facts.copy(audioTracks = known(listOf(AudioTrack(known(AudioFormat.OtherCodec))))) + val target = targets(input = nonAac, tracks = listOf(audio(channels, known(AudioFormat.OtherCodec))))!! + assertEquals(channels, target.audioChannels) + assertFalse(target.copyAac) + } + assertNull(targets(tracks = listOf(audio(6, known(AudioFormat.OtherCodec))))) + assertNull(targets(tracks = listOf(audio(2, MediaFact.Unknown)))) + assertTrue(targets(tracks = listOf(audio(6, known(AudioFormat.Aac(AudioChannelLayout.FivePointOne)))))!!.copyAac) + } + @Test fun choosesContainerFirstTrackEvenWhenExtractorOrderDiffers() { + val multi = facts.copy(audioTracks = known(listOf(AudioTrack(), AudioTrack()))) + assertEquals("42", targets(input = multi, tracks = listOf(audio(id = 99, index = 3), audio(id = 42, index = 1)))!!.firstAudioTrackId) + assertNull(targets(tracks = listOf(audio().copy(sourceTrackId = MediaFact.Unknown)))) + } + @Test fun permitsOnlyProvenIntegralCadenceReductionAndKeepsVfrTimestamps() { + assertNotNull(targets(input = facts.copy(cadence = known(Cadence.Variable)))) + val cfr = conversion.copy(outputCadence = OutputCadence.ConstantFrameRate) + assertNotNull(targets(cfr, facts.copy(frameRate = known(240.0)))) + assertNotNull(targets(cfr, facts.copy(frameRate = known(240.00004008351402)))) + assertNull(targets(cfr, facts.copy(frameRate = known(240.0), cadence = known(Cadence.Variable)))) + assertNull(targets(cfr, facts.copy(frameRate = known(2.0)))) + assertNull(targets(cfr, facts.copy(frameRate = known(239.76)))) + } + + private class FakeEngine : SdrExportEngine { + var output: File? = null + var completions: ((String?, String?) -> Unit)? = null + var failures: ((SdrConversionFailure) -> Unit)? = null + var starts = 0 + var cancellations = 0 + override fun start(input: File, output: File, completed: (String?, String?) -> Unit, failed: (SdrConversionFailure) -> Unit) { + starts++; this.output = output; completions = completed; failures = failed + output.writeText("owned partial bytes") + } + override fun cancel() { cancellations++ } + fun complete(video: String? = "video/avc", audio: String? = "audio/mp4a-latm") { completions!!(video, audio) } + } + private inner class Run( + validation: StandardInputOutputValidation = StandardInputOutputValidation.Accepted(facts, + StandardInputPolicyEvaluator().evaluate(facts, selection())), + capability: SdrEncoderCapability? = SdrEncoderCapability("fake", 1), + duringValidation: (() -> Unit)? = null, + ) { + val sourceFile = File.createTempFile("customer-", ".mp4", context.cacheDir).apply { writeText("original") } + val unrelated = File(context.cacheDir, "mux-upload/customer-copy.mp4").apply { parentFile!!.mkdirs(); writeText("customer") } + val engine = FakeEngine() + val results = mutableListOf() + val adapter = SdrConversionAdapter(context, preflight = { _, _, _, _ -> capability }, + validate = { _, _, _, _, _ -> duringValidation?.invoke(); validation }, createEngine = { _, _, _, _, _ -> engine }) + val attempt = adapter.start(sourceFile, metadata(), samples(), conversion, results::add) + fun idle() { shadowOf(attempt.looper).idle() } + fun assertInputSafe() { assertEquals("original", sourceFile.readText()); assertEquals("customer", unrelated.readText()) } + } + @Test fun successRetainsVerifiedFileUntilOwnerDeletesItAndIgnoresLateEvents() { + val run = Run(); run.idle(); run.engine.complete(); run.idle() + val result = run.results.single() as SdrConversionResult.Completed + assertTrue(result.output.file.exists()); run.assertInputSafe() + run.engine.failures!!(SdrConversionFailure.Export); run.attempt.cancel() + assertEquals(1, run.results.size) + assertTrue(result.output.delete()); assertTrue(result.output.delete()) + assertEquals(1, run.engine.cancellations) + } + @Test fun exportFailureDeletesPartialAndPublishesExactlyOnce() { + val run = Run(); run.idle(); run.engine.failures!!(SdrConversionFailure.Export); run.idle() + assertEquals(listOf(SdrConversionResult.Failed(SdrConversionFailure.Export)), run.results) + assertFalse(run.engine.output!!.exists()); run.assertInputSafe() + run.engine.complete(); assertEquals(1, run.results.size) + } + @Test fun cancelTerminatesWithoutMedia3CallbackAndSuppressesQueuedCompletion() { + val run = Run(); run.idle(); run.engine.complete(); run.attempt.cancel(); run.idle() + assertEquals(listOf(SdrConversionResult.Cancelled), run.results) + assertFalse(run.engine.output!!.exists()); run.assertInputSafe() + } + @Test fun cancelBeforeStartAllocatesNothing() { + val run = Run(); run.attempt.cancel(); run.idle() + assertEquals(listOf(SdrConversionResult.Cancelled), run.results) + assertEquals(0, run.engine.starts); run.assertInputSafe() + } + @Test fun forbiddenFallbackWinsOverLaterCompletion() { + val run = Run(); run.idle(); run.engine.failures!!(SdrConversionFailure.ForbiddenFallback); run.engine.complete(); run.idle() + assertEquals(listOf(SdrConversionResult.Failed(SdrConversionFailure.ForbiddenFallback)), run.results) + assertFalse(run.engine.output!!.exists()) + } + @Test fun codecChangeAndInvalidOutputCannotProduceCompletedResult() { + val changed = Run(); changed.idle(); changed.engine.complete("video/hevc"); changed.idle() + assertEquals(SdrConversionResult.Failed(SdrConversionFailure.CodecChanged), changed.results.single()) + assertFalse(changed.engine.output!!.exists()) + val invalid = Run(StandardInputOutputValidation.Rejected(OutputRejection.EmptyOrUnreadable)) + invalid.idle(); invalid.engine.complete(); invalid.idle() + assertEquals(SdrConversionResult.Failed(SdrConversionFailure.OutputInvalid), invalid.results.single()) + assertFalse(invalid.engine.output!!.exists()) + } + @Test fun unavailableCapabilityDoesNotStartOrAllocate() { + val run = Run(capability = null); run.idle() + assertEquals(SdrConversionResult.Failed(SdrConversionFailure.CapabilityUnavailable), run.results.single()) + assertEquals(0, run.engine.starts); run.assertInputSafe() + } + @Test fun sourceChangeDuringValidationRejectsGeneratedBytes() { + lateinit var run: Run + run = Run(duringValidation = { run.sourceFile.appendText("changed") }) + run.idle(); run.engine.complete(); run.idle() + assertEquals(SdrConversionResult.Failed(SdrConversionFailure.SourceChanged), run.results.single()) + assertFalse(run.engine.output!!.exists()) + } + @Test fun cancellationDuringValidationWinsOverAcceptedOutput() { + lateinit var run: Run + run = Run(duringValidation = { run.attempt.cancel() }) + run.idle(); run.engine.complete(); run.idle() + assertEquals(SdrConversionResult.Cancelled, run.results.single()) + assertFalse(run.engine.output!!.exists()); run.assertInputSafe() + } + @Test fun wrongProfileOrAudioLayoutRejectsEvenAnAcceptedSharedVerdict() { + for (wrong in listOf(facts.copy(videoProfile = known(VideoProfile.H264High)), + facts.copy(audioTracks = known(listOf(AudioTrack(known(AudioFormat.Aac(AudioChannelLayout.Mono)))))))) { + val run = Run(StandardInputOutputValidation.Accepted(wrong, StandardInputPolicyEvaluator().evaluate(wrong, selection()))) + run.idle(); run.engine.complete(); run.idle() + assertEquals(SdrConversionResult.Failed(SdrConversionFailure.OutputInvalid), run.results.single()) + assertFalse(run.engine.output!!.exists()) + } + } + @Test fun failedDeletionRetainsExactCleanupAuthorityAndCannotSweepCustomerParent() { + val run = Run(); run.idle() + val path = run.engine.output!! + assertTrue(path.delete()); assertTrue(path.mkdir()) + val obstacle = File(path, "obstacle").apply { writeText("locked") } + run.attempt.cancel(); run.idle() + assertEquals(SdrConversionResult.Cancelled, run.results.single()) + assertEquals(path, run.attempt.pendingCleanup!!.file) + run.assertInputSafe() + assertTrue(obstacle.delete()); assertTrue(run.attempt.pendingCleanup!!.delete()) + } + @Test fun diskEstimateIsBoundedAndOwnershipCannotDeleteOtherFiles() { + assertNull(targets()!!.estimatedOutputBytes(Double.POSITIVE_INFINITY)) + assertNull(SdrGeneratedFile.allocate(context.cacheDir, Long.MAX_VALUE)) + val customer = File(context.cacheDir, "mux-upload/source.mp4").apply { parentFile!!.mkdirs(); writeText("customer") } + val generated = SdrGeneratedFile.allocate(context.cacheDir, 1)!! + assertEquals("standard-input", generated.file.parentFile!!.name) + assertTrue(generated.delete()); assertTrue(customer.exists()) + } +} From 064c2e14a9e8fa9cb00868f087dc9178453c5354 Mon Sep 17 00:00:00 2001 From: Eugene Morozov Date: Thu, 8 Oct 2026 11:17:23 -0700 Subject: [PATCH 26/49] Preserve server audio selection for multi-track inputs --- .../standardization/SdrEncodingTargets.kt | 2 ++ .../StandardInputOutputValidator.kt | 2 ++ .../standardization/StandardInputPlanner.kt | 6 +++++ .../UploadPreparationLifecycleTests.kt | 19 +++++++++++++++ .../SdrConversionAdapterTests.kt | 15 ++++++++++-- .../StandardInputOutputValidatorTests.kt | 7 ++++++ .../StandardInputPlannerTests.kt | 23 ++++++++++--------- .../StandardInputTimelineInspectorTests.kt | 11 ++++++--- .../standardization/UploadPreparationTests.kt | 12 ++++++++++ 9 files changed, 81 insertions(+), 16 deletions(-) diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/SdrEncodingTargets.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/SdrEncodingTargets.kt index 4ba96c57..97aae14c 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/SdrEncodingTargets.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/SdrEncodingTargets.kt @@ -70,6 +70,8 @@ internal data class SdrEncodingTargets( } else if (sourceRate != rate || facts.cadence == MediaFact.Unknown) return null val audioFacts = facts.audioTracks.valueOrNull ?: return null val audio = metadata.tracks.filter { it.kind == TrackKind.Audio }.sortedBy { it.containerIndex.valueOrNull ?: Int.MAX_VALUE } + // Direct adapter callers must also preserve Mux's audio choice for multi-track inputs. + if (audioFacts.size > 1 || audio.size > 1) return null if (audio.size != audioFacts.size || audio.any { it.containerIndex == MediaFact.Unknown }) return null if ((conversion.outputAudio == OutputAudio.None) != audio.isEmpty() && conversion.outputAudio != OutputAudio.AacFromFirstTrackIfPresent) return null diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputOutputValidator.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputOutputValidator.kt index 9a67dfb9..00d8b171 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputOutputValidator.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputOutputValidator.kt @@ -89,6 +89,8 @@ internal class StandardInputOutputValidator( val sourceAudio = source.audioTracks.valueOrNull val outputAudio = output.audioTracks.valueOrNull + // Output format/timing cannot prove that a removed track was not Mux's ingest choice. + if ((sourceAudio?.size ?: 0) > 1) missing.add(OutputExpectation.Audio) val wantsAudio = when (conversion.outputAudio) { OutputAudio.None -> false OutputAudio.AacFromFirstTrack -> true diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputPlanner.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputPlanner.kt index bd71c0bb..3d2d90fe 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputPlanner.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/StandardInputPlanner.kt @@ -96,6 +96,7 @@ internal sealed interface OriginalReason { internal sealed interface FallbackReason { data object InsufficientEvidence : FallbackReason data object UnsafeVideoTrackStructure : FallbackReason + data object UnverifiedAudioSelection : FallbackReason data class UnsupportedConversion( val conversion: StandardInputConversion, val missingCapabilities: Set, @@ -169,6 +170,11 @@ internal class StandardInputPlanner( if (evaluation.outcome == PolicyStatus.Compliant) OriginalReason.StandardInput else OriginalReason.NoKnownStandardInputViolation) + // The first track can differ from the primary/default track Mux ingests. + // Preserve all original bytes so server audio selection is unchanged. + if ((facts.audioTracks.valueOrNull?.size ?: 0) > 1) + return fallback(FallbackReason.UnverifiedAudioSelection) + val conversion = buildConversion(facts, selection, toneMap) ?: return fallback(FallbackReason.InsufficientEvidence) val missingCapabilities = capabilities.missingCapabilities(conversion) diff --git a/library/src/test/java/com/mux/video/upload/internal/UploadPreparationLifecycleTests.kt b/library/src/test/java/com/mux/video/upload/internal/UploadPreparationLifecycleTests.kt index 42f6ba78..ccbac73c 100644 --- a/library/src/test/java/com/mux/video/upload/internal/UploadPreparationLifecycleTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/UploadPreparationLifecycleTests.kt @@ -8,6 +8,7 @@ import com.mux.video.upload.api.MuxUploadManager import com.mux.video.upload.api.UploadStatus import com.mux.video.upload.internal.standardization.PreparedUpload import com.mux.video.upload.internal.standardization.SdrGeneratedFile +import com.mux.video.upload.internal.standardization.* import io.mockk.* import kotlinx.coroutines.* import kotlinx.coroutines.flow.MutableSharedFlow @@ -84,6 +85,24 @@ class UploadPreparationLifecycleTests : AbsRobolectricTest() { dispatcher.scheduler.runCurrent() assertEquals(1, lateResults) } + @Test fun multiAudioFallbackUploadsOriginalWithOneSuccessfulResultEvenAfterResumeGateIsVerified() { + val facts = compliantFacts().copy(averageBitrate = known(9_000_000L), audioTracks = known(listOf( + AudioTrack(known(AudioFormat.Aac(AudioChannelLayout.Stereo))), + AudioTrack(known(AudioFormat.Aac(AudioChannelLayout.Mono)))))) + val metadata = MediaMetadataInspection(known(ContainerKind.IsoBaseMedia), emptyList(), MediaFact.Unknown, facts, 0) + val preparation = UploadPreparation(inspectMetadata = { MetadataInspectionResult.Success(metadata) }, + inspectSamples = { _, _, _ -> MediaSampleInspection(facts, SampleScanStatus.Complete) }, + convert = { _, _, _, _, _ -> error("Multi-audio export must never select a replacement track") }) + prepare = { preparation.prepare(it, context, generatedResumeVerified = true) } + val upload = observe(MuxUpload.create(info())) + upload.start() + pump { upload.isSuccessful } + assertArrayEquals(source.readBytes(), chunks.flatMap { it.sliceData.toList() }.toByteArray()) + assertEquals(16L, upload.currentProgress.totalBytes) + assertEquals(1, results.size) + assertTrue(results.single().isSuccess) + assertTrue(source.exists()); assertTrue(sibling.exists()) + } @Test fun automaticLegacyRestorationUsesOriginalOffsetWithoutPreparation() { writeUploadState(info(), MuxUpload.Progress(bytesUploaded = 8, totalBytes = 16)) diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/SdrConversionAdapterTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/SdrConversionAdapterTests.kt index 0c3b350a..cbdf6ef9 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/SdrConversionAdapterTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/SdrConversionAdapterTests.kt @@ -137,9 +137,12 @@ class SdrConversionAdapterTests { assertNull(targets(tracks = listOf(audio(2, MediaFact.Unknown)))) assertTrue(targets(tracks = listOf(audio(6, known(AudioFormat.Aac(AudioChannelLayout.FivePointOne)))))!!.copyAac) } - @Test fun choosesContainerFirstTrackEvenWhenExtractorOrderDiffers() { + @Test fun multipleAudioTracksCannotCreateAnEncoderTarget() { val multi = facts.copy(audioTracks = known(listOf(AudioTrack(), AudioTrack()))) - assertEquals("42", targets(input = multi, tracks = listOf(audio(id = 99, index = 3), audio(id = 42, index = 1)))!!.firstAudioTrackId) + assertNull(targets(input = multi, tracks = listOf(audio(id = 99, index = 3), audio(id = 42, index = 1)))) + assertNull(targets(input = multi)) + assertNull(targets(tracks = listOf(audio(id = 99, index = 3), audio(id = 42, index = 1)))) + assertEquals("42", targets(tracks = listOf(audio(id = 42, index = 1)))!!.firstAudioTrackId) assertNull(targets(tracks = listOf(audio().copy(sourceTrackId = MediaFact.Unknown)))) } @Test fun permitsOnlyProvenIntegralCadenceReductionAndKeepsVfrTimestamps() { @@ -206,6 +209,14 @@ class SdrConversionAdapterTests { assertTrue(result.output.delete()); assertTrue(result.output.delete()) assertEquals(1, run.engine.cancellations) } + @Test fun directMultiAudioExportStopsBeforeCreatingEngineOrOutput() { + val run = Run(inputMetadata = metadata(listOf(audio(id = 2), audio(id = 3, index = 2)))) + run.idle() + assertEquals(listOf(SdrConversionResult.Failed(SdrConversionFailure.UnsupportedPlan)), run.results) + assertEquals(0, run.engine.starts) + assertNull(run.engine.output) + run.assertInputSafe() + } @Test fun exportFailureDeletesPartialAndPublishesExactlyOnce() { val run = Run(); run.idle(); run.engine.failures!!(SdrConversionFailure.Export); run.idle() assertEquals(listOf(SdrConversionResult.Failed(SdrConversionFailure.Export)), run.results) diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputOutputValidatorTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputOutputValidatorTests.kt index c7144e06..ee58296c 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputOutputValidatorTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputOutputValidatorTests.kt @@ -78,6 +78,13 @@ class StandardInputOutputValidatorTests { assertTrue(validate(output.copy(frameRate = known(5.0)), outputTime = timeline(3.2), conversion = low) is StandardInputOutputValidation.Accepted) mismatch(OutputExpectation.Duration, validate(output.copy(frameRate = known(5.0)), outputTime = timeline(3.200001), conversion = low)) } + @Test fun matchingFirstTrackFormatDoesNotProveMuxAudioSelection() { + val multi = source.copy(audioTracks = known(listOf( + AudioTrack(known(AudioFormat.Aac(AudioChannelLayout.Stereo))), + AudioTrack(known(AudioFormat.Aac(AudioChannelLayout.Mono)))))) + val rejection = reason(validate(input = multi)) as OutputRejection.InsufficientPlanEvidence + assertTrue(rejection.expectations.contains(OutputExpectation.Audio)) + } @Test fun avStartToleranceIs50msAndAbsenceIsExplicit() { assertTrue(validate(outputTime = timeline(offset = AudioVideoStartOffset.Seconds(-0.05))) is StandardInputOutputValidation.Accepted) mismatch(OutputExpectation.AudioVideoStartOffset, validate(outputTime = timeline(offset = AudioVideoStartOffset.Seconds(0.050001)))) diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputPlannerTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputPlannerTests.kt index 55e97c73..f1be4d4c 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputPlannerTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputPlannerTests.kt @@ -84,14 +84,17 @@ class StandardInputPlannerTests { assertEquals(setOf(PolicyRequirement.Audio), planner.plan(facts).evaluation.unknownRequirements) } - @Test fun anotherViolationWithMultipleAudioTracksSelectsFirstTrack() { - val facts = compliantFacts().copy(frameRate = known(121.0), - audioTracks = known(listOf(AudioTrack(known(AudioFormat.OtherCodec)), AudioTrack()))) - val conversion = conversion(facts) - assertEquals(OutputAudio.AacFromFirstTrack, conversion.outputAudio) - assertEquals(setOf(PolicyRequirement.FrameRate), conversion.requirementsToRemediate) - assertEquals(StandardInputAction.UploadOriginal(OriginalReason.NoKnownStandardInputViolation), - planner.plan(facts.copy(frameRate = known(30.0))).action) + @Test fun conversionWithMultipleAudioTracksFallsBackEvenWithFullEncoderCapabilities() { + for (codec in listOf(VideoCodec.H264, VideoCodec.Hevc)) { + val facts = compliantFacts(codec).copy( + audioTracks = known(listOf(AudioTrack(known(AudioFormat.OtherCodec)), AudioTrack()))) + for (source in listOf(facts.copy(frameRate = known(121.0)), + facts.copy(averageBitrate = known(9_000_000L)), facts.copy(displayDimensions = known(Dimensions(2560, 1440))))) { + assertEquals(FallbackReason.UnverifiedAudioSelection, fallback(source)) + assertTrue(planner.plan(source, capabilities = fullCapabilities()).evaluation.nonCompliantRequirements.isNotEmpty()) + } + assertEquals(StandardInputAction.UploadOriginal(OriginalReason.NoKnownStandardInputViolation), planner.plan(facts).action) + } } @Test fun conversionDoesNotGuessUnknownAudioPresenceMeansSilence() { @@ -109,8 +112,7 @@ class StandardInputPlannerTests { remediableRequirements = setOf(PolicyRequirement.FrameRate), ) val facts = compliantFacts().copy(frameRate = known(121.0)) - for (audio in listOf(MediaFact.Unknown, known(listOf(AudioTrack())), - known(listOf(AudioTrack(known(AudioFormat.OtherCodec)), AudioTrack())))) { + for (audio in listOf(MediaFact.Unknown, known(listOf(AudioTrack())))) { val action = planner.plan(facts.copy(audioTracks = audio), capabilities = videoCapabilities).action assertTrue("Audio must have its own proven AAC path: $action", action is StandardInputAction.Fallback) assertTrue((action as StandardInputAction.Fallback).reason is FallbackReason.UnsupportedConversion) @@ -124,7 +126,6 @@ class StandardInputPlannerTests { val facts = compliantFacts().copy(frameRate = known(121.0)) for (audio in listOf(facts.audioTracks, MediaFact.Unknown, known(listOf(AudioTrack())), known(listOf(AudioTrack(known(AudioFormat.OtherCodec)))), - known(listOf(AudioTrack(known(AudioFormat.OtherCodec)), AudioTrack())), known(listOf(AudioTrack(known(AudioFormat.Aac(AudioChannelLayout.FivePointOne))))))) { val source = facts.copy(audioTracks = audio) assertTrue(fallback(source, videoOnly) is FallbackReason.UnsupportedConversion) diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspectorTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspectorTests.kt index 439fa552..5d34b5a4 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspectorTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/StandardInputTimelineInspectorTests.kt @@ -170,7 +170,7 @@ class StandardInputTimelineInspectorTests : AbsRobolectricTest() { assertEquals(known(3.1), result.timeline.durationSeconds) assertEquals(known(AudioVideoStartOffset.Seconds(0.1)), result.timeline.audioVideoStartOffset) } - @Test fun droppedSecondaryAudioDoesNotExtendTheValidatedTimeline() { + @Test fun inspectingAllAudioTimelinesDoesNotAuthorizeDroppingMuxPrimaryTrack() { val firstAudio = compliantFacts().audioTracks.valueOrNull!!.single() val input = metadata(true, true).copy(facts = compliantFacts().copy(audioTracks = known(listOf(firstAudio, AudioTrack())))) val output = inspect(audio = true, audioOffset = 100, videoDeltas = List(3) { 200 }, @@ -180,12 +180,17 @@ class StandardInputTimelineInspectorTests : AbsRobolectricTest() { videoDeltas = List(3) { 200 }, input = input) assertEquals(SampleScanStatus.Complete, source.status) assertEquals(9, source.sampleCount) - val conversion = (StandardInputPlanner().plan(source.facts.copy(averageBitrate = known(9_000_000L)), + assertEquals(StandardInputAction.Fallback(FallbackReason.UnverifiedAudioSelection), + StandardInputPlanner().plan(source.facts.copy(averageBitrate = known(9_000_000L)), options(), fullCapabilities()).action) + // A plan for the first track alone cannot justify dropping another source track. + val conversion = (StandardInputPlanner().plan(source.facts.copy(averageBitrate = known(9_000_000L), + audioTracks = known(listOf(firstAudio))), options(), fullCapabilities()).action as StandardInputAction.Convert).conversion val facts = output.facts.copy(rotationDegrees = known(0), encodedDimensions = known(conversion.outputDimensions)) val validator = StandardInputOutputValidator() val result = validator.validateFacts(facts, source.facts, source.timeline, output.timeline, conversion) - assertTrue("Secondary audio duration changed acceptance: $result", result is StandardInputOutputValidation.Accepted) + val rejection = (result as StandardInputOutputValidation.Rejected).reason as OutputRejection.InsufficientPlanEvidence + assertTrue(OutputExpectation.Audio in rejection.expectations) assertEquals(known(3.1), source.timeline.durationSeconds) val shortened = validator.validateFacts(facts, source.facts, source.timeline, output.timeline.copy(firstAudioDurationSeconds = known(2.0)), conversion) as StandardInputOutputValidation.Rejected diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/UploadPreparationTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/UploadPreparationTests.kt index c9adf9af..189ccdc1 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/UploadPreparationTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/UploadPreparationTests.kt @@ -55,6 +55,18 @@ class UploadPreparationTests : AbsRobolectricTest() { preparation.prepare(upload(), context)) } } + @Test fun multipleAudioTracksSelectOriginalEvenWhenGeneratedResumeIsVerified() = runBlocking { + for (codec in listOf(VideoCodec.H264, VideoCodec.Hevc)) { + val facts = source(codec).copy(averageBitrate = known(9_000_000L), audioTracks = known(listOf( + AudioTrack(known(AudioFormat.Aac(AudioChannelLayout.Stereo))), + AudioTrack(known(AudioFormat.Aac(AudioChannelLayout.Mono)))))) + val preparation = UploadPreparation(inspectMetadata = { MetadataInspectionResult.Success(metadata(facts)) }, + inspectSamples = { _, _, _ -> MediaSampleInspection(facts, SampleScanStatus.Complete, timeline = timeline) }, + convert = { _, _, _, _, _ -> error("Multi-audio must keep server selection on original bytes") }) + assertEquals(PreparedUpload.Original(PreparationDiagnostic.UnsupportedPlan), + preparation.prepare(upload(), context, generatedResumeVerified = true)) + } + } @Test fun adapterReceivesPlannedCodecAndOnlyValidatedCompletionCanSelectGenerated() = runBlocking { for (codec in listOf(VideoCodec.H264, VideoCodec.Hevc)) { From b0a3a25bf404d7065057617ce92a612aa1ae9278 Mon Sep 17 00:00:00 2001 From: Eugene Morozov Date: Wed, 7 Oct 2026 13:58:02 -0700 Subject: [PATCH 27/49] NAT-547: harden SDR conversion and add reproducible review coverage --- library/build.gradle | 5 + library/src/androidTest/README.md | 59 ++++++ .../SdrConversionFixtureTests.kt | 119 ++++++++++++ .../standardization/Media3SdrExportEngine.kt | 84 ++++++--- .../upload/internal/standardization/README.md | 12 +- .../standardization/SdrEncodingTargets.kt | 81 ++++++-- .../SdrConversionAdapterTests.kt | 66 ++++++- .../SdrMedia3ConfigurationTests.kt | 174 ++++++++++++++++++ 8 files changed, 548 insertions(+), 52 deletions(-) create mode 100644 library/src/androidTest/README.md create mode 100644 library/src/androidTest/java/com/mux/video/upload/internal/standardization/SdrConversionFixtureTests.kt create mode 100644 library/src/test/java/com/mux/video/upload/internal/standardization/SdrMedia3ConfigurationTests.kt diff --git a/library/build.gradle b/library/build.gradle index d7eff249..5446e470 100644 --- a/library/build.gradle +++ b/library/build.gradle @@ -35,6 +35,9 @@ android { buildFeatures { buildConfig true } + testOptions { + targetSdk = 36 + } } muxDistribution { @@ -98,4 +101,6 @@ dependencies { testImplementation 'androidx.test.ext:junit:1.2.1' testImplementation "io.mockk:mockk:1.13.9" testImplementation 'org.robolectric:robolectric:4.11.1' + androidTestImplementation 'androidx.test:runner:1.6.2' + androidTestImplementation 'androidx.test.ext:junit:1.2.1' } diff --git a/library/src/androidTest/README.md b/library/src/androidTest/README.md new file mode 100644 index 00000000..7d6f5e0b --- /dev/null +++ b/library/src/androidTest/README.md @@ -0,0 +1,59 @@ +# SDR conversion fixtures + +`SdrConversionFixtureTests` runs the real Media3 adapter and shared output validator. Media stays outside Git and the test APK. With no staged directory, the suite is skipped; a staged directory with a missing fixture fails. Use a dedicated device/emulator and specify its serial on every adb command. + +Generate the following fixtures with FFmpeg built with libx264, libx265, AAC, and libmp3lame. These recipes reproduce the scenario contracts, not byte-identical encodes across FFmpeg versions. Record the FFmpeg version, source SHA-256s, API/device/encoder, test output, and any captured generated media in the external evidence directory. The multi-audio first track is stereo, non-default and 3 seconds; the default secondary is mono and 6 seconds. HEVC success remains device-dependent and must pass reinspection. + +```sh +SDR_FIXTURES=/absolute/path/to/external/sdr-fixtures +mkdir -p "$SDR_FIXTURES" +ffmpeg -y -f lavfi -i testsrc2=size=1920x1080:rate=30:duration=3 \ + -vf setparams=color_trc=bt709:color_primaries=bt709:colorspace=bt709 \ + -c:v libx264 -profile:v baseline -pix_fmt yuv420p -bf 0 \ + -color_primaries bt709 -color_trc bt709 -colorspace bt709 "$SDR_FIXTURES/no-audio.mp4" +ffmpeg -y -i "$SDR_FIXTURES/no-audio.mp4" \ + -f lavfi -i sine=frequency=440:sample_rate=48000:duration=3 \ + -f lavfi -i sine=frequency=880:sample_rate=48000:duration=6 \ + -filter_complex '[1:a]aformat=channel_layouts=stereo[first]' \ + -map 0:v -map '[first]' -map 2:a -c:v copy -c:a aac \ + -disposition:a:0 0 -disposition:a:1 default "$SDR_FIXTURES/multi-audio.mp4" +ffmpeg -y -i "$SDR_FIXTURES/no-audio.mp4" -c copy \ + -metadata:s:v:0 rotate=90 "$SDR_FIXTURES/rotated.mp4" +ffmpeg -y -i "$SDR_FIXTURES/no-audio.mp4" \ + -f lavfi -i sine=frequency=440:sample_rate=48000:duration=3 \ + -map 0:v -map 1:a -c:v copy -c:a libmp3lame -ac 2 "$SDR_FIXTURES/mp3-stereo.mp4" +ffmpeg -y -f lavfi -i testsrc2=size=512x288:rate=240:duration=2 \ + -vf setparams=color_trc=bt709:color_primaries=bt709:colorspace=bt709 \ + -c:v libx264 -profile:v baseline -pix_fmt yuv420p -bf 0 \ + -color_primaries bt709 -color_trc bt709 -colorspace bt709 "$SDR_FIXTURES/high-rate.mp4" +ffmpeg -y -f lavfi -i testsrc2=size=512x288:rate=60:duration=30 \ + -vf "select='if(lt(t,7.5),1,not(mod(n,3)))',setparams=color_trc=bt709:color_primaries=bt709:colorspace=bt709" -fps_mode vfr \ + -c:v libx264 -profile:v baseline -pix_fmt yuv420p -bf 0 -g 10000 -keyint_min 10000 -sc_threshold 0 -crf 18 \ + -color_primaries bt709 -color_trc bt709 -colorspace bt709 "$SDR_FIXTURES/vfr.mp4" +ffmpeg -y -i "$SDR_FIXTURES/no-audio.mp4" -c:v libx264 -profile:v baseline -pix_fmt yuv420p -bf 0 \ + -vf setparams=color_trc=linear:color_primaries=bt709:colorspace=bt709 \ + -color_primaries bt709 -color_trc linear -colorspace bt709 "$SDR_FIXTURES/linear.mp4" +for SDR_DURATION in 3 12; do + if [ "$SDR_DURATION" = 3 ]; then SDR_HEVC_NAME=hevc-small-open-gop; else SDR_HEVC_NAME=hevc-long-open-gop; fi + ffmpeg -y -f lavfi -i "testsrc2=size=512x288:rate=30:duration=$SDR_DURATION" \ + -vf setparams=color_trc=bt709:color_primaries=bt709:colorspace=bt709 \ + -c:v libx265 -pix_fmt yuv420p -x265-params 'open-gop=1:keyint=30:min-keyint=30:scenecut=0:bframes=0' \ + -color_primaries bt709 -color_trc bt709 -colorspace bt709 "$SDR_FIXTURES/$SDR_HEVC_NAME.mp4" +done +``` + +Build/install the test APK, then stage fixtures. Choose the serial of your dedicated test device; the commands below do not select an arbitrary connected device. + +```sh +SDR_DEVICE=emulator-5556 +./gradlew :library:assembleDebugAndroidTest +adb -s "$SDR_DEVICE" install -r library/build/outputs/apk/androidTest/debug/library-debug-androidTest.apk +adb -s "$SDR_DEVICE" push "$SDR_FIXTURES/." /data/local/tmp/nat547-fixtures/ +adb -s "$SDR_DEVICE" shell run-as com.mux.video.upload.test mkdir -p files/sdr-fixtures +adb -s "$SDR_DEVICE" shell run-as com.mux.video.upload.test cp -r /data/local/tmp/nat547-fixtures/. files/sdr-fixtures/ +adb -s "$SDR_DEVICE" shell am instrument -w -r \ + -e class com.mux.video.upload.internal.standardization.SdrConversionFixtureTests \ + com.mux.video.upload.test/androidx.test.runner.AndroidJUnitRunner +``` + +The H.264 matrix requires accepted, independently reinspected output, including preserved VFR timestamps and first stereo audio selection. LINEAR must fail before export. HEVC may succeed or fail conservatively on an unproven device; a safe failure is not evidence of working HEVC conversion. The cancellation test checks source bytes, tracked output cleanup, and one terminal result. API 23 and physical-device HEVC, quality/performance, and Mux ingest remain separate acceptance gates. diff --git a/library/src/androidTest/java/com/mux/video/upload/internal/standardization/SdrConversionFixtureTests.kt b/library/src/androidTest/java/com/mux/video/upload/internal/standardization/SdrConversionFixtureTests.kt new file mode 100644 index 00000000..3c6426d4 --- /dev/null +++ b/library/src/androidTest/java/com/mux/video/upload/internal/standardization/SdrConversionFixtureTests.kt @@ -0,0 +1,119 @@ +package com.mux.video.upload.internal.standardization + +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import com.mux.video.upload.internal.InputStandardization +import com.mux.video.upload.internal.MaximumResolution +import org.junit.Assert.* +import org.junit.Assume.assumeTrue +import org.junit.Test +import org.junit.runner.RunWith +import java.io.File +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicInteger + +/** Stage external fixtures as documented in the adapter README; never package media in the APK. */ +@RunWith(AndroidJUnit4::class) +class SdrConversionFixtureTests { + private val instrumentation = InstrumentationRegistry.getInstrumentation() + private val context = instrumentation.targetContext + private val fixtures = File(instrumentation.context.filesDir, "sdr-fixtures") + + private fun source(name: String): File { + assumeTrue("Stage SDR fixtures before running this suite", fixtures.isDirectory) + val fixture = File(fixtures, name) + assertTrue("Missing fixture $name", fixture.isFile) + return File(context.cacheDir, "mux-upload/source-$name").apply { + parentFile!!.mkdirs(); fixture.copyTo(this, overwrite = true) + } + } + private fun inspect(file: File): Triple { + val metadata = (MediaMetadataInspector().inspect(file) as MetadataInspectionResult.Success).inspection + val samples = StandardInputTimelineInspector().inspect(file, metadata) + assertEquals(SampleScanStatus.Complete, samples.status) + // Direct adapter exercise: these test flags do not establish hardware capability proof. + val capabilities = PlanningCapabilities(true, true, setOf(VideoCodec.H264, VideoCodec.Hevc), PolicyRequirement.entries.toSet(), + canPrepareCompliantAacAudio = true) + val action = StandardInputPlanner().plan(samples.facts, + InputStandardization(true, MaximumResolution.Preset1280x720), capabilities).action + assertTrue("${file.name}: $action\n${samples.facts}\n$metadata", action is StandardInputAction.Convert) + return Triple(metadata, samples, (action as StandardInputAction.Convert).conversion) + } + private fun ownedFiles(): Set = File(context.cacheDir, "mux-upload/standard-input").listFiles() + ?.map { it.name }?.toSet() ?: emptySet() + + private fun export(name: String): SdrConversionResult { + val file = source(name) + val original = file.readBytes() + val (metadata, samples, conversion) = inspect(file) + val before = ownedFiles() + val done = CountDownLatch(1) + val calls = AtomicInteger() + var result: SdrConversionResult? = null + val attempt = SdrConversionAdapter(context).start(file, metadata, samples, conversion) { + result = it; calls.incrementAndGet(); done.countDown() + } + try { + assertTrue("Timed out $name", done.await(90, TimeUnit.SECONDS)) + File(instrumentation.context.filesDir, "sdr-evidence").apply { mkdirs() }.let { area -> + File(area, "$name.txt").writeText("$metadata\n${samples.facts}\n${samples.timeline}\n$conversion\n$result") + } + assertEquals(1, calls.get()) + assertArrayEquals(original, file.readBytes()) + if (result is SdrConversionResult.Completed) { + val completed = result as SdrConversionResult.Completed + assertEquals(MediaFact.Known(conversion.outputCodec), completed.facts.videoCodec) + assertEquals(MediaFact.Known(conversion.outputDimensions), completed.facts.encodedDimensions) + assertEquals(MediaFact.Known(0), completed.facts.rotationDegrees) + assertTrue(StandardInputOutputValidator().validateGeneratedOutput(completed.output.file, samples.facts, + samples.timeline, conversion) is StandardInputOutputValidation.Accepted) + assertTrue(completed.output.delete()) + } + assertEquals(before, ownedFiles()) + return checkNotNull(result) + } finally { + attempt.cancel(); (result as? SdrConversionResult.Completed)?.output?.delete(); file.delete() + } + } + @Test fun h264ResizeSilentPortraitNonAacAndCadenceExportsPassReinspection() { + for (name in listOf("multi-audio.mp4", "no-audio.mp4", "rotated.mp4", "mp3-stereo.mp4", "high-rate.mp4", "vfr.mp4")) { + val result = export(name) + assertTrue("$name: $result", result is SdrConversionResult.Completed) + if (name == "multi-audio.mp4") assertEquals(MediaFact.Known(listOf(AudioTrack(MediaFact.Known( + AudioFormat.Aac(AudioChannelLayout.Stereo))))), (result as SdrConversionResult.Completed).facts.audioTracks) + } + } + @Test fun hevcEitherPassesFullValidationOrRejectsSafelyWithoutChangingCodecFamily() { + for (name in listOf("hevc-small-open-gop.mp4", "hevc-long-open-gop.mp4")) { + val result = export(name) + assertTrue("$name: $result", result is SdrConversionResult.Completed || result in listOf( + SdrConversionResult.Failed(SdrConversionFailure.CapabilityUnavailable), + SdrConversionResult.Failed(SdrConversionFailure.Export), SdrConversionResult.Failed(SdrConversionFailure.OutputInvalid))) + } + } + @Test fun linearTaggedInputFallsBackBeforeExport() { + assertEquals(SdrConversionResult.Failed(SdrConversionFailure.UnsupportedPlan), export("linear.mp4")) + } + @Test fun cancellationRemovesOnlyOwnedPartialAndDoesNotPublishLateResults() { + val file = source("multi-audio.mp4") + val original = file.readBytes() + val (metadata, samples, conversion) = inspect(file) + val before = ownedFiles() + val done = CountDownLatch(1) + val calls = AtomicInteger() + var result: SdrConversionResult? = null + val attempt = SdrConversionAdapter(context).start(file, metadata, samples, conversion) { + result = it; calls.incrementAndGet(); done.countDown() + } + try { + attempt.cancel() + assertTrue(done.await(30, TimeUnit.SECONDS)) + assertEquals(SdrConversionResult.Cancelled, result) + Thread.sleep(500) + assertEquals(1, calls.get()) + assertEquals(before, ownedFiles()) + assertArrayEquals(original, file.readBytes()) + } finally { attempt.cancel(); file.delete() } + } +} diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/Media3SdrExportEngine.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/Media3SdrExportEngine.kt index 027ffade..ff606d0f 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/Media3SdrExportEngine.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/Media3SdrExportEngine.kt @@ -11,6 +11,7 @@ import android.util.Pair import androidx.media3.common.C import androidx.media3.common.Format import androidx.media3.common.MediaItem +import androidx.media3.common.MimeTypes import androidx.media3.common.util.Clock import androidx.media3.common.util.MediaFormatUtil import androidx.media3.common.util.UnstableApi @@ -51,7 +52,7 @@ internal class Media3SdrExportEngine( .setRemoveAudio(targets.audioChannels == null) .setEffects(Effects(emptyList(), listOf(Presentation.createForWidthAndHeight( conversion.outputDimensions.width, conversion.outputDimensions.height, Presentation.LAYOUT_SCALE_TO_FIT)))) - if (conversion.outputCadence == OutputCadence.ConstantFrameRate) item.setFrameRate(30) + if (conversion.outputCadence == OutputCadence.ConstantFrameRate) item.setFrameRate(conversion.outputFrameRate.toInt()) val items = listOf(item.build()) val sequence = if (targets.audioChannels == null) EditedMediaItemSequence.withVideoFrom(items) else EditedMediaItemSequence.withAudioAndVideoFrom(items) @@ -61,23 +62,11 @@ internal class Media3SdrExportEngine( .setLooper(looper) .setPortraitEncodingEnabled(true) .setVideoMimeType(targets.videoMime) - .setAudioMimeType("audio/mp4a-latm") + .setAudioMimeType(MimeTypes.AUDIO_AAC) .setAssetLoaderFactory(assetLoader) .setEncoderFactory(StrictSdrEncoderFactory(context, conversion, targets, capability)) - .addListener(object : Transformer.Listener { - override fun onCompleted(composition: Composition, exportResult: ExportResult) { - if (targets.audioChannels != null && (exportResult.channelCount != targets.audioChannels || - exportResult.sampleRate != targets.audioSampleRate)) failed(SdrConversionFailure.OutputInvalid) - else completed(exportResult.videoMimeType, exportResult.audioMimeType) - } - override fun onError(composition: Composition, exportResult: ExportResult, exportException: ExportException) { - failed(SdrConversionFailure.Export) - } - override fun onFallbackApplied(composition: Composition, originalTransformationRequest: TransformationRequest, - fallbackTransformationRequest: TransformationRequest) { - failed(SdrConversionFailure.ForbiddenFallback) - } - }).build() + // Retain Media3's default no-output-sample watchdog (10s; 25s on emulators). + .addListener(SdrExportListener(targets, completed, failed)).build() transformer = exporter exporter.start(composition, output.absolutePath) } @@ -89,6 +78,25 @@ internal class Media3SdrExportEngine( } } +@UnstableApi +internal class SdrExportListener( + private val targets: SdrEncodingTargets, private val completed: (String?, String?) -> Unit, + private val failed: (SdrConversionFailure) -> Unit, +) : Transformer.Listener { + override fun onCompleted(composition: Composition, exportResult: ExportResult) { + if (targets.audioChannels != null && (exportResult.channelCount != targets.audioChannels || + exportResult.sampleRate != targets.audioSampleRate)) failed(SdrConversionFailure.OutputInvalid) + else completed(exportResult.videoMimeType, exportResult.audioMimeType) + } + override fun onError(composition: Composition, exportResult: ExportResult, exportException: ExportException) { + failed(SdrConversionFailure.Export) + } + override fun onFallbackApplied(composition: Composition, originalTransformationRequest: TransformationRequest, + fallbackTransformationRequest: TransformationRequest) { + failed(SdrConversionFailure.ForbiddenFallback) + } +} + /** MP4 Format.id is the container track ID. Never choose a preferred/default secondary track. */ @UnstableApi internal class FirstAudioTrackSelector(context: Context, private val trackId: String?) : DefaultTrackSelector(context) { @@ -97,7 +105,10 @@ internal class FirstAudioTrackSelector(context: Context, private val trackId: St .setExceedAudioConstraintsIfNecessary(true).setExceedRendererCapabilitiesIfNecessary(true)) } override fun selectAudioTrack(mappedTrackInfo: MappedTrackInfo, rendererFormatSupports: Array>, - rendererMixedMimeTypeAdaptationSupports: IntArray, params: Parameters): Pair? { + rendererMixedMimeTypeAdaptationSupports: IntArray, params: Parameters): Pair? = + selectFirstAudioTrack(mappedTrackInfo) + + internal fun selectFirstAudioTrack(mappedTrackInfo: MappedTrackInfo): Pair? { if (trackId == null) return null for (renderer in 0 until mappedTrackInfo.rendererCount) { if (mappedTrackInfo.getRendererType(renderer) != C.TRACK_TYPE_AUDIO) continue @@ -132,23 +143,50 @@ internal class StrictSdrEncoderFactory( override fun audioNeedsEncoding() = targets.audioChannels != null && !targets.copyAac override fun createForAudioEncoding(format: Format, logSessionId: LogSessionId?): Codec { - check(format.sampleMimeType == "audio/mp4a-latm" && format.channelCount == targets.audioChannels && + check(format.sampleMimeType == MimeTypes.AUDIO_AAC && format.channelCount == targets.audioChannels && format.sampleRate == targets.audioSampleRate) return audioDelegate.createForAudioEncoding(format, logSessionId) } override fun createForVideoEncoding(format: Format, logSessionId: LogSessionId?): Codec { + val exact = requestedVideoFormat(format) + val media = SdrVideoEncoderConfiguration.create(exact, targets, capability.level) + return DefaultCodec(context, exact, media, capability.name, false, null) + } + + internal fun videoConfiguration(format: Format): MediaFormat = + SdrVideoEncoderConfiguration.create(requestedVideoFormat(format), targets, capability.level) + + private fun requestedVideoFormat(format: Format): Format { check(format.sampleMimeType == targets.videoMime && format.width == conversion.outputDimensions.width && format.height == conversion.outputDimensions.height && format.rotationDegrees == 0 && format.colorInfo?.colorTransfer == C.COLOR_TRANSFER_SDR) - val exact = format.buildUpon().setFrameRate(conversion.outputFrameRate.toFloat()).setAverageBitrate(targets.bitrate).build() - val media = MediaFormatUtil.createMediaFormatFromFormat(exact) + return format.buildUpon().setFrameRate(conversion.outputFrameRate.toFloat()).setAverageBitrate(targets.bitrate).build() + } +} + +/** Shared preflight/export keys; capability metadata remains only an advertised-format guard. */ +@UnstableApi +internal object SdrVideoEncoderConfiguration { + fun create(format: Format, targets: SdrEncodingTargets, level: Int, + sdk: Int = Build.VERSION.SDK_INT, device: String = Build.DEVICE, + soc: String = if (Build.VERSION.SDK_INT >= 31) Build.SOC_MODEL else ""): MediaFormat { + val media = MediaFormatUtil.createMediaFormatFromFormat(format) media.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface) media.setInteger(MediaFormat.KEY_BITRATE_MODE, MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_VBR) media.setInteger(MediaFormat.KEY_PROFILE, targets.platformProfile) - media.setInteger(MediaFormat.KEY_LEVEL, capability.level) + media.setInteger(MediaFormat.KEY_LEVEL, level) media.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, targets.keyframeIntervalSeconds) - if (Build.VERSION.SDK_INT >= 29) media.setInteger(MediaFormat.KEY_MAX_B_FRAMES, 0) - return DefaultCodec(context, exact, media, capability.name, false, null) + if (sdk >= 29) media.setInteger(MediaFormat.KEY_MAX_B_FRAMES, 0) + // Media3 1.11.1's empirical encoder performance settings, retaining its exceptions. + if (sdk >= 25) { + media.setInteger(MediaFormat.KEY_PRIORITY, 1) + val overflow = sdk in 31..34 && soc in setOf("SM8550", "SM7450", "SM6450", "SC9863A", "T612", "T606", "T603") + media.setInteger(MediaFormat.KEY_OPERATING_RATE, if (sdk == 26) 30 else if (overflow) 1000 else Int.MAX_VALUE) + } + // Redmi Note 9 Pro rejects high KEY_FRAME_RATE. This is a codec hint; timestamps + // and Media3's cadence remain planned. Lowering it only shortens the requested GOP. + if (sdk < 30 && device == "joyeuse" && format.frameRate > 30) media.setFloat(MediaFormat.KEY_FRAME_RATE, 30f) + return media } } diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/README.md b/library/src/main/java/com/mux/video/upload/internal/standardization/README.md index 5585b973..ad2d3b4c 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/README.md +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/README.md @@ -13,13 +13,13 @@ Each attempt allocates only its tracked unique `sdr-*.mp4` file under `cacheDir/ H.264 stays H.264, HEVC stays HEVC, and another proven SDR source uses H.264. Dimensions are the planner's exact display-oriented even dimensions, with orientation baked into pixels and no upscaling. Portrait encoding is explicitly enabled. Video always encodes, including an identity presentation effect, so bitrate/GOP remediation cannot silently remux the source. -The 8-bit 4:2:0 path requests H.264 Baseline (no B frames) or HEVC Main, VBR at 6 Mbps through a 2048-pixel long side or 15 Mbps above it, additionally capped to 75% of the plan's average-bitrate ceiling. GOP targets use 90% of the planned interval ceiling, rounded down to whole seconds, following Swift's headroom convention. Output validation remains the final bitrate, IDR/closed-GOP, profile, pixel, cadence, and timeline gate. +The 8-bit 4:2:0 path requests H.264 Baseline (no B frames) or HEVC Main, VBR at 6 Mbps through a 2048-pixel long side or 15 Mbps above it, additionally capped to 75% of the plan's average-bitrate ceiling. GOP targets use 90% of the planned interval ceiling, rounded down to whole seconds. For preserved VFR timestamps, the target is also reduced by the nominal rate times the longest measured presentation gap: encoders commonly convert the interval to a nominal-rate frame count. If a positive whole-second target cannot fit, preparation falls back before export. Output validation remains the final bitrate, IDR/closed-GOP, profile, pixel, cadence, and timeline gate. -Media3's default video factory overrides H.264 profile requests. The adapter therefore supplies Media3 `DefaultCodec` with the exact platform profile, level, dimensions, bitrate, nominal rate, surface input, and GOP interval. Transformer owns decoding, effects, encoding, and muxing. The AAC delegate disables both encoder and format fallback. Every `onFallbackApplied`, mismatched output MIME, or invalid generated file fails the attempt; a listed encoder never proves a usable path. +Media3's default video factory overrides H.264 profile requests. The adapter therefore supplies Media3 `DefaultCodec` with the exact platform profile, level, dimensions, bitrate, nominal rate, surface input, and GOP interval. Preflight and export share those configuration keys, including Media3 1.11.1's API/SoC performance settings and the Redmi Note 9 Pro high-rate hint workaround. Planned timestamps remain unchanged. Transformer owns decoding, effects, encoding, and muxing. Its default no-output-sample watchdog remains enabled (10 seconds normally, 25 seconds on emulators); timeout errors use the same terminal cleanup as other export failures. Native calls cannot be forcibly interrupted. The AAC delegate disables both encoder and format fallback. Every `onFallbackApplied`, mismatched output MIME, or invalid generated file fails the attempt; a listed encoder never proves a usable path. -The bounded advertised-format preflight examines at most 128 codecs and checks a 500 ms budget between platform calls. Native calls cannot be preempted. It checks source decode and exact output encode support before creating a file, and pins one supported video encoder. It does not turn advertised capabilities into the planner's device-path proof. +The bounded advertised-format preflight gives cold native enumeration a separate two-second cooperative limit and capability queries a 500 ms limit, checked between calls. These are conservative development guards, not measured hardware performance thresholds. Only matching capability queries count toward the 128-query cap; unrelated inventory entries cannot reject an otherwise supported device merely by exceeding 128 entries. It checks source decode and exact output encode support before creating a file, and pins one supported video encoder. It does not turn advertised capabilities into the planner's device-path proof. -Preserved cadence keeps source timestamps. Constant-rate reduction is limited to proven constant cadence with an integral source/30 fps ratio; a one-part-per-million allowance covers microsecond-rounded rate measurement. Media3 drops frames at 30 fps and the shared validator checks actual output cadence and duration. Frame duplication, fractional-rate resampling, 10-bit SDR encoding, and HDR/tone mapping have no proof in this adapter and fail conservatively. NAT-550 owns tone mapping. +Preserved cadence keeps source timestamps. Constant-rate reduction is limited to proven constant cadence with an integral source/30 fps ratio; a one-part-per-million allowance covers microsecond-rounded rate measurement. Media3 drops frames at the planned 30 fps and the shared validator checks actual output cadence and duration. LINEAR is correctly classified as SDR by inspection, but this adapter rejects its explicit transfer tags before export until color handling is proven. Frame duplication, fractional-rate resampling, 10-bit SDR encoding, and HDR/tone mapping have no proof in this adapter and fail conservatively. NAT-550 owns tone mapping. ## Audio decision @@ -29,8 +29,8 @@ The asset loader's selector matches the first audio track's container ID, rather ## Evidence and remaining limits -At implementation time, 210 unit tests pass in each debug/release variant; release SDK/sample builds and lint-vital pass. External API 36 emulator checks cover six accepted exports: H.264 resize, silent H.264, rotated portrait, MP3 stereo to AAC, short HEVC Main, and 240-to-30 fps H.264. Two HEVC failures exercise safe rejection: 720p exceeds the emulator's 512-pixel encoder limit; a longer small export emits CRA/open GOP and fails the existing validator. Cancellation removes generated bytes and preserves source copies. These are nine cases across three instrumentation tests, including the grouped export matrix. +Committed unit tests cover encoding targets, VFR interval targeting with the shared validator, strict encoder configuration, performance/workaround boundaries, cold/large codec inventories, first-track selection, Media3 timeout/fallback callbacks, and adapter ownership/cancellation. The committed device suite and [fixture recipes](../../../../../../../../androidTest/README.md) cover resize, silent H.264, rotated portrait, MP3 stereo to AAC, HEVC, 240-to-30 fps, preserved VFR, LINEAR fallback, and cancellation. Fixtures are staged outside the APK and remain external to product Git. Run these checks on each Media3 upgrade; unit configuration checks do not prove hardware behavior. The distinguishable multi-audio source has 3s video, 3s first stereo AAC (track ID 2, non-default), and 6s secondary mono AAC (ID 3, default). Output contains the first stereo track: its copied AAC bytes have the same SHA-256 as source audio ordinal zero and differ from the default secondary. This proves local selection only; Mux ingest parity remains pending. -Media binaries, fixture harness, output probes, and measurements stay outside product Git in `/private/tmp/nat547-sdr`. API 23-29 execution, physical hardware, long HEVC closed-GOP support, 10-bit SDR, HDR, high-resolution device coverage, visual quality/performance acceptance, and a live Mux upload are unverified. Emulator timings do not establish physical-device performance limits. NAT-548 can use the validated SDR path for the early upload journey; no predecessor merge is required to continue the stack. +API 36 emulator evidence includes a short accepted HEVC export and a longer export rejected for CRA/open GOP; the software encoder also cannot encode 720p. These observations do not establish a universal HEVC limitation or justify changing codec families. API 23-29 execution, physical hardware, long HEVC closed-GOP support, 10-bit SDR, HDR, high-resolution device coverage, visual quality/performance acceptance, and a live Mux upload remain verification gates. Emulator timings do not establish physical-device performance limits. NAT-548 can use the validated SDR path for the early upload journey; no predecessor merge is required to continue the stack. diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/SdrEncodingTargets.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/SdrEncodingTargets.kt index 4853a32a..9ce7d088 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/SdrEncodingTargets.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/SdrEncodingTargets.kt @@ -4,8 +4,11 @@ import android.media.MediaCodecInfo import android.media.MediaCodecInfo.CodecProfileLevel import android.media.MediaCodecList import android.media.MediaFormat +import androidx.media3.common.Format +import androidx.media3.common.MimeTypes import kotlin.math.abs import kotlin.math.ceil +import kotlin.math.floor internal enum class SdrConversionFailure { UnsupportedPlan, InsufficientSourceEvidence, CapabilityUnavailable, DiskSpace, @@ -34,6 +37,11 @@ internal data class SdrEncodingTargets( fun from(conversion: StandardInputConversion, metadata: MediaMetadataInspection, source: MediaSampleInspection): SdrEncodingTargets? { val facts = source.facts + // LINEAR is SDR, but this surface path has no proof of its transfer handling. + if (metadata.tracks.any { it.video?.let { video -> + video.platformColor.transfer.valueOrNull == MediaFormat.COLOR_TRANSFER_LINEAR || + video.configuration.color.transfer.valueOrNull == MediaFormat.COLOR_TRANSFER_LINEAR + } == true }) return null if (source.status != SampleScanStatus.Complete || conversion.toneMapsToSdr || conversion.sourceDynamicRange != DynamicRange.Sdr || facts.dynamicRange != MediaFact.Known(DynamicRange.Sdr) || facts.videoCodec != MediaFact.Known(conversion.sourceCodec) || facts.videoTrackCount != MediaFact.Known(1) || @@ -73,13 +81,29 @@ internal data class SdrEncodingTargets( AudioChannelLayout.FivePointOne) else channels !in 1..2))) return null val interval = conversion.outputPolicyLimits.maximumKeyframeIntervals[expectedCodec] ?: return null if (!interval.isFinite() || interval < 1) return null + var gopSeconds = floor(interval * 0.9).toInt() + if (conversion.outputCadence == OutputCadence.PreserveSourceTimestamps && + facts.cadence == MediaFact.Known(Cadence.Variable)) { + val times = source.timeline.videoPresentationSeconds.valueOrNull!! + if (times.size < 2 || times.any { !it.isFinite() }) return null + var longestGap = 0.0 + for (index in 1 until times.size) { + val gap = times[index] - times[index - 1] + if (gap <= 0) return null + longestGap = maxOf(longestGap, gap) + } + // Encoders commonly turn the interval into nominal-rate frames. Bound their + // elapsed span using the slowest observed gap; validation still proves output. + gopSeconds = minOf(gopSeconds, floor(interval * 0.9 / (rate * longestGap)).toInt()) + if (gopSeconds < 1) return null + } return SdrEncodingTargets( - if (expectedCodec == VideoCodec.Hevc) "video/hevc" else "video/avc", + if (expectedCodec == VideoCodec.Hevc) MimeTypes.VIDEO_H265 else MimeTypes.VIDEO_H264, if (expectedCodec == VideoCodec.Hevc) VideoProfile.HevcMain else VideoProfile.H264Baseline, if (expectedCodec == VideoCodec.Hevc) CodecProfileLevel.HEVCProfileMain else CodecProfileLevel.AVCProfileBaseline, minOf(6_000_000L.takeIf { size.longSide <= 2048 } ?: 15_000_000L, conversion.outputPolicyLimits.maximumAverageBitrate * 3 / 4).toInt().takeIf { it > 0 } ?: return null, - maxOf(1, (interval * 0.9).toInt()), channels, sampleRate, copy, first?.sourceTrackId?.valueOrNull?.toString(), + maxOf(1, gopSeconds), channels, sampleRate, copy, first?.sourceTrackId?.valueOrNull?.toString(), ) } } @@ -88,19 +112,34 @@ internal data class SdrEncodingTargets( internal data class SdrEncoderCapability(val name: String, val level: Int) /** Bounded advertised-format guard. This does not manufacture planner/device-path proof. */ -internal class SdrCapabilityPreflight { +internal class SdrCapabilityPreflight( + private val codecInfos: () -> Array = { MediaCodecList(MediaCodecList.REGULAR_CODECS).codecInfos }, + private val nanoTime: () -> Long = System::nanoTime, +) { fun videoEncoder(conversion: StandardInputConversion, metadata: MediaMetadataInspection, targets: SdrEncodingTargets, isCancelled: () -> Boolean): SdrEncoderCapability? { - val started = System.nanoTime() - fun withinBudget() = !isCancelled() && System.nanoTime() - started < 500_000_000L - val infos = MediaCodecList(MediaCodecList.REGULAR_CODECS).codecInfos - if (infos.size > 128 || !withinBudget()) return null + val enumerationStarted = nanoTime() + if (isCancelled()) return null + val infos = codecInfos() + // Cold native enumeration has its own cooperative bound, not the query budget. + if (isCancelled() || nanoTime() - enumerationStarted >= 2_000_000_000L) return null + val started = nanoTime() + var queries = 0 + fun withinBudget() = !isCancelled() && nanoTime() - started < 500_000_000L + fun supports(info: MediaCodecInfo, mime: String, format: MediaFormat): Boolean { + if (!withinBudget() || queries >= 128) return false + queries++ + return runCatching { info.getCapabilitiesForType(mime).isFormatSupported(format) }.getOrDefault(false) + } val video = metadata.tracks.singleOrNull { it.kind == TrackKind.Video } ?: return null val sourceMime = video.mimeType.valueOrNull ?: return null val sourceSize = video.video?.encodedDimensions?.valueOrNull ?: return null - fun supportsDecode(mime: String, format: MediaFormat): Boolean = infos.any { info -> - withinBudget() && !info.isEncoder && info.supportedTypes.any { it.equals(mime, true) } && - runCatching { info.getCapabilitiesForType(mime).isFormatSupported(format) }.getOrDefault(false) + fun supportsDecode(mime: String, format: MediaFormat): Boolean { + for (info in infos) { + if (!withinBudget()) return false + if (!info.isEncoder && info.supportedTypes.any { it.equals(mime, true) } && supports(info, mime, format)) return true + } + return false } val sourceFormat = MediaFormat.createVideoFormat(sourceMime, sourceSize.width, sourceSize.height) val sourceProfile = when (video.video.profile.valueOrNull) { @@ -116,26 +155,28 @@ internal class SdrCapabilityPreflight { if (targets.audioChannels != null && !targets.copyAac) { val audioMime = metadata.tracks.first { it.sourceTrackId.valueOrNull?.toString() == targets.firstAudioTrackId }.mimeType.valueOrNull ?: return null if (!supportsDecode(audioMime, MediaFormat.createAudioFormat(audioMime, targets.audioSampleRate!!, targets.audioChannels))) return null - val aac = MediaFormat.createAudioFormat("audio/mp4a-latm", targets.audioSampleRate, targets.audioChannels) + val aac = MediaFormat.createAudioFormat(MimeTypes.AUDIO_AAC, targets.audioSampleRate, targets.audioChannels) aac.setInteger(MediaFormat.KEY_BIT_RATE, 160_000) aac.setInteger(MediaFormat.KEY_AAC_PROFILE, CodecProfileLevel.AACObjectLC) - if (infos.none { withinBudget() && it.isEncoder && it.supportedTypes.contains("audio/mp4a-latm") && - runCatching { it.getCapabilitiesForType("audio/mp4a-latm").isFormatSupported(aac) }.getOrDefault(false) }) return null + if (infos.none { withinBudget() && it.isEncoder && it.supportedTypes.contains(MimeTypes.AUDIO_AAC) && + supports(it, MimeTypes.AUDIO_AAC, aac) }) return null } for (info in infos) { if (!withinBudget()) return null if (!info.isEncoder || !info.supportedTypes.contains(targets.videoMime)) continue + if (queries >= 128) return null + queries++ val candidate = runCatching { val caps = info.getCapabilitiesForType(targets.videoMime) val level = caps.profileLevels.filter { it.profile == targets.platformProfile }.maxOfOrNull { it.level } ?: return@runCatching null val size = conversion.outputDimensions - val format = MediaFormat.createVideoFormat(targets.videoMime, size.width, size.height) - format.setFloat(MediaFormat.KEY_FRAME_RATE, conversion.outputFrameRate.toFloat()) - format.setInteger(MediaFormat.KEY_BIT_RATE, targets.bitrate) - format.setInteger(MediaFormat.KEY_PROFILE, targets.platformProfile) - format.setInteger(MediaFormat.KEY_LEVEL, level) - format.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface) - format.setInteger(MediaFormat.KEY_BITRATE_MODE, MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_VBR) + // A device workaround may lower KEY_FRAME_RATE as a configure-time hint. + // Still guard the actual planned size/rate against advertised video limits. + if (caps.videoCapabilities?.areSizeAndRateSupported(size.width, size.height, conversion.outputFrameRate) != true) + return@runCatching null + val requested = Format.Builder().setSampleMimeType(targets.videoMime).setWidth(size.width).setHeight(size.height) + .setFrameRate(conversion.outputFrameRate.toFloat()).setAverageBitrate(targets.bitrate).build() + val format = SdrVideoEncoderConfiguration.create(requested, targets, level) if (caps.isFormatSupported(format)) SdrEncoderCapability(info.name, level) else null }.getOrNull() if (candidate != null && withinBudget()) return candidate diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/SdrConversionAdapterTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/SdrConversionAdapterTests.kt index c6a0d507..29630eff 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/SdrConversionAdapterTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/SdrConversionAdapterTests.kt @@ -1,6 +1,12 @@ package com.mux.video.upload.internal.standardization import android.content.Context +import android.media.MediaFormat +import androidx.media3.common.MediaItem +import androidx.media3.transformer.Composition +import androidx.media3.transformer.EditedMediaItem +import androidx.media3.transformer.EditedMediaItemSequence +import androidx.media3.transformer.ExportResult import androidx.test.core.app.ApplicationProvider import org.junit.Assert.* import org.junit.Test @@ -31,6 +37,39 @@ class SdrConversionAdapterTests { private fun targets(plan: StandardInputConversion = conversion, input: MediaFacts = facts, tracks: List = listOf(audio())) = SdrEncodingTargets.from(plan, metadata(tracks), samples(input)) + @Test fun vfrGopTargetUsesSlowestPresentationGapAndValidatorStillChecksActualOutput() { + val times = (0..900).map { if (it <= 450) it / 60.0 else 7.5 + (it - 450) / 20.0 } + val variable = facts.copy(cadence = known(Cadence.Variable)) + val time = timeline.copy(durationSeconds = known(30.0), videoDurationSeconds = known(30.0), + firstAudioDurationSeconds = known(30.0), videoPresentationSeconds = known(times)) + val source = samples(variable).copy(timeline = time) + val target = SdrEncodingTargets.from(conversion, metadata(), source)!! + assertTrue(target.keyframeIntervalSeconds in 1..12) + // Model an encoder that converts seconds to a frame count at the nominal 30 fps. + fun exportedGap(seconds: Int): Double { + val frames = seconds * 30 + return times.indices.filter { it + frames < times.size }.maxOf { times[it + frames] - times[it] } + } + fun validate(seconds: Int) = StandardInputOutputValidator().validateFacts(variable.copy( + maximumKeyframeIntervalSeconds = known(exportedGap(seconds)), encodedDimensions = known(conversion.outputDimensions), + rotationDegrees = known(0)), variable, time, time, conversion) + assertTrue(validate(18) is StandardInputOutputValidation.Rejected) + assertTrue(validate(target.keyframeIntervalSeconds) is StandardInputOutputValidation.Accepted) + } + + @Test fun linearTaggedSdrIsRejectedBeforeAllocatingOrStartingExport() { + val format = MediaFormat.createVideoFormat("video/avc", 1920, 1080).apply { + setInteger(MediaFormat.KEY_COLOR_TRANSFER, MediaFormat.COLOR_TRANSFER_LINEAR) + } + val video = MediaTrackMetadataReader.read(0, format) + val inspection = metadata().copy(tracks = listOf(video, audio())) + assertEquals(known(DynamicRange.Sdr), video.video!!.dynamicRange) + assertNull(SdrEncodingTargets.from(conversion, inspection, samples())) + val run = Run(inputMetadata = inspection); run.idle() + assertEquals(SdrConversionResult.Failed(SdrConversionFailure.UnsupportedPlan), run.results.single()) + assertEquals(0, run.engine.starts); run.assertInputSafe() + } + @Test fun pinsCodecFamilyAndTargetsBelowPublishedCeilings() { assertEquals("video/avc", targets()!!.videoMime) assertEquals(6_000_000, targets()!!.bitrate) @@ -66,7 +105,8 @@ class SdrConversionAdapterTests { assertNull(targets(tracks = listOf(audio().copy(sourceTrackId = MediaFact.Unknown)))) } @Test fun permitsOnlyProvenIntegralCadenceReductionAndKeepsVfrTimestamps() { - assertNotNull(targets(input = facts.copy(cadence = known(Cadence.Variable)))) + assertNotNull(SdrEncodingTargets.from(conversion, metadata(), samples(facts.copy(cadence = known(Cadence.Variable))) + .copy(timeline = timeline.copy(videoPresentationSeconds = known(listOf(0.0, 0.03, 0.08)))))) val cfr = conversion.copy(outputCadence = OutputCadence.ConstantFrameRate) assertNotNull(targets(cfr, facts.copy(frameRate = known(240.0)))) assertNotNull(targets(cfr, facts.copy(frameRate = known(240.00004008351402)))) @@ -93,6 +133,7 @@ class SdrConversionAdapterTests { StandardInputPolicyEvaluator().evaluate(facts, selection())), capability: SdrEncoderCapability? = SdrEncoderCapability("fake", 1), duringValidation: (() -> Unit)? = null, + inputMetadata: MediaMetadataInspection = metadata(), ) { val sourceFile = File.createTempFile("customer-", ".mp4", context.cacheDir).apply { writeText("original") } val unrelated = File(context.cacheDir, "mux-upload/customer-copy.mp4").apply { parentFile!!.mkdirs(); writeText("customer") } @@ -100,8 +141,15 @@ class SdrConversionAdapterTests { val results = mutableListOf() val adapter = SdrConversionAdapter(context, preflight = { _, _, _, _ -> capability }, validate = { _, _, _, _, _ -> duringValidation?.invoke(); validation }, createEngine = { _, _, _, _, _ -> engine }) - val attempt = adapter.start(sourceFile, metadata(), samples(), conversion, results::add) - fun idle() { shadowOf(attempt.looper).idle() } + val attempt = adapter.start(sourceFile, inputMetadata, samples(), conversion, results::add) + fun idle() { + if (results.isNotEmpty()) return + try { shadowOf(attempt.looper).idle() } + catch (error: IllegalStateException) { + // PAUSED-mode idle posts a barrier; terminal cleanup can quit before that post. + if (results.isEmpty() || error.message?.contains("handler thread dead") != true) throw error + } + } fun assertInputSafe() { assertEquals("original", sourceFile.readText()); assertEquals("customer", unrelated.readText()) } } @Test fun successRetainsVerifiedFileUntilOwnerDeletesItAndIgnoresLateEvents() { @@ -119,6 +167,18 @@ class SdrConversionAdapterTests { assertFalse(run.engine.output!!.exists()); run.assertInputSafe() run.engine.complete(); assertEquals(1, run.results.size) } + @Test fun media3MuxerWatchdogErrorUsesTerminalCleanupAndSuppressesLateCompletion() { + val run = Run(); run.idle() + val composition = Composition.Builder(EditedMediaItemSequence.withVideoFrom(listOf( + EditedMediaItem.Builder(MediaItem.fromUri("file:///test.mp4")).build()))).build() + SdrExportListener(targets()!!, run.engine.completions!!, run.engine.failures!!) + .onError(composition, ExportResult.Builder().build(), muxerTimeoutException()) + run.idle() + assertEquals(listOf(SdrConversionResult.Failed(SdrConversionFailure.Export)), run.results) + assertFalse(run.engine.output!!.exists()); run.assertInputSafe() + run.engine.complete(); run.attempt.cancel() + assertEquals(1, run.results.size); assertEquals(1, run.engine.cancellations) + } @Test fun cancelTerminatesWithoutMedia3CallbackAndSuppressesQueuedCompletion() { val run = Run(); run.idle(); run.engine.complete(); run.attempt.cancel(); run.idle() assertEquals(listOf(SdrConversionResult.Cancelled), run.results) diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/SdrMedia3ConfigurationTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/SdrMedia3ConfigurationTests.kt new file mode 100644 index 00000000..fed2acee --- /dev/null +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/SdrMedia3ConfigurationTests.kt @@ -0,0 +1,174 @@ +package com.mux.video.upload.internal.standardization + +import android.content.Context +import android.media.MediaCodecInfo +import android.media.MediaFormat +import androidx.media3.common.C +import androidx.media3.common.ColorInfo +import androidx.media3.common.Format +import androidx.media3.common.MediaItem +import androidx.media3.common.TrackGroup +import androidx.media3.exoplayer.source.TrackGroupArray +import androidx.media3.exoplayer.trackselection.MappingTrackSelector +import androidx.media3.transformer.Composition +import androidx.media3.transformer.EditedMediaItem +import androidx.media3.transformer.EditedMediaItemSequence +import androidx.media3.transformer.ExportException +import androidx.media3.transformer.ExportResult +import androidx.media3.transformer.TransformationRequest +import androidx.test.core.app.ApplicationProvider +import io.mockk.* +import org.junit.Assert.* +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.annotation.Config +import org.robolectric.annotation.Implementation +import org.robolectric.annotation.Implements +import org.robolectric.shadows.MediaCodecInfoBuilder + +@RunWith(RobolectricTestRunner::class) +@Config(manifest = Config.NONE, sdk = [28], shadows = [SupportedFormatShadow::class]) +class SdrMedia3ConfigurationTests { + private val context = ApplicationProvider.getApplicationContext() + private val facts = compliantFacts().copy(cadence = known(Cadence.Constant)) + private val conversion = (StandardInputPlanner().plan(facts.copy(averageBitrate = known(9_000_000L)), + options(), fullCapabilities()).action as StandardInputAction.Convert).conversion + private val targets = SdrEncodingTargets("video/avc", VideoProfile.H264Baseline, + MediaCodecInfo.CodecProfileLevel.AVCProfileBaseline, 6_000_000, 18, null, null, false, null) + private val format = Format.Builder().setSampleMimeType("video/avc").setWidth(conversion.outputDimensions.width) + .setHeight(conversion.outputDimensions.height).setFrameRate(30f).setAverageBitrate(targets.bitrate) + .setColorInfo(ColorInfo.SDR_BT709_LIMITED).build() + private val video = MediaTrackMetadataReader.read(0, MediaFormat.createVideoFormat("video/avc", 1920, 1080)) + private val metadata = MediaMetadataInspection(known(ContainerKind.IsoBaseMedia), listOf(video), MediaFact.Unknown, facts, 0) + + @Test fun strictFactoryRejectsMimeDimensionsRotationAndLinearBeforeCodecCreation() { + val factory = StrictSdrEncoderFactory(context, conversion, targets, SdrEncoderCapability("chosen", 4)) + val configuration = factory.videoConfiguration(format) + assertEquals(targets.platformProfile, configuration.getInteger(MediaFormat.KEY_PROFILE)) + assertEquals(4, configuration.getInteger(MediaFormat.KEY_LEVEL)) + assertEquals(18, configuration.getInteger(MediaFormat.KEY_I_FRAME_INTERVAL)) + assertEquals(1, configuration.getInteger(MediaFormat.KEY_PRIORITY)) + for (bad in listOf(format.buildUpon().setSampleMimeType("video/hevc").build(), + format.buildUpon().setWidth(640).build(), format.buildUpon().setRotationDegrees(90).build(), + format.buildUpon().setColorInfo(ColorInfo.SDR_BT709_LIMITED.buildUpon().setColorTransfer(C.COLOR_TRANSFER_LINEAR).build()).build())) { + assertThrows(IllegalStateException::class.java) { factory.videoConfiguration(bad) } + } + } + + @Test fun performanceSettingsRetainApiAndChipsetExceptionsWithoutChangingProfile() { + val old = SdrVideoEncoderConfiguration.create(format, targets, 4, sdk = 23) + assertFalse(old.containsKey(MediaFormat.KEY_PRIORITY)); assertFalse(old.containsKey(MediaFormat.KEY_OPERATING_RATE)) + assertEquals(30, SdrVideoEncoderConfiguration.create(format, targets, 4, sdk = 26).getInteger(MediaFormat.KEY_OPERATING_RATE)) + val affected = SdrVideoEncoderConfiguration.create(format, targets, 4, sdk = 31, soc = "SM8550") + assertEquals(1000, affected.getInteger(MediaFormat.KEY_OPERATING_RATE)) + assertEquals(targets.platformProfile, affected.getInteger(MediaFormat.KEY_PROFILE)) + assertEquals(0, affected.getInteger(MediaFormat.KEY_MAX_B_FRAMES)) + assertEquals(Int.MAX_VALUE, SdrVideoEncoderConfiguration.create(format, targets, 4, sdk = 35, soc = "SM8550") + .getInteger(MediaFormat.KEY_OPERATING_RATE)) + } + + @Test fun redmiFrameRateWorkaroundDoesNotIncreaseLowRateGopOrResampleTimestamps() { + val high = format.buildUpon().setFrameRate(120f).build() + assertEquals(30f, SdrVideoEncoderConfiguration.create(high, targets, 4, sdk = 29, device = "joyeuse") + .getFloat(MediaFormat.KEY_FRAME_RATE), 0f) + val low = format.buildUpon().setFrameRate(5f).build() + assertEquals(5f, SdrVideoEncoderConfiguration.create(low, targets, 4, sdk = 29, device = "joyeuse") + .getFloat(MediaFormat.KEY_FRAME_RATE), 0f) + assertEquals(120f, high.frameRate, 0f) + } + + private fun codec(encoder: Boolean, mime: String): MediaCodecInfo { + val capabilityFormat = if (mime.startsWith("video/")) MediaFormat.createVideoFormat(mime, 1920, 1080) + else MediaFormat.createAudioFormat(mime, 48_000, 2) + val capabilities = MediaCodecInfoBuilder.CodecCapabilitiesBuilder.newBuilder().setMediaFormat(capabilityFormat) + .setIsEncoder(encoder).apply { + if (mime.startsWith("video/")) setColorFormats(intArrayOf(MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface)) + } + .setProfileLevels(arrayOf(MediaCodecInfo.CodecProfileLevel().apply { + profile = targets.platformProfile; level = MediaCodecInfo.CodecProfileLevel.AVCLevel51 + })).build() + return MediaCodecInfoBuilder.newBuilder().setName(if (encoder) "chosen" else "decoder") + .setIsEncoder(encoder).setCapabilities(capabilities).build() + } + + @Test fun coldEnumerationHasSeparateBoundAndUnrelatedCodecsDoNotExhaustQueries() { + var now = 0L + val unrelated = codec(false, "audio/mpeg") + val encoder = codec(true, "video/avc") + val decoder = codec(false, "video/avc") + val infos = Array(200) { unrelated } + arrayOf(decoder, encoder) + val queried = mutableListOf() + SupportedFormatShadow.supported = { queried.add(it); true } + val preflight = SdrCapabilityPreflight(codecInfos = { now += 800_000_000L; infos }, nanoTime = { now }) + assertEquals(SdrEncoderCapability("chosen", MediaCodecInfo.CodecProfileLevel.AVCLevel51), + preflight.videoEncoder(conversion, metadata, targets) { false }) + assertEquals(2, queried.size) + assertEquals(18, queried.last().getInteger(MediaFormat.KEY_I_FRAME_INTERVAL)) + assertEquals(1, queried.last().getInteger(MediaFormat.KEY_PRIORITY)) + } + + @Test fun preflightStillBoundsColdInitializationQueriesAndCancellation() { + var now = 0L + val encoder = codec(true, "video/avc") + val slowDecoder = codec(false, "video/avc") + var queries = 0 + SupportedFormatShadow.supported = { queries++; now += 500_000_000L; true } + val infos = arrayOf(slowDecoder, encoder) + assertNull(SdrCapabilityPreflight({ now += 2_000_000_000L; infos }, { now }) + .videoEncoder(conversion, metadata, targets) { false }) + now = 0 + assertNull(SdrCapabilityPreflight({ infos }, { now }).videoEncoder(conversion, metadata, targets) { false }) + assertEquals(1, queries) + var enumerated = false + assertNull(SdrCapabilityPreflight({ enumerated = true; infos }, { 0L }).videoEncoder(conversion, metadata, targets) { true }) + assertFalse(enumerated) + val rejected = codec(false, "video/avc") + queries = 0 + SupportedFormatShadow.supported = { queries++; false } + assertNull(SdrCapabilityPreflight({ Array(200) { rejected } }, { 0L }).videoEncoder(conversion, metadata, targets) { false }) + assertEquals(128, queries) + } + + @Test fun firstTrackSelectorIgnoresPreferredSecondaryAndFailsOnMissingIdentity() { + val first = Format.Builder().setId("2").setSampleMimeType("audio/mp4a-latm").setChannelCount(2).build() + val secondary = first.buildUpon().setId("3").setSelectionFlags(C.SELECTION_FLAG_DEFAULT).setChannelCount(1).build() + val groups = TrackGroupArray(TrackGroup("secondary", secondary), TrackGroup("first", first)) + val mapped = mockk { + every { rendererCount } returns 1 + every { getRendererType(0) } returns C.TRACK_TYPE_AUDIO + every { getTrackGroups(0) } returns groups + } + val selection = FirstAudioTrackSelector(context, "2").selectFirstAudioTrack(mapped)!! + assertEquals("2", selection.first.group.getFormat(selection.first.tracks.single()).id) + assertNull(FirstAudioTrackSelector(context, null).selectFirstAudioTrack(mapped)) + assertThrows(IllegalStateException::class.java) { FirstAudioTrackSelector(context, "missing").selectFirstAudioTrack(mapped) } + } + + @Test fun exportListenerRoutesMuxerWatchdogAndFallbackToFailureAndChecksCopiedAudio() { + val composition = Composition.Builder(EditedMediaItemSequence.withVideoFrom(listOf( + EditedMediaItem.Builder(MediaItem.fromUri("file:///test.mp4")).build()))).build() + val failures = mutableListOf() + var completed = false + val listener = SdrExportListener(targets.copy(audioChannels = 2, audioSampleRate = 48_000, copyAac = true), + { _, _ -> completed = true }, failures::add) + listener.onError(composition, ExportResult.Builder().build(), muxerTimeoutException()) + listener.onFallbackApplied(composition, TransformationRequest.Builder().build(), TransformationRequest.Builder().build()) + listener.onCompleted(composition, ExportResult.Builder().setChannelCount(1).setSampleRate(48_000).build()) + assertEquals(listOf(SdrConversionFailure.Export, SdrConversionFailure.ForbiddenFallback, SdrConversionFailure.OutputInvalid), failures) + assertFalse(completed) + listener.onCompleted(composition, ExportResult.Builder().setChannelCount(2).setSampleRate(48_000).build()) + assertTrue(completed) + } +} + +internal fun muxerTimeoutException(): ExportException = ExportException::class.java + .getDeclaredMethod("createForMuxer", Throwable::class.java, Int::class.javaPrimitiveType) + .apply { isAccessible = true }.invoke(null, IllegalStateException("No samples"), ExportException.ERROR_CODE_MUXING_TIMEOUT) as ExportException + +/** Only advertised format answers are faked; codec roles, MIME and profiles use Android objects. */ +@Implements(MediaCodecInfo.CodecCapabilities::class) +class SupportedFormatShadow { + @Implementation fun isFormatSupported(format: MediaFormat): Boolean = supported(format) + companion object { var supported: (MediaFormat) -> Boolean = { true } } +} From 7ded0b27ffba956ba11eb315c15c107948559534 Mon Sep 17 00:00:00 2001 From: Eugene Morozov Date: Thu, 8 Oct 2026 13:45:27 -0700 Subject: [PATCH 28/49] Fix upload preparation lifecycle review findings --- .../com/mux/video/upload/api/MuxUpload.kt | 129 ++- .../mux/video/upload/api/MuxUploadManager.kt | 73 +- .../upload/internal/TranscoderContext.kt | 845 ------------------ .../video/upload/internal/UploadAttempt.kt | 34 +- .../mux/video/upload/internal/UploadInfo.kt | 7 +- .../video/upload/internal/UploadJobFactory.kt | 20 +- .../upload/internal/UploadPersistence.kt | 75 +- .../standardization/SdrConversionAdapter.kt | 5 + .../standardization/UploadPreparation.kt | 18 +- .../UploadPreparationLifecycleTests.kt | 260 +++++- .../standardization/UploadPreparationTests.kt | 65 +- 11 files changed, 535 insertions(+), 996 deletions(-) delete mode 100644 library/src/main/java/com/mux/video/upload/internal/TranscoderContext.kt diff --git a/library/src/main/java/com/mux/video/upload/api/MuxUpload.kt b/library/src/main/java/com/mux/video/upload/api/MuxUpload.kt index 34821d8e..c4b6ac20 100644 --- a/library/src/main/java/com/mux/video/upload/api/MuxUpload.kt +++ b/library/src/main/java/com/mux/video/upload/api/MuxUpload.kt @@ -9,6 +9,12 @@ import com.mux.video.upload.internal.UploadInfo import com.mux.video.upload.internal.update import com.mux.video.upload.internal.writeUploadState import com.mux.video.upload.internal.forgetUploadState +import com.mux.video.upload.internal.readUploadResumeState +import com.mux.video.upload.internal.UploadCancelledException +import kotlinx.coroutines.flow.filterNotNull +import kotlinx.coroutines.flow.flatMapLatest +import kotlinx.coroutines.flow.flowOf +import kotlinx.coroutines.flow.map import kotlinx.coroutines.* import java.io.File @@ -34,9 +40,12 @@ import java.io.File * @see MuxUploadManager */ class MuxUpload private constructor( - private var uploadInfo: UploadInfo, + uploadInfo: UploadInfo, private val autoManage: Boolean = true, ) { + private var uploadInfo: UploadInfo = uploadInfo + get() = field.session.current.value ?: field + set(value) { field = value; value.session.current.value = value } /** * File containing the video to be uploaded @@ -73,14 +82,12 @@ class MuxUpload private constructor( /** * If the upload has failed, gets the error associated with the failure */ - val error get() = _error ?: uploadInfo.statusFlow?.value?.getError() - private var _error: Exception? = null + val error get() = currentStatus.getError() /** * True if the upload was successful, false otherwise */ - val isSuccessful get() = uploadInfo.statusFlow?.value?.isSuccessful() ?: _successful - private var _successful: Boolean = false + val isSuccessful get() = currentStatus.isSuccessful() private var resultListener: UploadEventListener>? = null private var progressListener: UploadEventListener? = null @@ -97,6 +104,7 @@ class MuxUpload private constructor( private val logger get() = MuxUploadSdk.logger init { + if (uploadInfo.session.current.value == null) uploadInfo.session.current.value = uploadInfo // Catch state if an upload was already in progress // no need to observe: the Flow will have the most-recent values when queried uploadInfo.statusFlow?.value?.let { status -> this.lastKnownStatus = status } @@ -127,13 +135,11 @@ class MuxUpload private constructor( forceRestart: Boolean = false, coroutineScope: CoroutineScope = CoroutineScope(Dispatchers.Default) ) { - if (cancelled) return + if (cancelled || uploadInfo.attempt?.isCancelled() == true) return if (!forceRestart && uploadInfo.uploadJob?.isActive == true && uploadInfo.attempt?.isStopped() != true) { observeUpload(uploadInfo) return } - _successful = false - _error = null // Get an updated UploadInfo with a job & event channels uploadInfo = if (autoManage) { // We may or may not get a fresh worker, depends on if the upload is already going @@ -141,10 +147,12 @@ class MuxUpload private constructor( } else { // If we're not managing the worker, the job is purely internal to this object if (forceRestart) { - val blocked = uploadInfo.generatedResumeBlocked || uploadInfo.attempt?.preparation?.generatedRequestStarted == true - uploadInfo.attempt?.cancel { if (!blocked) forgetUploadState(uploadInfo) } - uploadInfo.uploadJob?.cancel() - if (!blocked) forgetUploadState(uploadInfo) + val blocked = uploadInfo.generatedResumeBlocked || uploadInfo.attempt?.preparation?.generatedRequestStarted == true || + readUploadResumeState(uploadInfo).generatedResumeBlocked + val previous = uploadInfo + if (previous.attempt == null) forgetUploadState(previous) + else previous.attempt.cancel { forgetUploadState(previous) } + previous.uploadJob?.cancel() uploadInfo = uploadInfo.update(attempt = null, statusFlow = null, restoredFromOriginal = false, generatedResumeBlocked = blocked) } @@ -162,12 +170,15 @@ class MuxUpload private constructor( * If the upload had failed, it will be restarted and this function will suspend until it * completes * - * If the upload already succeeded, the old result will be returned immediately + * If the upload already succeeded, the old result will be returned immediately. + * After [cancel], this returns a failure without restarting. Cancellation of the calling + * coroutine still throws [CancellationException]. */ @Throws @Suppress("unused") @JvmSynthetic suspend fun awaitSuccess(): Result { + if (cancelled || uploadInfo.attempt?.isCancelled() == true) return Result.failure(UploadCancelledException()) val status = uploadStatus // base our logic on a stable snapshot of the status return if (status is UploadStatus.UploadSuccess) { Result.success(status) // If we succeeded already, don't start again @@ -186,6 +197,7 @@ class MuxUpload private constructor( */ @Suppress("MemberVisibilityCanBePrivate") fun pause() { + if (cancelled || uploadInfo.attempt?.isCancelled() == true || isPaused) return uploadInfo = if (autoManage) { observerJob?.cancel("user requested pause") /*uploadInfo =*/ MuxUploadManager.pauseJob(uploadInfo) @@ -199,8 +211,9 @@ class MuxUpload private constructor( } /** - * Cancels this upload. The upload job will be canceled and it will not be possible to start this - * job again where it left off + * Cancels this upload and ends this handle. Later [start] calls have no effect and + * [awaitSuccess] returns a failure. Create a new handle to start another upload. + * A destination that received generated bytes requires a new Direct Upload URL. */ @Suppress("MemberVisibilityCanBePrivate") fun cancel() { @@ -209,8 +222,9 @@ class MuxUpload private constructor( if (autoManage) { MuxUploadManager.cancelJob(uploadInfo) } else { - uploadInfo.attempt?.cancel { forgetUploadState(uploadInfo) } - uploadInfo.uploadJob?.cancel("user requested cancel") + val current = uploadInfo + if (current.attempt == null) forgetUploadState(current) else current.attempt.cancel { forgetUploadState(current) } + current.uploadJob?.cancel("user requested cancel") } observerJob?.cancel("user requested cancel") } @@ -222,13 +236,6 @@ class MuxUpload private constructor( */ @MainThread fun setProgressListener(listener: UploadEventListener?) { - if (listener == null) { - observerJob?.cancel("clearing listeners") - observerJob = null - } else { - observeUpload(uploadInfo) - } - progressListener = listener if (!cancelled && uploadInfo.attempt?.isCancelled() != true) lastKnownProgress?.let { listener?.onEvent(it) } observeUpload(uploadInfo) @@ -241,15 +248,9 @@ class MuxUpload private constructor( */ @MainThread fun setResultListener(listener: UploadEventListener>?) { - if (listener == null) { - observerJob?.cancel("clearing listeners") - observerJob = null - } else { - observeUpload(uploadInfo) - } - if (resultListener !== listener) deliveredResultStatus = null resultListener = listener + observeUpload(uploadInfo) notifyResult(currentStatus) } @@ -260,14 +261,8 @@ class MuxUpload private constructor( */ @MainThread fun setStatusListener(listener: UploadEventListener?) { - if (listener == null) { - observerJob?.cancel("clearing listeners") - observerJob = null - } else { - observeUpload(uploadInfo) - } - statusListener = listener + observeUpload(uploadInfo) if (!cancelled && uploadInfo.attempt?.isCancelled() != true) listener?.onEvent(currentStatus) } @@ -283,40 +278,29 @@ class MuxUpload private constructor( statusListener = null } - private fun newObserveProgressJob(upload: UploadInfo): Job? { - // Job that collects and notifies state updates on the main thread (suspending on main is safe) - return upload.statusFlow?.let { flow -> - callbackScope.launch { - flow.collect { status -> - if (cancelled || uploadInfo.attempt !== upload.attempt || upload.attempt?.isCancelled() == true) return@collect - // Update the status of our upload - lastKnownStatus = status - - statusListener?.onEvent(status) - // A listener can synchronously pause, cancel, or start a replacement attempt. - if (cancelled || uploadInfo.attempt !== upload.attempt || upload.attempt?.isCancelled() == true || - (upload.attempt?.isStopped() == true && status !is UploadStatus.UploadPaused)) return@collect - // Notify the specific listeners - when (status) { - is UploadStatus.Uploading -> { progressListener?.onEvent(status.uploadProgress) } - is UploadStatus.UploadPaused -> { progressListener?.onEvent(status.uploadProgress) } - is UploadStatus.UploadSuccess -> { - _successful = true - progressListener?.onEvent(status.uploadProgress) - if (!cancelled && upload.attempt?.isCancelled() != true && uploadInfo.attempt === upload.attempt) - notifyResult(status) - } - is UploadStatus.UploadFailed -> { - progressListener?.onEvent(status.uploadProgress) // Make sure we're most up-to-date - if (status.exception !is CancellationException && upload.attempt?.isCancelled() != true && - uploadInfo.attempt === upload.attempt) { - _error = status.exception - notifyResult(status) - } - } - else -> { } // no relevant info - } + @OptIn(ExperimentalCoroutinesApi::class) + private fun newObserveProgressJob(upload: UploadInfo): Job = callbackScope.launch { + upload.session.current.filterNotNull().flatMapLatest { observed -> + (observed.statusFlow ?: flowOf(UploadStatus.Ready)).map { observed to it } + }.collect { (observed, status) -> + if (cancelled || uploadInfo.attempt !== observed.attempt || observed.attempt?.isCancelled() == true) return@collect + lastKnownStatus = status + statusListener?.onEvent(status) + // A listener can synchronously pause, cancel, or replace the attempt. + if (cancelled || uploadInfo.attempt !== observed.attempt || observed.attempt?.isCancelled() == true || + (observed.attempt?.isStopped() == true && status !is UploadStatus.UploadPaused)) return@collect + when (status) { + is UploadStatus.Uploading -> progressListener?.onEvent(status.uploadProgress) + is UploadStatus.UploadPaused -> progressListener?.onEvent(status.uploadProgress) + is UploadStatus.UploadSuccess -> { + progressListener?.onEvent(status.uploadProgress) + if (!cancelled && observed.attempt?.isCancelled() != true && uploadInfo.attempt === observed.attempt) notifyResult(status) + } + is UploadStatus.UploadFailed -> { + progressListener?.onEvent(status.uploadProgress) + if (observed.attempt?.isCancelled() != true && uploadInfo.attempt === observed.attempt) notifyResult(status) } + else -> {} } } } @@ -338,7 +322,8 @@ class MuxUpload private constructor( private fun observeUpload(uploadInfo: UploadInfo) { observerJob?.cancel("switching observers") - observerJob = newObserveProgressJob(uploadInfo) + observerJob = if (!cancelled && (resultListener != null || progressListener != null || statusListener != null)) + newObserveProgressJob(uploadInfo) else null } /** diff --git a/library/src/main/java/com/mux/video/upload/api/MuxUploadManager.kt b/library/src/main/java/com/mux/video/upload/api/MuxUploadManager.kt index d4e203f7..d9b97c35 100644 --- a/library/src/main/java/com/mux/video/upload/api/MuxUploadManager.kt +++ b/library/src/main/java/com/mux/video/upload/api/MuxUploadManager.kt @@ -45,7 +45,7 @@ object MuxUploadManager { uploadsByFilename[videoFile.absolutePath]?.let { MuxUpload.create(it) } /** - * Finds all in-progress or paused uploads and returns [MuxUpload] objects representing them. You + * Finds all in-progress, paused, or failed uploads and returns [MuxUpload] objects representing them. You * don't need to hold these specific instances except where they're locally used. The upload jobs * will continue in parallel with the rest of your app */ @@ -63,16 +63,23 @@ object MuxUploadManager { @JvmSynthetic @MainThread internal fun resumeCachedJobs(includePaused: Boolean): List { - return readAllCachedUploads(includePaused) - .filter { uploadInfo -> - val exists = uploadInfo.inputFile.exists() - if (!exists) { - forgetUploadState(uploadInfo) - } - exists - } - .map { uploadInfo -> startJob(uploadInfo, restart = false) } - .map { MuxUpload.create(it) } + val restored = readAllCachedUploads().filter { upload -> + upload.inputFile.exists().also { if (!it) forgetUploadState(upload) } + }.map { upload -> + if (!includePaused && readUploadResumeState(upload).paused) { + uploadsByFilename.getOrPut(upload.inputFile.absolutePath) { + val saved = readUploadResumeState(upload) + val preparation = UploadPreparationState().apply { originalSelected = saved.originalSelected } + val attempt = UploadAttempt(preparation, MuxUpload.Progress( + bytesUploaded = if (saved.generatedResumeBlocked) 0 else saved.bytesSent, + totalBytes = upload.inputFile.length()), previousPersistenceOwnerId = saved.attemptId) + attempt.pause {} + upload.update(attempt = attempt, statusFlow = attempt.status).also { it.session.current.value = it } + } + } else startJob(upload, restart = false) + } + notifyListListeners() + return restored.map { MuxUpload.create(it) } } /** @@ -113,7 +120,7 @@ object MuxUploadManager { internal fun pauseJob(upload: UploadInfo): UploadInfo { assertMainThread() // Paused jobs stay in the manager and remain persisted - uploadsByFilename[upload.inputFile.absolutePath]?.let { + uploadsByFilename[upload.inputFile.absolutePath]?.takeIf { it.session === upload.session }?.let { it.attempt?.pause { state -> writeUploadState(it, state) } cancelJobInner(it) notifyListListeners() @@ -127,12 +134,17 @@ object MuxUploadManager { @MainThread internal fun cancelJob(upload: UploadInfo) { assertMainThread() - val current = uploadsByFilename[upload.inputFile.absolutePath] ?: upload - observerJobsByFilename.remove(current.inputFile.absolutePath)?.cancel() - current.attempt?.cancel { forgetUploadState(current) } - cancelJobInner(current) - uploadsByFilename -= current.inputFile.absolutePath - forgetUploadState(current) + val current = uploadsByFilename[upload.inputFile.absolutePath] + if (current != null && current.session !== upload.session) { + // An old destination's handle must not stop or forget its replacement. + upload.attempt?.cancel {} + upload.uploadJob?.cancel() + return + } + val cancelled = current ?: upload + observerJobsByFilename.remove(cancelled.inputFile.absolutePath)?.cancel() + cancelAttempt(cancelled) + uploadsByFilename -= cancelled.inputFile.absolutePath notifyListListeners() } @@ -140,12 +152,13 @@ object MuxUploadManager { @MainThread internal fun jobFinished(upload: UploadInfo, forgetJob: Boolean = true) { assertMainThread() + // Persistence belongs to the attempt, even when it was never registered with the manager. + if (forgetJob) forgetUploadState(upload) if (uploadsByFilename[upload.inputFile.absolutePath]?.attempt !== upload.attempt) return observerJobsByFilename.remove(upload.inputFile.absolutePath)?.cancel() - uploadsByFilename -= upload.inputFile.absolutePath - if (forgetJob) { - forgetUploadState(upload) - } + // Keep failed jobs discoverable so existing handles follow a manager-driven retry. + if (forgetJob || upload.statusFlow?.value !is UploadStatus.UploadFailed) + uploadsByFilename -= upload.inputFile.absolutePath notifyListListeners() } @@ -160,6 +173,12 @@ object MuxUploadManager { upload.uploadJob?.cancel() } + private fun cancelAttempt(upload: UploadInfo) { + val attempt = upload.attempt + if (attempt == null) forgetUploadState(upload) else attempt.cancel { forgetUploadState(upload) } + upload.uploadJob?.cancel() + } + private fun insertOrUpdateUpload(upload: UploadInfo, restart: Boolean): UploadInfo { val filename = upload.inputFile.absolutePath val previous = uploadsByFilename[filename] @@ -169,12 +188,12 @@ object MuxUploadManager { var source = previous ?: upload if (restart || newDestination) { val generatedMayExistRemotely = !newDestination && (source.generatedResumeBlocked || - source.attempt?.preparation?.generatedRequestStarted == true) - previous?.attempt?.cancel { if (!generatedMayExistRemotely) forgetUploadState(source) } - previous?.uploadJob?.cancel() - if (!generatedMayExistRemotely) forgetUploadState(upload) + source.attempt?.preparation?.generatedRequestStarted == true || readUploadResumeState(upload).generatedResumeBlocked) + cancelAttempt(source) source = upload.update(attempt = null, uploadJob = source.uploadJob, - statusFlow = null, restoredFromOriginal = false, generatedResumeBlocked = generatedMayExistRemotely) + statusFlow = null, restoredFromOriginal = false, generatedResumeBlocked = generatedMayExistRemotely).also { + it.session = if (newDestination) upload.session else source.session + } } val newUpload = startUploadJob(source) uploadsByFilename += upload.inputFile.absolutePath to newUpload diff --git a/library/src/main/java/com/mux/video/upload/internal/TranscoderContext.kt b/library/src/main/java/com/mux/video/upload/internal/TranscoderContext.kt deleted file mode 100644 index ff443772..00000000 --- a/library/src/main/java/com/mux/video/upload/internal/TranscoderContext.kt +++ /dev/null @@ -1,845 +0,0 @@ -package com.mux.video.upload.internal - -import android.R.id.input -import android.content.Context -import android.media.* -import android.media.MediaCodec.BufferInfo -import android.media.MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420SemiPlanar -import android.os.Build -import android.util.Log -import androidx.annotation.RequiresApi -import com.mux.video.upload.MuxUploadSdk -import io.github.crow_misia.libyuv.FilterMode -import io.github.crow_misia.libyuv.Nv12Buffer -import org.json.JSONArray -import java.io.File -import java.io.OutputStream -import java.nio.BufferOverflowException -import java.nio.ByteBuffer -import java.util.* -import kotlin.experimental.and - - -@RequiresApi(Build.VERSION_CODES.LOLLIPOP) -internal class TranscoderContext private constructor( - private var uploadInfo: UploadInfo, - private val appContext: Context, - private val sessionId: String -) { - private val logger get() = MuxUploadSdk.logger - - val MAX_ALLOWED_BITRATE: Int - val MAX_ALLOWED_FRAMERATE = 120 - val MIN_ALLOWED_FRAMERATE = 5 - val MAX_ALLOWED_WIDTH: Int - val MAX_ALLOWED_HEIGTH: Int - val OPTIMAL_FRAMERATE = 30 - val I_FRAME_INTERVAL = 5 // in seconds - val MAX_ALLOWED_I_FRAME_INTERVAL: Int - val OUTPUT_SAMPLERATE = 48000 - val OUTPUT_NUMBER_OF_CHANNELS = 2 - val OUTPUT_AUDIO_BITRATE = 96000 - - private val extractor: MediaExtractor = MediaExtractor() - private var muxer: MediaMuxer? = null - private var videoTrackIndex = -10 - private var audioTrackIndex = -10 - private var outputVideoTrackIndex = -1 - private var outputAudioTrackIndex = -1 - // Used to configure decoders - private var inputAudioFormat: MediaFormat? = null - private var inputVideoFormat: MediaFormat? = null - - // Used to configure encoders - private var outputAudioFormat: MediaFormat? = null - private var outputVideoFormat: MediaFormat? = null - - // This is what decoder actually provide as an output, bit different then what we used to configure it - private var videoDecoderOutputFormat: MediaFormat? = null - private var audioDecoderOutputFormat: MediaFormat? = null - private var decodedFrameWidth: Int = -1; - private var decodedFrameHeight: Int = -1; - private var targetedWidth = -1 - private var targetedHeight = -1 - private var targetedFramerate = -1 - private var targetedIFrameInterval = -1 - private var targetedBitrate = -1 - private var scaledSizeYuv: Nv12Buffer? = null - private var resampleCreated = false - private var resample: Resample = Resample() - private val audioFrames = ArrayList() - - // Input parameters - private var inputWidth = -1 - private var inputHeighth = -1 - private var inputBitrate = -1 - private var inputFramerate = -1 - private var inputChannelCount = -1 - private var inputSamplerate = -1 - - // Wait indefinetly for negative value, exit imidetly on 0, or timeout after a given us+ - private var dequeueTimeout:Long = 0; - private var eofReached: Boolean = false; - private var transcodeAudio = false; - private var muxerConfigured = false; - private var numberOfDecodedFrames = 0; - private var numberOfEncodedFrames = 0; - private var numberOfDecodedSamples = 0; - private var numberOfEncodedSamples = 0; - private var numberOfInputFrames = -1; - private var numberOfLostAudioFrames = 0; - - private var videoDecoder:MediaCodec? = null - private var audioDecoder:MediaCodec? = null - private var videoEncoder:MediaCodec? = null - private var audioEncoder:MediaCodec? = null - private var videoOutputStream:OutputStream? = null; - private var audioOutputStream:OutputStream? = null; - private var rawAudioOutputStream:OutputStream? = null; - private var resampledAudioOutputStream:OutputStream? = null; - var fileTranscoded = false - private var configured = false - private var nonStandardInputReasons:JSONArray = JSONArray() - private var inputFileDurationMs:Long = 0 // Ms - private var errorDescription = "" - - init { - if (uploadInfo.inputStandardization.maximumResolution == MaximumResolution.Preset3840x2160) { - this.MAX_ALLOWED_BITRATE = 20000000 - this.MAX_ALLOWED_I_FRAME_INTERVAL = 10 - this.MAX_ALLOWED_WIDTH = 4096 - this.MAX_ALLOWED_HEIGTH = 4096 - } else { - this.MAX_ALLOWED_BITRATE = 8000000 - this.MAX_ALLOWED_I_FRAME_INTERVAL = 5 - this.MAX_ALLOWED_WIDTH = uploadInfo.inputStandardization.maximumResolution.width - this.MAX_ALLOWED_HEIGTH = uploadInfo.inputStandardization.maximumResolution.height - } - } - - companion object { - const val LOG_TAG = "TranscoderContext" - - @JvmSynthetic - internal fun create(uploadInfo: UploadInfo, appContext: Context, sessionId: String): TranscoderContext { - return TranscoderContext(uploadInfo, appContext, sessionId) - } - } - - private fun getEncoders(mimeType: String, hwCapableOnly:Boolean): ArrayList { - val list = MediaCodecList(MediaCodecList.REGULAR_CODECS); - var result:ArrayList = ArrayList(); - for(codecInfo in list.codecInfos) { - logger.v("CodecInfo", codecInfo.name) - if(codecInfo.name.contains(mimeType) && codecInfo.isEncoder) { - if (!hwCapableOnly) { - result.add(codecInfo); - } else if (codecInfo.isHardwareAcceleratedCompat) { - result.add(codecInfo); - } - } - } - return result - } - - private fun configure() { - val cacheDir = File(appContext.cacheDir, "mux-upload") -// val cacheDir = File(appContext.externalCacheDir, "mux-upload") - cacheDir.mkdirs() - -// val videoOutput = File(cacheDir, "video.h264") -// videoOutputStream = videoOutput.outputStream() -// -// val audioOutput = File(cacheDir, "audio.aac") -// audioOutputStream = audioOutput.outputStream() -// -// val rawAudioOutput = File(cacheDir, "audio_original.raw") -// rawAudioOutputStream = rawAudioOutput.outputStream() -// -// val resampledAudioOutput = File(cacheDir, "audio_resampled.raw") -// resampledAudioOutputStream = resampledAudioOutput.outputStream() - - val destFile = File(cacheDir, UUID.randomUUID().toString() + ".mp4") -// val destFile = File(cacheDir, "output.mp4") - destFile.createNewFile() - - muxer = MediaMuxer(destFile.absolutePath, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4) - uploadInfo = uploadInfo.update(standardizedFile = destFile) - - try { - configureDecoders() - configureAudioEncoder() - configured = true - } catch (e:Exception) { - logger.e(LOG_TAG, "Failed to initialize.", e) - } - } - - private fun checkIfTranscodingIsNeeded(): Boolean { - var shouldStandardize = false - try { - extractor.setDataSource(uploadInfo.inputFile.absolutePath) - for (i in 0 until extractor.trackCount) { - val format = extractor.getTrackFormat(i) - val mime = format.getString(MediaFormat.KEY_MIME) - var inputDuration: Long = -1; - if (mime?.lowercase()?.contains("video") == true) { - inputWidth = format.getInteger(MediaFormat.KEY_WIDTH) - inputHeighth = format.getInteger(MediaFormat.KEY_HEIGHT) - // Check if resolution is greater then 720p - if ((inputWidth > MAX_ALLOWED_WIDTH && inputHeighth > MAX_ALLOWED_HEIGTH) - || (inputHeighth > MAX_ALLOWED_WIDTH && inputWidth > MAX_ALLOWED_HEIGTH) - ) { - logger.v(LOG_TAG, "Should standardize because the size is incorrect") - shouldStandardize = true - if (inputWidth > inputHeighth) { - targetedWidth = MAX_ALLOWED_WIDTH - targetedHeight = targetedWidth * (inputHeighth / inputWidth) - } else { - targetedHeight = MAX_ALLOWED_WIDTH - targetedWidth = targetedHeight * (inputWidth / inputHeighth) - } - nonStandardInputReasons.put("video_resolution") - } else { - targetedWidth = inputWidth - targetedHeight = inputHeighth - } - scaledSizeYuv = Nv12Buffer.allocate(targetedWidth, targetedHeight) - - // Check if compersion is h264 - if (!mime.equals(MediaFormat.MIMETYPE_VIDEO_AVC)) { - logger.v(LOG_TAG, "Should standardize because the input is not h.264") - shouldStandardize = true - nonStandardInputReasons.put("video_codec") - } - inputBitrate = format.getIntegerCompat(MediaFormat.KEY_BIT_RATE, -1) - inputDuration = format.getLongCompat(MediaFormat.KEY_DURATION, -1) - if (inputBitrate == -1 && inputDuration != -1L) { - inputBitrate = - ((uploadInfo.inputFile.length() * 8) / (inputDuration / 1000000)).toInt() - } - if (inputBitrate > MAX_ALLOWED_BITRATE) { - logger.v( - LOG_TAG, - "Should standardize because the input bitrate is too high" - ) - shouldStandardize = true - targetedBitrate = MAX_ALLOWED_BITRATE - nonStandardInputReasons.put("video_bitrate") - } - inputFramerate = format.getIntegerCompat(MediaFormat.KEY_FRAME_RATE, -1) - targetedFramerate = OPTIMAL_FRAMERATE - if (inputFramerate > MAX_ALLOWED_FRAMERATE || inputFramerate < MIN_ALLOWED_FRAMERATE) { - logger.v( - LOG_TAG, - "Should standardize because the input frame rate is too high" - ) - shouldStandardize = true - targetedFramerate = OPTIMAL_FRAMERATE - nonStandardInputReasons.put("video_framerate") - } else { - targetedFramerate = inputFramerate - } - val iFrameInterval = format.getInteger(MediaFormat.KEY_I_FRAME_INTERVAL) - if (iFrameInterval > MAX_ALLOWED_I_FRAME_INTERVAL) { - shouldStandardize = true - targetedIFrameInterval = I_FRAME_INTERVAL - } else { - targetedIFrameInterval = iFrameInterval - } - videoTrackIndex = i; - inputVideoFormat = format; - extractor.selectTrack(i) - } - if (mime?.lowercase()?.contains("audio") == true) { - // check if audio need to be standardized - audioTrackIndex = i; - inputAudioFormat = format; - extractor.selectTrack(i) - inputChannelCount = format.getInteger(MediaFormat.KEY_CHANNEL_COUNT) - inputSamplerate = format.getInteger(MediaFormat.KEY_SAMPLE_RATE) - if (inputChannelCount > 2) { - // We do not support this - transcodeAudio = false - } else { - if (!mime.equals(MediaFormat.MIMETYPE_AUDIO_AAC)) { - transcodeAudio = true; - } -// if (format.getInteger(MediaFormat.KEY_SAMPLE_RATE) != 48000) { -// transcodeAudio = true; -// } - } - } - } - } catch (ex:Exception) { - logger.e( - LOG_TAG, - "Couldn't completely inspect input. Will standardize? $shouldStandardize", - ex - ) - } - return shouldStandardize - } - - private fun configureDecoders() { - // Init decoders and encoders - numberOfInputFrames = inputVideoFormat!!.getIntegerCompat("frame-count", -1) - videoDecoder = - MediaCodec.createDecoderByType(inputVideoFormat!!.getString(MediaFormat.KEY_MIME)!!) - videoDecoder!!.configure(inputVideoFormat, null, null, 0) - videoDecoder!!.start() - if (transcodeAudio) { - audioDecoder = - MediaCodec.createDecoderByType(inputAudioFormat!!.getString(MediaFormat.KEY_MIME)!!) - audioDecoder!!.configure(inputAudioFormat, null, null, 0) - audioDecoder!!.start() - } - } - - private fun configureAudioEncoder() { - if (transcodeAudio) { - outputAudioFormat = MediaFormat.createAudioFormat(MediaFormat.MIMETYPE_AUDIO_AAC, - inputSamplerate, inputChannelCount) - outputAudioFormat!!.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC) - outputAudioFormat!!.setInteger(MediaFormat.KEY_PROFILE, 2) - outputAudioFormat!!.setInteger(MediaFormat.KEY_BIT_RATE, OUTPUT_AUDIO_BITRATE) - outputAudioFormat!!.setInteger("max-bitrate", OUTPUT_AUDIO_BITRATE) - outputAudioFormat!!.setInteger("aac-format-adif", 0) - val audioEncoders = getEncoders("aac", false) - for (encoder in audioEncoders) { - try { - // TODO see the codec capabileties - val codecCap = encoder.getCapabilitiesForType(MediaFormat.MIMETYPE_AUDIO_AAC) - audioEncoder = MediaCodec.createByCodecName(encoder.name) - audioEncoder!!.configure(outputAudioFormat,null, null, MediaCodec.CONFIGURE_FLAG_ENCODE) - audioEncoder!!.start() - break - } catch (err:java.lang.Exception) { - logger.w(LOG_TAG, "Couldn't evaluate audio encoder ${encoder.name}. Skipping it", err) - } - } - } else { - outputAudioFormat = inputAudioFormat - } - } - - private fun configureVideoEncoder() { - // We will need this when we apply the image resize - decodedFrameWidth = videoDecoderOutputFormat!!.getInteger(MediaFormat.KEY_WIDTH) - decodedFrameHeight = videoDecoderOutputFormat!!.getInteger(MediaFormat.KEY_HEIGHT) - - outputVideoFormat = MediaFormat.createVideoFormat(MediaFormat.MIMETYPE_VIDEO_AVC, targetedWidth, targetedHeight) - // This is NV12 actually - outputVideoFormat!!.setInteger(MediaFormat.KEY_COLOR_FORMAT, - COLOR_FormatYUV420SemiPlanar) - - outputVideoFormat!!.setInteger(MediaFormat.KEY_ROTATION, inputVideoFormat!!.getInteger(MediaFormat.KEY_ROTATION)) - outputVideoFormat!!.setInteger( - MediaFormat.KEY_FRAME_RATE, - targetedFramerate - ) - outputVideoFormat!!.setInteger("slice-height", targetedHeight + targetedHeight/2); - outputVideoFormat!!.setInteger("stride", targetedWidth); - outputVideoFormat!!.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, targetedIFrameInterval) - outputVideoFormat!!.setInteger( - MediaFormat.KEY_BITRATE_MODE, - MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_VBR - ) - outputVideoFormat!!.setInteger(MediaFormat.KEY_BIT_RATE, targetedBitrate) - - - val encoders = getEncoders("avc", true) - for (encoder in encoders) { - try { - val codecCap = encoder.getCapabilitiesForType("video/avc") - for (profile in codecCap.profileLevels ) { - if (profile.profile == MediaCodecInfo.CodecProfileLevel.AVCProfileHigh ) { - outputVideoFormat!!.setInteger(MediaFormat.KEY_PROFILE, profile.profile); - outputVideoFormat!!.setInteger(MediaFormat.KEY_LEVEL, profile.level) - break - } - } - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) { - outputVideoFormat!!.setInteger(MediaFormat.KEY_QUALITY, codecCap.encoderCapabilities.qualityRange.upper) - } - videoEncoder = MediaCodec.createByCodecName(encoder.name) - // Check if B-frame encoding is supported - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { - if (codecCap.isFeatureSupported("FEATURE_B_FRAME")) { - // Enable B-frames by setting the appropriate parameter - outputVideoFormat!!.setInteger(MediaFormat.KEY_MAX_B_FRAMES, 2) - outputVideoFormat!!.setInteger(MediaFormat.KEY_OUTPUT_REORDER_DEPTH, 2) - } - } - videoEncoder!!.configure(outputVideoFormat,null, null, MediaCodec.CONFIGURE_FLAG_ENCODE) - break; - } catch (err:java.lang.Exception) { - logger.w(LOG_TAG, "Couldn't evaluate video encoder ${encoder.name}. Skipping it", err) - } - } - videoEncoder!!.start() - } - - private fun releaseCodecs() { - logger.v(LOG_TAG, "releaseCodecs(): called") - videoDecoder?.safeDispose() - audioDecoder?.safeDispose() - videoEncoder?.safeDispose() - audioEncoder?.safeDispose() - extractor.safeDispose() - muxer?.safeDispose() - } - - private fun configureMuxer() { - outputVideoTrackIndex = muxer!!.addTrack(videoEncoder!!.outputFormat) - muxer!!.setOrientationHint(inputVideoFormat!!.getInteger(MediaFormat.KEY_ROTATION)) - if (transcodeAudio) { - outputAudioTrackIndex = muxer!!.addTrack(audioEncoder!!.outputFormat) - } else { - // Audio copy if present - if (inputAudioFormat != null ){ - outputAudioTrackIndex = muxer!!.addTrack(inputAudioFormat!!) - } - } - - muxer!!.start() - } - - @JvmSynthetic - internal suspend fun process(): UploadInfo { - logger.v(LOG_TAG, "process() starting") - if (!checkIfTranscodingIsNeeded()) { - logger.i(LOG_TAG, "Standardization was not required. Skipping") - return uploadInfo - } - - logger.i(LOG_TAG, "Standardizing input") - configure() - if (!configured) { - logger.e( - LOG_TAG, - "Skipped: Components could not be configured. Check the logs for errors" - ) - return uploadInfo; - } - - val started = System.currentTimeMillis() - val metrics = UploadMetrics.create() - var maxStandardInputRes = "" - try { - while (!eofReached) { - if (extractor.sampleTrackIndex == audioTrackIndex) { - muxAudioFrame() - } else { - muxVideoFrame() - } - } - // Get queued frames from encoder/decoder when we reach EOF - videoDecoder!!.flush() - videoEncoder!!.flush() - muxVideoFrame() - muxer!!.stop() - fileTranscoded = true - } catch (err:Exception) { - err.message?.let { errorDescription += it } - logger.e(LOG_TAG, "Failed to standardize input file ${uploadInfo.inputFile}", err) - } finally { - releaseCodecs() - } - val ended = System.currentTimeMillis(); - val duration = ended - started - logger.i(LOG_TAG, "Transcoding duration time: $duration") - logger.i(LOG_TAG, "Original file size: ${uploadInfo.inputFile.length()}") - logger.i(LOG_TAG, "Transcoded file size: ${uploadInfo.standardizedFile?.length()}") - maxStandardInputRes = (MAX_ALLOWED_WIDTH / MAX_ALLOWED_HEIGTH).toString() - if (fileTranscoded && !uploadInfo.optOut) { - metrics.reportStandardizationSuccess(started, ended, inputFileDurationMs, - nonStandardInputReasons, maxStandardInputRes, sessionId, uploadInfo) - } else if(!uploadInfo.optOut) { - metrics.reportStandardizationFailed(started, ended, inputFileDurationMs, - errorDescription, nonStandardInputReasons, maxStandardInputRes, sessionId, uploadInfo) - } - return uploadInfo - } - - private fun muxVideoFrame() { - val frames = getVideoFrames() - for (frame in frames) { -// if (frame.isBFrame()) { -// Log.i("Muxer", "We got B frame"); -// } - if (frame.isKeyFrame()) { - logger.i( - "Muxer", "Muxed video sample, size: " + frame.info.size - + ", pts: " + frame.info.presentationTimeUs - ) - } - inputFileDurationMs = frame.info.presentationTimeUs * 1000; - muxer!!.writeSampleData(outputVideoTrackIndex, frame.buff, frame.info) - } - } - - private fun muxAudioFrame() { - if (transcodeAudio) { - if (!eofReached) { - // This will advance the extractor - feedAudioDecoder() - } - val decodedFrames = getDecodedAudioFrame() - for (decoded in decodedFrames ) { - feedAudioEncoder(decoded) - decoded.release() - } - // iterate encoded audio frames and mux them - val encodedAudioFrames = getEncodedAudioFrames() - for(frame:AVFrame in encodedAudioFrames) { - if (outputAudioTrackIndex == -1) { - // Muxer not initialized yet, store these and mux later - audioFrames.add(frame) - } else { - // if we have some accumulated audio samples write them first - for (queuedFrame in audioFrames) { - muxAudioFrame(queuedFrame) - } - audioFrames.clear() - muxAudioFrame(frame) - } - } - } else { - copyAudioFrame(); - } - } - - private fun muxAudioFrame(frame:AVFrame) { - inputFileDurationMs = frame.info.presentationTimeUs * 1000; - muxer!!.writeSampleData( - outputAudioTrackIndex, - frame.buff, - frame.info - ) -// Log.i( -// "Muxer", "Muxed audio sample, size: " + frame.info.size -// + ", pts: " + frame.info.presentationTimeUs -// ) - } - - private fun getVideoFrames() : ArrayList { - if (!eofReached) { - // This will advance the extractor - feedVideoDecoder() - } - val decodedFrames = getDecodedVideoFrame() - for (decoded in decodedFrames ) { - feedVideoEncoder(decoded); - decoded.release() - } - return getEncodedVideoFrames() - } - - private fun feedVideoDecoder() { - val inIndex: Int = videoDecoder!!.dequeueInputBuffer(dequeueTimeout) - if (inIndex >= 0) { - val buffer: ByteBuffer = videoDecoder!!.getInputBuffer(inIndex)!!; - val sampleSize = extractor.readSampleData(buffer, 0) - if (sampleSize < 0) { - // We have reached the end of video - eofReached = true; -// return null - } else { - videoDecoder!!.queueInputBuffer(inIndex, 0, sampleSize, extractor.sampleTime, 0) - extractor.advance() - } - } - } - - private fun getDecodedVideoFrame():ArrayList { - var info = BufferInfo() - var outputBuffer:ByteBuffer? = null - val result = ArrayList() - var outIndex = videoDecoder!!.dequeueOutputBuffer(info, dequeueTimeout); - while(outIndex > 0) { - outputBuffer = videoDecoder!!.getOutputBuffer(outIndex); - result.add(AVFrame( - outIndex, outputBuffer!!, info, decodedFrameWidth, decodedFrameHeight, - videoDecoder!!, true - )) - numberOfDecodedFrames++ - outIndex = videoDecoder!!.dequeueOutputBuffer(info, dequeueTimeout); - } - when (outIndex) { - MediaCodec.INFO_OUTPUT_FORMAT_CHANGED -> { - // This give us real image height, to avoid corruptions in video - videoDecoderOutputFormat = videoDecoder!!.outputFormat; - configureVideoEncoder() - } - MediaCodec.INFO_TRY_AGAIN_LATER -> { - // Timedout also not good - } - } - return result - } - - private fun feedVideoEncoder(rawInput:AVFrame) { - val inIndex: Int = videoEncoder!!.dequeueInputBuffer(dequeueTimeout) - if (inIndex >= 0) { - // Scale input to match output - rawInput.yuvBuffer!!.scale(scaledSizeYuv!!, FilterMode.BILINEAR) - val buffer: ByteBuffer = videoEncoder!!.getInputBuffer(inIndex)!!; - buffer.clear() - scaledSizeYuv!!.write(buffer) - videoEncoder!!.queueInputBuffer(inIndex, 0, - buffer.capacity(), rawInput.info.presentationTimeUs, 0) - } - } - - fun getEncodedVideoFrames():ArrayList { - val result = ArrayList() - if (videoEncoder == null) { - return result; - } - var info = BufferInfo() - var outIndex = videoEncoder!!.dequeueOutputBuffer(info, dequeueTimeout) - var outputBuffer:ByteBuffer? - while (outIndex >= 0) { - if (!muxerConfigured) { - configureMuxer() - muxerConfigured = true; - } - outputBuffer = videoEncoder!!.getOutputBuffer(outIndex) - val buff = ByteBuffer.allocate(info.size) - outputBuffer!!.get(buff.array(), 0, info.size) - result.add(AVFrame(outIndex, buff, info, 0, 0, - videoEncoder, true, false)) - numberOfEncodedFrames++ - videoEncoder!!.releaseOutputBuffer(outIndex, false) - info = BufferInfo() - outIndex = videoEncoder!!.dequeueOutputBuffer(info, dequeueTimeout) - } - return result; - } - - ///////////////////////////////////////////////////////////////////////////////////////////// - ///////////// Audio //////////////////////////////////////////////////////////////////////// - - private fun copyAudioFrame() { - val audioFrame = getNextAudioFrame() - if (outputAudioTrackIndex == -1) { - // Muxer not initialized yet, store these and mux later - audioFrames.add(audioFrame!!) - } else { - // if we have some accumulated audio samples write them first - for (audioFrame in audioFrames) { - muxAudioFrame(audioFrame) - } - audioFrames.clear() - muxAudioFrame(audioFrame!!) - } - } - - private fun getNextAudioFrame(): AVFrame? { - val extractorBuffer:ByteBuffer = ByteBuffer.allocate(1024) - val extractedFrame = AVFrame(-1, extractorBuffer, BufferInfo(), isRaw = false) - val sampleSize = extractor.readSampleData(extractorBuffer, 0); - if (sampleSize == -1) { - eofReached = true; - return null; - } else { - extractedFrame.info.size = sampleSize - extractedFrame.info.presentationTimeUs = extractor.sampleTime - extractor.advance() - } - return extractedFrame; - } - - private fun feedAudioDecoder() { - val inIndex: Int = audioDecoder!!.dequeueInputBuffer(dequeueTimeout) - if (inIndex >= 0) { - val buffer: ByteBuffer = audioDecoder!!.getInputBuffer(inIndex)!!; - val sampleSize = extractor.readSampleData(buffer, 0) - if (sampleSize < 0) { - // We have reached the end of video - eofReached = true; - } else { - audioDecoder!!.queueInputBuffer(inIndex, 0, sampleSize, extractor.sampleTime, 0) - extractor.advance() - } - } - } - - private fun getDecodedAudioFrame():ArrayList { - var info = BufferInfo() - var outputBuffer:ByteBuffer? = null - val result = ArrayList() - var outIndex = audioDecoder!!.dequeueOutputBuffer(info, dequeueTimeout); - while(outIndex > 0) { - outputBuffer = audioDecoder!!.getOutputBuffer(outIndex); - result.add(AVFrame( - outIndex, outputBuffer!!, info, 0, 0, - audioDecoder!!, true - )) - numberOfDecodedSamples++ - outIndex = audioDecoder!!.dequeueOutputBuffer(info, dequeueTimeout); - } - when (outIndex) { - MediaCodec.INFO_OUTPUT_FORMAT_CHANGED -> { - // Not sure what to do here - audioDecoderOutputFormat = audioDecoder!!.outputFormat; -// configureEncoders() - } - MediaCodec.INFO_TRY_AGAIN_LATER -> { - // Timedout also not good - } - } - return result - } - - private fun feedAudioEncoder(rawInput:AVFrame) { - rawInput.buff.rewind() - // Resampling not working as expected, we need a better solution -// var tmp:ByteBuffer = ByteBuffer.allocate(rawInput.info.size) -// tmp.put(rawInput.buff) -// rawInput.buff.rewind() -// tmp.rewind() -// rawAudioOutputStream!!.write(tmp.array(), 0, tmp.remaining()) -// tmp.rewind() -// val resampled = resample.resample(tmp.array(), rawInput.info.size, true, inputSamplerate, OUTPUT_SAMPLERATE) -//// val output_len = resample.resampleEx(rawInput.buff, buffer, rawInput.buff.remaining()) -// if (resampled == null) { -// logger.i(LOG_TAG, "It is a problem :-D") -// } -// resampledAudioOutputStream!!.write(resampled, 0, resampled!!.size) -// var bytesQueued = 0; - -// while (bytesQueued < resampled!!.size) { - val inIndex: Int = audioEncoder!!.dequeueInputBuffer(dequeueTimeout) - if (inIndex >= 0) { - // resample input to match output - val buffer: ByteBuffer = audioEncoder!!.getInputBuffer(inIndex)!! - buffer.rewind() -// val remaining = resampled!!.size - bytesQueued -// var toWrite = buffer.capacity() -// if (remaining < buffer.capacity()) { -// toWrite = remaining -// } - buffer.put(rawInput.buff) - buffer.rewind() - audioEncoder!!.queueInputBuffer( - inIndex, 0, - buffer.remaining(), rawInput.info.presentationTimeUs, 0 - ) -// bytesQueued+=toWrite - } else { - logger.e(LOG_TAG, "We lost audio frame :-D") - numberOfLostAudioFrames++; - } -// } - } - - private fun getEncodedAudioFrames():ArrayList { - val result = ArrayList() - if (audioEncoder == null) { - return result; - } - var info = BufferInfo() - var outIndex = audioEncoder!!.dequeueOutputBuffer(info, dequeueTimeout) - var outputBuffer:ByteBuffer? - while (outIndex >= 0) { - if (!muxerConfigured) { - // TODO maybe note that audio is ready to be configured in a muxer - } - outputBuffer = audioEncoder!!.getOutputBuffer(outIndex) - val buff = ByteBuffer.allocate(info.size) - outputBuffer!!.get(buff.array(), 0, info.size) - result.add(AVFrame(outIndex, buff, info, 0, 0, - audioEncoder, true, false)) - numberOfEncodedSamples++ - audioEncoder!!.releaseOutputBuffer(outIndex, false) - info = BufferInfo() - outIndex = audioEncoder!!.dequeueOutputBuffer(info, dequeueTimeout) - } - return result; - } - - //////////////////////////////////////////////////////////////////////////////////////////////// - ///////////////////////////// Helpers ////////////////////////////////////////////////////////// - - private fun findAnnexBPosition(buff:ByteBuffer, startSearchAt:Int, buffSize:Int): Int { - // We are assuming integer is 4 bytes on every device, we also assume anexB is 4 bytes long - // instead of 3 which is also possible sometimes - for(i in startSearchAt..buffSize - 4) { - if (buff.getInt(i) == 1) { - return i; - } - } - return -1 - } - private fun convertAnnexBtoAvcc(buff:ByteBuffer, buffSize:Int) { - val positions = ArrayList() - var annexBPos = findAnnexBPosition(buff, 0, buffSize) - while (annexBPos != -1) { - positions.add(annexBPos) - annexBPos = findAnnexBPosition(buff, annexBPos + 4, buffSize) - } - for (i in 0..positions.size -1) { - var naluLength = 0 - if (i == positions.size -1) { - // This is the last position - naluLength = buffSize - positions.get(i) - 4 - } else { - naluLength = positions.get(i + 1) - positions.get(i) -4; - } - buff.position(positions.get(i)) - buff.putInt(naluLength) - } - } - - //////////////////////////////////////////////////////////////////////////////////////////////// - //////////////////////////////////////////////////////////////////////////////////////////////// - - - class AVFrame constructor(val index:Int, val buff:ByteBuffer, val info:BufferInfo, - val width:Int = 0, val heigth:Int = 0, - val codec:MediaCodec? = null, val shouldRelease:Boolean = true, val isRaw:Boolean = true){ - - // TODO support other color formats, NV12 is default decoder format for AVC - var yuvBuffer:Nv12Buffer? = null; - - init { - if (isRaw) { - yuvBuffer = Nv12Buffer.wrap(buff, width, heigth) - } - } - - fun release() { - if (shouldRelease) { - codec?.releaseOutputBuffer(index, false); - } - } - - fun getNalType(): Int { - return (buff.get(4) and 0x1F).toInt() - } - - fun isBFrame(): Boolean { - val nalType = getNalType() - return (nalType == 2 || nalType == 3 || nalType == 4) - } - - fun isKeyFrame(): Boolean { - val nalType = getNalType() - // Sometimes key frame is packed with pps and sps - return (nalType == 5 || nalType == 7 || nalType == 8) - } - - fun clone():AVFrame { - val buffCopy = ByteBuffer.allocate(info.size) - buffCopy.get(buff.array(), 0, info.size) - val infoCopy = BufferInfo() - infoCopy.size = info.size - infoCopy.offset = info.offset - infoCopy.presentationTimeUs = info.presentationTimeUs - infoCopy.flags = info.flags - return AVFrame(index, buffCopy, infoCopy, width, heigth, codec, shouldRelease, isRaw) - } - } -} diff --git a/library/src/main/java/com/mux/video/upload/internal/UploadAttempt.kt b/library/src/main/java/com/mux/video/upload/internal/UploadAttempt.kt index c665a633..8f5305dc 100644 --- a/library/src/main/java/com/mux/video/upload/internal/UploadAttempt.kt +++ b/library/src/main/java/com/mux/video/upload/internal/UploadAttempt.kt @@ -4,6 +4,12 @@ import com.mux.video.upload.api.MuxUpload import com.mux.video.upload.api.UploadStatus import com.mux.video.upload.internal.standardization.SdrGeneratedFile import kotlinx.coroutines.flow.MutableStateFlow +import java.util.UUID + +/** Handles for one destination follow replacements through this shared record. */ +internal class UploadSession { + val current = MutableStateFlow(null) +} /** Shared across in-process pause/resume, never reconstructed as a generated payload from disk. */ internal class UploadPreparationState { @@ -28,7 +34,9 @@ internal class UploadPreparationState { internal class UploadAttempt( val preparation: UploadPreparationState, initialProgress: MuxUpload.Progress, + val previousPersistenceOwnerId: String? = null, ) { + val id: String = UUID.randomUUID().toString() val status = MutableStateFlow(UploadStatus.Started) private var stopped = false private var cancelled = false @@ -61,10 +69,10 @@ internal class UploadAttempt( @Synchronized fun beginTransport(generated: Boolean, persist: () -> Unit) { if (stopped) throw kotlinx.coroutines.CancellationException("Upload stopped") - if (generated) { - preparation.generatedRequestStarted = true - persist() - } + if (generated) preparation.generatedRequestStarted = true + else preparation.originalSelected = true + // Persist payload selection before the first request, even without an acknowledgement. + persist() } @Synchronized fun publish(value: UploadStatus) { @@ -72,7 +80,7 @@ internal class UploadAttempt( } @Synchronized fun acknowledge(progress: MuxUpload.Progress, persist: () -> Unit) { - if (stopped || terminal) return + if (stopped || terminal || superseded) return confirmed = progress persist() status.value = UploadStatus.Uploading(progress) @@ -85,16 +93,18 @@ internal class UploadAttempt( persist(confirmed) } - @Synchronized fun cancel(forget: () -> Unit) { - if (cancelled) return - stopped = true - cancelled = true - forget() + fun cancel(forget: () -> Unit) { + synchronized(this) { + if (cancelled) return + stopped = true + cancelled = true + forget() + } preparation.deleteOwnedFile() } @Synchronized fun finish(value: UploadStatus): Boolean { - if (stopped || terminal) return false + if (stopped || terminal || superseded) return false terminal = true status.value = value return true @@ -104,3 +114,5 @@ internal class UploadAttempt( internal class GeneratedResumeBlockedException : IllegalStateException( "Generated payload identity or server offset is unverified. Create a new Direct Upload." ) + +internal class UploadCancelledException : IllegalStateException("Upload was cancelled. Create a new upload handle.") diff --git a/library/src/main/java/com/mux/video/upload/internal/UploadInfo.kt b/library/src/main/java/com/mux/video/upload/internal/UploadInfo.kt index 592e181e..52fef0a0 100644 --- a/library/src/main/java/com/mux/video/upload/internal/UploadInfo.kt +++ b/library/src/main/java/com/mux/video/upload/internal/UploadInfo.kt @@ -82,7 +82,6 @@ internal data class UploadInfo( @JvmSynthetic internal val inputStandardization: InputStandardization = InputStandardization(), @JvmSynthetic internal val remoteUri: Uri, @JvmSynthetic internal val inputFile: File, - @JvmSynthetic internal val standardizedFile: File? = null, @JvmSynthetic internal val chunkSize: Int, @JvmSynthetic internal val retriesPerChunk: Int, @JvmSynthetic internal val optOut: Boolean, @@ -92,6 +91,8 @@ internal data class UploadInfo( @JvmSynthetic internal val restoredFromOriginal: Boolean = false, @JvmSynthetic internal val generatedResumeBlocked: Boolean = false, ) { + internal var session = UploadSession() + fun isRunning(): Boolean = attempt?.isStopped() != true && (statusFlow?.value?.let { it is UploadStatus.Uploading || it is UploadStatus.Started || it is UploadStatus.Preparing @@ -108,7 +109,6 @@ internal fun UploadInfo.update( inputStandardization: InputStandardization = this.inputStandardization, remoteUri: Uri = this.remoteUri, file: File = this.inputFile, - standardizedFile: File? = this.standardizedFile, chunkSize: Int = this.chunkSize, retriesPerChunk: Int = this.retriesPerChunk, optOut: Boolean = this.optOut, @@ -121,7 +121,6 @@ internal fun UploadInfo.update( inputStandardization, remoteUri, file, - standardizedFile, chunkSize, retriesPerChunk, optOut, @@ -130,4 +129,4 @@ internal fun UploadInfo.update( attempt, restoredFromOriginal, generatedResumeBlocked, -) +).also { it.session = session } diff --git a/library/src/main/java/com/mux/video/upload/internal/UploadJobFactory.kt b/library/src/main/java/com/mux/video/upload/internal/UploadJobFactory.kt index 9e996aa9..e6f36d3b 100644 --- a/library/src/main/java/com/mux/video/upload/internal/UploadJobFactory.kt +++ b/library/src/main/java/com/mux/video/upload/internal/UploadJobFactory.kt @@ -35,13 +35,14 @@ internal class UploadJobFactory internal constructor( fun createUploadJob(uploadInfo: UploadInfo, outerScope: CoroutineScope): UploadInfo { uploadInfo.attempt?.supersede() + val saved = readUploadResumeState(uploadInfo) val preparation = uploadInfo.attempt?.preparation ?: UploadPreparationState().apply { - originalSelected = uploadInfo.restoredFromOriginal || hasOriginalResumeState(uploadInfo) + originalSelected = uploadInfo.restoredFromOriginal || saved.originalSelected } val startTime = System.currentTimeMillis() val attempt = UploadAttempt(preparation, MuxUpload.Progress( - bytesUploaded = readLastByteForFile(uploadInfo), totalBytes = uploadInfo.inputFile.length(), - startTime = startTime, updatedTime = startTime)) + bytesUploaded = if (saved.generatedResumeBlocked) 0 else saved.bytesSent, totalBytes = uploadInfo.inputFile.length(), + startTime = startTime, updatedTime = startTime), previousPersistenceOwnerId = saved.attemptId) var runningInfo = uploadInfo.update(attempt = attempt, statusFlow = attempt.status.asStateFlow()) // Lazy startup lets the returned identity be installed before any completion callback. val job = outerScope.async(start = CoroutineStart.LAZY) { @@ -51,7 +52,7 @@ internal class UploadJobFactory internal constructor( // An old export or request must finish cancellation before another attempt uses its files. uploadInfo.uploadJob?.join() ensureActive() - if (uploadInfo.generatedResumeBlocked || preparation.generatedRequestStarted || uploadInfo.standardizedFile != null) + if (uploadInfo.generatedResumeBlocked || saved.generatedResumeBlocked || preparation.generatedRequestStarted) throw GeneratedResumeBlockedException() if (!preparation.originalSelected && preparation.verified == null && uploadInfo.isStandardizationRequested()) { attempt.publish(UploadStatus.Preparing) @@ -135,9 +136,15 @@ internal class UploadJobFactory internal constructor( } catch (e: Exception) { val failure = UploadStatus.UploadFailed(e, attempt.confirmedProgress()) if (attempt.finish(failure)) { - MuxUploadSdk.logger.e("MuxUpload", "Upload failed") + val category = when (e) { + is GeneratedResumeBlockedException -> "GeneratedResumeBlocked" + is java.io.IOException -> "Io" + is IllegalStateException, is IllegalArgumentException -> "InvalidPayload" + else -> "Unexpected" + } + MuxUploadSdk.logger.e("MuxUpload", "Upload failed: $category") if (!uploadInfo.optOut) metrics.reportUploadFailed(startTime, System.currentTimeMillis(), 0, - "Upload failed", sessionId, uploadInfo) + "Upload failed: $category", sessionId, uploadInfo) withContext(Dispatchers.Main) { MuxUploadManager.jobFinished(runningInfo, false) } } Result.failure(e) @@ -146,6 +153,7 @@ internal class UploadJobFactory internal constructor( } } runningInfo = runningInfo.update(uploadJob = job) + runningInfo.session.current.value = runningInfo job.start() return runningInfo } diff --git a/library/src/main/java/com/mux/video/upload/internal/UploadPersistence.kt b/library/src/main/java/com/mux/video/upload/internal/UploadPersistence.kt index 956fddf1..e9e804d1 100644 --- a/library/src/main/java/com/mux/video/upload/internal/UploadPersistence.kt +++ b/library/src/main/java/com/mux/video/upload/internal/UploadPersistence.kt @@ -9,6 +9,7 @@ import org.json.JSONArray import org.json.JSONObject import java.io.File import java.util.* +import java.security.MessageDigest @JvmSynthetic internal fun initializeUploadPersistence(appContext: Context) { @@ -32,22 +33,28 @@ internal fun writeUploadState(uploadInfo: UploadInfo, state: MuxUpload.Progress) retriesPerChunk = uploadInfo.retriesPerChunk, optOut = uploadInfo.optOut, inputStandardization = uploadInfo.inputStandardization, + originalSelected = uploadInfo.attempt?.preparation?.originalSelected ?: true, + attemptId = uploadInfo.attempt?.id, generatedResumeBlocked = uploadInfo.attempt?.preparation?.generatedRequestStarted == true || uploadInfo.generatedResumeBlocked, ) ) } +internal data class UploadResumeState( + val bytesSent: Long = 0, + val originalSelected: Boolean = false, + val generatedResumeBlocked: Boolean = false, + val paused: Boolean = false, + val attemptId: String? = null, +) + +internal fun readUploadResumeState(upload: UploadInfo): UploadResumeState = UploadPersistence.readState(upload) + @JvmSynthetic -internal fun readLastByteForFile(upload: UploadInfo): Long { - val entry = UploadPersistence.readEntries()[upload.inputFile.absolutePath] - return entry?.takeIf { it.url == upload.remoteUri.toString() }?.bytesSent ?: 0 +internal fun readLastByteForFile(upload: UploadInfo): Long = readUploadResumeState(upload).let { + if (it.generatedResumeBlocked) 0 else it.bytesSent } -internal fun hasOriginalResumeState(upload: UploadInfo): Boolean = - UploadPersistence.readEntries()[upload.inputFile.absolutePath]?.let { - it.url == upload.remoteUri.toString() && !it.generatedResumeBlocked - } == true - @JvmSynthetic internal fun forgetUploadState(uploadInfo: UploadInfo) { UploadPersistence.removeForFile(uploadInfo) @@ -68,7 +75,7 @@ internal fun readAllCachedUploads(includePaused: Boolean = true): List) { + private fun writeEntries(entries: Map, blocks: Set) { val entriesJson = JSONArray() entries.forEach { entriesJson.put(it.value.toJson()) } - prefs.edit().putString(LIST_KEY, entriesJson.toString()).apply() + prefs.edit().putString(LIST_KEY, entriesJson.toString()) + .putString(BLOCKS_KEY, JSONArray(blocks.toList()).toString()).apply() } @Throws @@ -143,6 +175,15 @@ private object UploadPersistence { } } + private fun destinationKey(url: String): String = MessageDigest.getInstance("SHA-256") + .digest(url.toByteArray(Charsets.UTF_8)) + .joinToString("") { "%02x".format(it) } + + private fun readBlocks(): MutableSet { + val json = JSONArray(prefs.getString(BLOCKS_KEY, null) ?: "[]") + return (0 until json.length()).mapTo(mutableSetOf()) { json.getString(it) } + } + private fun checkInitialized() { if (!this::prefs.isInitialized) { throw IllegalStateException( @@ -164,12 +205,16 @@ private data class UploadEntry( val bytesSent: Long, val inputStandardization: InputStandardization, val generatedResumeBlocked: Boolean = false, + val originalSelected: Boolean = true, + val attemptId: String? = null, ) { fun toJson(): JSONObject { return JSONObject().apply { put("file", file.absolutePath) put("data", JSONObject().apply { put("generated_resume_blocked", generatedResumeBlocked) + put("original_selected", originalSelected) + put("attempt_id", attemptId) put("url", url) put("chunk_size", chunkSize) put("retries_per_chunk", retriesPerChunk) @@ -200,6 +245,8 @@ private fun JSONObject.parsePersistenceEntry(): UploadEntry { state = data.optInt("state"), bytesSent = data.optLong("bytes_sent"), generatedResumeBlocked = data.optBoolean("generated_resume_blocked", false), + originalSelected = data.optBoolean("original_selected", true), + attemptId = data.optString("attempt_id").takeIf { it.isNotEmpty() }, inputStandardization = data.optJSONObject("input_standardization").let { options -> InputStandardization( standardizationRequested = options?.optBoolean("requested", true) ?: true, diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt index 1a91fbd9..cea9a9a3 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt @@ -1,5 +1,6 @@ package com.mux.video.upload.internal.standardization +import kotlinx.coroutines.CompletableDeferred import android.content.Context import android.os.Handler import android.os.HandlerThread @@ -91,6 +92,7 @@ internal class SdrConversionAdapter( private val cancelled = AtomicBoolean(false) private val terminalLock = Any() private var terminal = false + private val released = CompletableDeferred() private var engine: SdrExportEngine? = null private var output: SdrGeneratedFile? = null /** Exact deletion authority retained if the OS refuses cleanup, including cancellation. */ @@ -107,6 +109,8 @@ internal class SdrConversionAdapter( handler.post { finish(SdrConversionResult.Cancelled) } } + suspend fun awaitRelease() { released.await() } + internal fun begin() { handler.post { guarded { if (cancelled.get()) return@guarded finish(SdrConversionResult.Cancelled) @@ -183,6 +187,7 @@ internal class SdrConversionAdapter( engine = null handler.removeCallbacksAndMessages(null) thread.quitSafely() + released.complete(Unit) } } } diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/UploadPreparation.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/UploadPreparation.kt index 76759c98..12e8d0f8 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/UploadPreparation.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/UploadPreparation.kt @@ -3,6 +3,8 @@ package com.mux.video.upload.internal.standardization import android.content.Context import com.mux.video.upload.internal.UploadInfo import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.NonCancellable +import kotlinx.coroutines.isActive import kotlinx.coroutines.ensureActive import kotlinx.coroutines.suspendCancellableCoroutine import kotlinx.coroutines.withContext @@ -32,7 +34,8 @@ internal class UploadPreparation( { file, metadata, cancelled -> StandardInputTimelineInspector().inspect(file, metadata, cancelled) }, private val convert: suspend (Context, UploadInfo, MediaMetadataInspection, MediaSampleInspection, StandardInputConversion) -> SdrConversionResult = { context, upload, metadata, source, conversion -> - suspendCancellableCoroutine { continuation -> + var export: SdrConversionAdapter.Attempt? = null + try { suspendCancellableCoroutine { continuation -> val attempt = SdrConversionAdapter(context, onPendingCleanup = { upload.attempt?.trackCleanup(it) }) .start(upload.inputFile, metadata, source, conversion) { result -> // Transfer ownership before dispatch: completion can race pause or cancellation. @@ -41,7 +44,10 @@ internal class UploadPreparation( if (value is SdrConversionResult.Completed && upload.attempt == null) value.output.delete() } } + export = attempt continuation.invokeOnCancellation { attempt.cancel() } + } } finally { + if (!coroutineContext.isActive) withContext(NonCancellable) { export?.awaitRelease() } } }, ) { @@ -49,6 +55,7 @@ internal class UploadPreparation( generatedResumeVerified: Boolean = false): PreparedUpload = withContext(Dispatchers.IO) { fun original(reason: PreparationDiagnostic) = PreparedUpload.Original(reason) if (!upload.isStandardizationRequested()) return@withContext original(PreparationDiagnostic.Original) + if (!generatedResumeVerified) return@withContext original(PreparationDiagnostic.GeneratedResumeUnavailable) val job = coroutineContext try { val metadata = (inspectMetadata(upload.inputFile) as? MetadataInspectionResult.Success)?.inspection @@ -60,14 +67,13 @@ internal class UploadPreparation( if (action is StandardInputAction.UploadOriginal) return@withContext original(PreparationDiagnostic.Original) // The adapter provides source-specific preflight and proves actual output. Planning keeps // unsupported HDR and unsafe structure out of this SDR boundary. - val conversion = (action as? StandardInputAction.Fallback)?.reason.let { - (it as? FallbackReason.UnsupportedConversion)?.conversion + val conversion = when (action) { + is StandardInputAction.Convert -> action.conversion + is StandardInputAction.Fallback -> (action.reason as? FallbackReason.UnsupportedConversion)?.conversion + is StandardInputAction.UploadOriginal -> null } ?: return@withContext original(PreparationDiagnostic.UnsupportedPlan) if (SdrEncodingTargets.from(conversion, metadata, source) == null) return@withContext original(PreparationDiagnostic.UnsupportedPlan) - // Until the Mux status/offset protocol is verified, new attempts upload original bytes. - // This gate also avoids spending conversion time on an unusable replacement payload. - if (!generatedResumeVerified) return@withContext original(PreparationDiagnostic.GeneratedResumeUnavailable) when (val result = convert(context, upload, metadata, source, conversion)) { is SdrConversionResult.Completed -> PreparedUpload.Generated(result.output) is SdrConversionResult.Failed -> original(PreparationDiagnostic.ConversionFailed(result.reason)) diff --git a/library/src/test/java/com/mux/video/upload/internal/UploadPreparationLifecycleTests.kt b/library/src/test/java/com/mux/video/upload/internal/UploadPreparationLifecycleTests.kt index ccbac73c..969abeb5 100644 --- a/library/src/test/java/com/mux/video/upload/internal/UploadPreparationLifecycleTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/UploadPreparationLifecycleTests.kt @@ -111,6 +111,7 @@ class UploadPreparationLifecycleTests : AbsRobolectricTest() { val json = org.json.JSONArray(prefs.getString("uploads", null)) json.getJSONObject(0).getJSONObject("data").remove("generated_resume_blocked") json.getJSONObject(0).getJSONObject("data").remove("input_standardization") + json.getJSONObject(0).getJSONObject("data").remove("original_selected") json.getJSONObject(0).getJSONObject("data").put("state", 0) prefs.edit().putString("uploads", json.toString()).commit() work = { _, _, _ -> awaitCancellation() } @@ -157,8 +158,9 @@ class UploadPreparationLifecycleTests : AbsRobolectricTest() { dispatcher.scheduler.runCurrent() assertEquals(1, chunks.size) assertEquals(1, readAllCachedUploads().size) - assertTrue(MuxUploadManager.allUploadJobs().isEmpty()) - val restored = MuxUpload.create(info()) + val restored = MuxUploadManager.findUploadByFile(source)!! + assertTrue(restored.isPaused) + assertEquals(1, MuxUploadManager.allUploadJobs().size) restored.start() pump { chunks.size == 2 } assertEquals(1, preparations) @@ -250,15 +252,19 @@ class UploadPreparationLifecycleTests : AbsRobolectricTest() { pump { upload.uploadStatus is UploadStatus.UploadFailed } assertTrue(upload.error is GeneratedResumeBlockedException) assertEquals(1, chunks.size) - val failedJob = internalInfo(upload).uploadJob - upload.start(forceRestart = true) - pump { internalInfo(upload).uploadJob !== failedJob && upload.uploadStatus is UploadStatus.UploadFailed } - assertTrue(upload.error is GeneratedResumeBlockedException) - assertEquals(1, chunks.size) assertTrue(readAllCachedUploads().single().generatedResumeBlocked) val restored = observe(MuxUploadManager.resumeAllCachedJobs().single()) pump { restored.uploadStatus is UploadStatus.UploadFailed } assertTrue(restored.error is GeneratedResumeBlockedException) + val failedJob = internalInfo(restored).uploadJob + restored.start(forceRestart = true) + pump { internalInfo(restored).uploadJob !== failedJob && restored.uploadStatus is UploadStatus.UploadFailed } + assertTrue(restored.error is GeneratedResumeBlockedException) + assertTrue(readUploadResumeState(info()).generatedResumeBlocked) + val fresh = MuxUpload.create(info()) + fresh.start() + pump { fresh.uploadStatus is UploadStatus.UploadFailed } + assertTrue(fresh.error is GeneratedResumeBlockedException) assertEquals(1, chunks.size) } @@ -321,7 +327,7 @@ class UploadPreparationLifecycleTests : AbsRobolectricTest() { } val upload = MuxUpload.create(info()) upload.start() - pump { upload.uploadStatus is UploadStatus.UploadFailed && MuxUploadManager.findUploadByFile(source) == null } + pump { upload.uploadStatus is UploadStatus.UploadFailed && MuxUploadManager.findUploadByFile(source) != null } assertEquals(4L, readLastByteForFile(info())) upload.cancel() assertTrue(readAllCachedUploads().isEmpty()) @@ -386,6 +392,244 @@ class UploadPreparationLifecycleTests : AbsRobolectricTest() { assertFalse(generated.file.exists()) } + + @Test fun unmanagedSuccessClearsResumeStateAndCannotBeAutomaticallyRestored() { + val upload = MuxUpload.Builder(info().remoteUri, source).manageUploadTask(false).build() + upload.start(); pump { upload.isSuccessful } + assertTrue(MuxUploadManager.allUploadJobs().isEmpty()) + assertTrue(readAllCachedUploads().isEmpty()) + val sent = chunks.size + MuxUploadSdk.initialize(context, true) + dispatcher.scheduler.runCurrent() + assertTrue(MuxUploadManager.allUploadJobs().isEmpty()) + assertEquals(sent, chunks.size) + } + + @Test fun freshHandleCannotUseBlockedGeneratedOffsetForOriginalBytes() { + val state = UploadPreparationState().apply { generatedRequestStarted = true } + val attempt = UploadAttempt(state, MuxUpload.Progress(bytesUploaded = 8, totalBytes = 16)) + val marked = info().update(attempt = attempt, statusFlow = attempt.status) + writeUploadState(marked, attempt.confirmedProgress()) + assertEquals(0L, readLastByteForFile(info())) + val upload = MuxUpload.create(info()) + upload.start(); pump { upload.uploadStatus is UploadStatus.UploadFailed } + assertTrue(upload.error is GeneratedResumeBlockedException) + assertTrue(chunks.isEmpty()) + assertEquals(0, preparations) + } + + @Test fun cancelledHandleReturnsFailureWithoutRestartingOrCancellingCaller() { + work = { _, _, _ -> awaitCancellation() } + val upload = MuxUpload.create(info()) + upload.start(); pump { chunks.isNotEmpty() } + val before = internalInfo(upload).uploadJob + upload.cancel(); upload.start(forceRestart = true) + assertSame(before, internalInfo(upload).uploadJob) + var result: Result? = null + val caller = scope.launch { result = upload.awaitSuccess() } + pump { caller.isCompleted } + assertTrue(result!!.exceptionOrNull() is UploadCancelledException) + assertFalse(caller.isCancelled) + assertTrue(scope.isActive) + assertEquals(1, chunks.size) + } + + @Test fun zeroBytePreparationPauseRunsPreparationAfterReconstruction() { + prepare = { awaitCancellation() } + val upload = MuxUpload.create(info()) + upload.start(); pump { preparations == 1 } + upload.pause(); dispatcher.scheduler.runCurrent() + val saved = readAllCachedUploads().single() + assertFalse(saved.restoredFromOriginal) + MuxUploadManager.jobFinished(internalInfo(upload), false) + prepare = { PreparedUpload.Original() } + val restored = MuxUploadManager.resumeAllCachedJobs().single() + pump { restored.isSuccessful } + assertEquals(2, preparations) + } + + @Test fun zeroAcknowledgedOriginalRequestRemainsOriginalAfterRestoration() { + work = { _, _, _ -> awaitCancellation() } + val upload = MuxUpload.create(info()) + upload.start(); pump { chunks.size == 1 } + upload.pause(); dispatcher.scheduler.runCurrent() + assertEquals(0L, readLastByteForFile(info())) + assertTrue(readAllCachedUploads().single().restoredFromOriginal) + MuxUploadManager.jobFinished(internalInfo(upload), false) + val restored = MuxUploadManager.resumeAllCachedJobs().single() + pump { chunks.size == 2 } + assertEquals(1, preparations) + assertTrue(restored.isRunning) + assertEquals(0L, chunks.last().startByte) + } + + @Test fun oldDestinationHandleCannotPauseCancelOrRestartItsReplacement() { + work = { _, _, _ -> awaitCancellation() } + val old = MuxUpload.create(info()) + old.start(); pump { chunks.size == 1 } + val replacement = MuxUpload.create(info().copy(remoteUri = Uri.parse("https://example.invalid/new-upload"))) + replacement.start(); pump { chunks.size == 2 } + old.pause(); old.start(forceRestart = true); old.cancel() + dispatcher.scheduler.runCurrent() + assertTrue(replacement.isRunning) + assertFalse(internalInfo(replacement).uploadJob!!.isCancelled) + assertTrue(MuxUploadManager.findUploadByFile(source)!!.isRunning) + assertEquals(2, chunks.size) + } + + @Test fun existingHandleFollowsResumedAttemptAndItsCompletion() { + work = { _, _, _ -> awaitCancellation() } + val old = observe(MuxUpload.create(info())) + old.start(); pump { chunks.size == 1 } + old.pause(); dispatcher.scheduler.runCurrent() + val second = MuxUploadManager.findUploadByFile(source)!! + second.start(); pump { chunks.size == 2 } + assertTrue(old.isRunning); assertFalse(old.isPaused) + work = { chunk, _, _ -> MuxUpload.Progress(bytesUploaded = chunk.contentLength.toLong()) } + second.pause(); second.start() + pump { old.isSuccessful } + assertTrue(second.isSuccessful) + assertEquals(1, results.size) + assertEquals(16L, old.currentProgress.bytesUploaded) + } + + @Test fun cancellingGeneratedRequestKeepsBlockAcrossNewDestinationAndReinitialization() { + val generated = generated(16) + prepare = { PreparedUpload.Generated(generated) } + work = { _, _, _ -> awaitCancellation() } + val old = MuxUpload.create(info()) + old.start(); pump { chunks.size == 1 } + old.cancel(); dispatcher.scheduler.runCurrent() + assertTrue(readAllCachedUploads().isEmpty()) + assertFalse(generated.file.exists()) + prepare = { PreparedUpload.Original() } + val next = MuxUpload.create(info().copy(remoteUri = Uri.parse("https://example.invalid/new-upload"))) + next.start(); pump { chunks.size == 2 } + next.cancel(); dispatcher.scheduler.runCurrent() + MuxUploadSdk.initialize(context, false) + val reused = MuxUpload.create(info()) + reused.start(); pump { reused.uploadStatus is UploadStatus.UploadFailed } + assertTrue(reused.error is GeneratedResumeBlockedException) + assertEquals(2, chunks.size) + } + + @Test fun staleUnmanagedCompletionCannotEraseDifferentDestinationResumeState() { + val staleAttempt = UploadAttempt(UploadPreparationState(), MuxUpload.Progress()) + val stale = info().update(attempt = staleAttempt, statusFlow = staleAttempt.status) + val currentAttempt = UploadAttempt(UploadPreparationState(), MuxUpload.Progress()) + val current = info().copy(remoteUri = Uri.parse("https://example.invalid/new-upload")) + .update(attempt = currentAttempt, statusFlow = currentAttempt.status) + writeUploadState(current, MuxUpload.Progress(bytesUploaded = 4)) + MuxUploadManager.jobFinished(stale) + assertEquals(4L, readLastByteForFile(current)) + } + + @Test fun staleCompletionCannotEraseNewAttemptPersistenceAtSameDestination() { + val oldAttempt = UploadAttempt(UploadPreparationState(), MuxUpload.Progress()) + val old = info().update(attempt = oldAttempt, statusFlow = oldAttempt.status) + writeUploadState(old, MuxUpload.Progress(bytesUploaded = 4)) + val newAttempt = UploadAttempt(UploadPreparationState(), MuxUpload.Progress(), oldAttempt.id) + val next = info().update(attempt = newAttempt, statusFlow = newAttempt.status) + writeUploadState(next, MuxUpload.Progress(bytesUploaded = 8)) + MuxUploadManager.jobFinished(old) + assertEquals(8L, readLastByteForFile(next)) + } + + @Test fun repeatedPauseDoesNotReplayStatusOrProgress() { + work = { _, _, _ -> awaitCancellation() } + val upload = observe(MuxUpload.create(info())) + upload.start(); pump { chunks.size == 1 } + upload.pause(); dispatcher.scheduler.runCurrent() + val count = statuses.size + val progressCount = progress.size + upload.pause(); dispatcher.scheduler.runCurrent() + assertEquals(count, statuses.size) + assertEquals(progressCount, progress.size) + } + + @Test fun clearingProgressListenerKeepsResultAndStatusListenersActive() { + val upload = observe(MuxUpload.create(info())) + upload.setProgressListener(null) + upload.start(); pump { upload.isSuccessful } + assertEquals(1, results.size) + assertTrue(statuses.any { it is UploadStatus.UploadSuccess }) + } + + @Test fun transportFailureReportsSafeCategoryAndRetainsPublicException() { + val failure = java.io.IOException("https://secret.invalid/private") + val logger = mockk(relaxed = true) + val metrics = mockk(relaxed = true) + MuxUploadSdk.useLogger(logger) + every { UploadMetrics.create() } returns metrics + work = { _, _, _ -> throw failure } + val upload = MuxUpload.create(info().copy(optOut = false)) + upload.start(); pump { upload.uploadStatus is UploadStatus.UploadFailed } + assertSame(failure, upload.error) + verify { logger.e("MuxUpload", "Upload failed: Io", null) } + coVerify { metrics.reportUploadFailed(any(), any(), any(), "Upload failed: Io", any(), any()) } + } + + + @Test fun legacyZeroByteRecordStillResumesOriginalWithoutPreparation() { + writeUploadState(info(), MuxUpload.Progress(totalBytes = 16)) + val prefs = context.getSharedPreferences("mux_upload", 0) + val json = org.json.JSONArray(prefs.getString("uploads", null)) + json.getJSONObject(0).getJSONObject("data").remove("original_selected") + json.getJSONObject(0).getJSONObject("data").put("state", 0) + prefs.edit().putString("uploads", json.toString()).commit() + val restored = MuxUploadManager.resumeAllCachedJobs().single() + pump { restored.isSuccessful } + assertEquals(0, preparations) + assertEquals(0L, chunks.first().startByte) + } + + @Test fun generatedDestinationBlockAlsoRejectsADifferentSourceFile() { + val state = UploadPreparationState().apply { generatedRequestStarted = true } + val attempt = UploadAttempt(state, MuxUpload.Progress(totalBytes = 16)) + val marked = info().update(attempt = attempt, statusFlow = attempt.status) + writeUploadState(marked, attempt.confirmedProgress()) + forgetUploadState(marked) + val different = MuxUpload.create(info().copy(inputFile = sibling)) + different.start(); pump { different.uploadStatus is UploadStatus.UploadFailed } + assertTrue(different.error is GeneratedResumeBlockedException) + assertTrue(chunks.isEmpty()) + } + + @Test fun forceRestartWithoutManagerEntryCancelsItsPreviousJob() { + work = { _, _, _ -> awaitCancellation() } + val unmanaged = MuxUpload.Builder(info().remoteUri, source).manageUploadTask(false).build() + unmanaged.start(); pump { chunks.size == 1 } + val previous = internalInfo(unmanaged).uploadJob!! + val managed = MuxUpload.create(internalInfo(unmanaged)) + managed.start(forceRestart = true) + pump { chunks.size == 2 } + assertTrue(previous.isCancelled) + assertTrue(managed.isRunning) + } + + + @Test fun failedHandleFollowsManagerRetryAndReceivesItsNewResult() { + work = { chunk, _, _ -> + if (chunk.startByte == 0L) MuxUpload.Progress(bytesUploaded = 4) + else throw java.io.IOException("Test failure") + } + val original = observe(MuxUpload.create(info())) + original.start(); pump { results.size == 1 } + assertTrue(original.error is java.io.IOException) + assertTrue(MuxUploadManager.findUploadByFile(source)!!.error is java.io.IOException) + work = { _, _, _ -> awaitCancellation() } + val resumed = MuxUploadManager.resumeAllCachedJobs().single() + pump { chunks.size == 3 } + assertTrue(original.isRunning) + assertNull(original.error) + assertEquals(4L, chunks.last().startByte) + resumed.pause() + work = { chunk, _, _ -> MuxUpload.Progress(bytesUploaded = chunk.contentLength.toLong()) } + resumed.start(); pump { original.isSuccessful && results.size == 2 } + assertTrue(results.first().isFailure) + assertTrue(results.last().isSuccess) + } + private fun observe(upload: MuxUpload) = upload.apply { setStatusListener { statuses += it } setResultListener { results += it } diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/UploadPreparationTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/UploadPreparationTests.kt index 189ccdc1..b4fbd208 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/UploadPreparationTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/UploadPreparationTests.kt @@ -41,7 +41,7 @@ class UploadPreparationTests : AbsRobolectricTest() { val preparation = UploadPreparation(inspectMetadata = { MetadataInspectionResult.Success(metadata(facts)) }, inspectSamples = { _, _, _ -> MediaSampleInspection(facts, SampleScanStatus.Complete, timeline = timeline) }, convert = { _, _, _, _, _ -> error("Compliant input must pass through") }) - assertEquals(PreparedUpload.Original(), preparation.prepare(upload(), context)) + assertEquals(PreparedUpload.Original(), preparation.prepare(upload(), context, true)) } } @@ -101,7 +101,7 @@ class UploadPreparationTests : AbsRobolectricTest() { @Test fun inspectionFailuresAndMissingTimelineSelectOriginalWithoutRawDiagnosticText() = runBlocking { val failed = UploadPreparation(inspectMetadata = { throw NoSuchMethodError("https://secret.invalid/path") }) - assertEquals(PreparedUpload.Original(PreparationDiagnostic.InspectionFailed), failed.prepare(upload(), context)) + assertEquals(PreparedUpload.Original(PreparationDiagnostic.InspectionFailed), failed.prepare(upload(), context, true)) val facts = source().copy(averageBitrate = known(9_000_000L)) val incomplete = UploadPreparation(inspectMetadata = { MetadataInspectionResult.Success(metadata(facts)) }, inspectSamples = { _, _, _ -> MediaSampleInspection(facts, SampleScanStatus.LimitExceeded) }, @@ -112,11 +112,70 @@ class UploadPreparationTests : AbsRobolectricTest() { @Test fun adapterCompletionRacingPauseTransfersVerifiedOwnershipBeforeCoroutineDispatch() = bridgeRace(cancel = false) @Test fun adapterCompletionRacingCancelDeletesOnlyItsOwnedFile() = bridgeRace(cancel = true) + + @Test fun closedGateSkipsAllMediaInspectionAndExport() = runBlocking { + val preparation = UploadPreparation(inspectMetadata = { error("Gate must precede metadata") }, + inspectSamples = { _, _, _ -> error("Gate must precede samples") }, + convert = { _, _, _, _, _ -> error("Gate must precede export") }) + assertEquals(PreparedUpload.Original(PreparationDiagnostic.GeneratedResumeUnavailable), preparation.prepare(upload(), context)) + } + + @Test fun supportedPlannerConversionReachesAdapter() = runBlocking { + val facts = source().copy(averageBitrate = known(9_000_000L)) + val plan = StandardInputPlanner().plan(facts, options(), fullCapabilities()) + assertTrue(plan.action is StandardInputAction.Convert) + mockkConstructor(StandardInputPlanner::class) + try { + every { anyConstructed().plan(any(), any(), any()) } returns plan + var exports = 0 + val preparation = UploadPreparation(inspectMetadata = { MetadataInspectionResult.Success(metadata(facts)) }, + inspectSamples = { _, _, _ -> MediaSampleInspection(facts, SampleScanStatus.Complete, timeline = timeline) }, + convert = { _, _, _, _, _ -> exports++; SdrConversionResult.Failed(SdrConversionFailure.OutputInvalid) }) + assertEquals(PreparedUpload.Original(PreparationDiagnostic.ConversionFailed(SdrConversionFailure.OutputInvalid)), + preparation.prepare(upload(), context, true)) + assertEquals(1, exports) + } finally { unmockkConstructor(StandardInputPlanner::class) } + } + + @Test fun cancellationWaitsForAdapterReleaseBeforePreparationJobCompletes() = runBlocking { + val facts = source().copy(averageBitrate = known(9_000_000L)) + val started = CompletableDeferred() + val cancellationRequested = CompletableDeferred() + val released = CompletableDeferred() + val callback = slot<(SdrConversionResult) -> Unit>() + val export = mockk { + every { cancel() } answers { cancellationRequested.complete(Unit); Unit } + coEvery { awaitRelease() } coAnswers { released.await() } + } + mockkConstructor(SdrConversionAdapter::class) + every { anyConstructed().start(any(), any(), any(), any(), capture(callback)) } answers { + started.complete(Unit); export + } + val preparation = UploadPreparation(inspectMetadata = { MetadataInspectionResult.Success(metadata(facts)) }, + inspectSamples = { _, _, _ -> MediaSampleInspection(facts, SampleScanStatus.Complete, timeline = timeline) }) + val job = launch(Dispatchers.Default) { preparation.prepare(upload(), context, true) } + try { + withTimeout(5000) { started.await() } + job.cancel() + withTimeout(5000) { cancellationRequested.await() } + assertFalse(job.isCompleted) + released.complete(Unit) + withTimeout(5000) { job.join() } + assertTrue(job.isCompleted) + callback.captured(SdrConversionResult.Cancelled) + } finally { + released.complete(Unit); job.cancelAndJoin(); unmockkConstructor(SdrConversionAdapter::class) + } + } + private fun bridgeRace(cancel: Boolean) = runBlocking { val facts = source().copy(averageBitrate = known(9_000_000L)) val callback = slot<(SdrConversionResult) -> Unit>() val started = CompletableDeferred() - val export = mockk { every { cancel() } just Runs } + val export = mockk { + every { cancel() } just Runs + coEvery { awaitRelease() } just Runs + } mockkConstructor(SdrConversionAdapter::class) every { anyConstructed().start(any(), any(), any(), any(), capture(callback)) } answers { started.complete(Unit); export From e077d82fb56bfbfd64457b2fc54de550cbc58879 Mon Sep 17 00:00:00 2001 From: Eugene Morozov Date: Wed, 7 Oct 2026 14:05:18 -0700 Subject: [PATCH 29/49] NAT-547: remove internal README notes from product changes --- library/src/androidTest/README.md | 59 ------------------- .../SdrConversionFixtureTests.kt | 2 +- .../upload/internal/standardization/README.md | 36 ----------- 3 files changed, 1 insertion(+), 96 deletions(-) delete mode 100644 library/src/androidTest/README.md delete mode 100644 library/src/main/java/com/mux/video/upload/internal/standardization/README.md diff --git a/library/src/androidTest/README.md b/library/src/androidTest/README.md deleted file mode 100644 index 7d6f5e0b..00000000 --- a/library/src/androidTest/README.md +++ /dev/null @@ -1,59 +0,0 @@ -# SDR conversion fixtures - -`SdrConversionFixtureTests` runs the real Media3 adapter and shared output validator. Media stays outside Git and the test APK. With no staged directory, the suite is skipped; a staged directory with a missing fixture fails. Use a dedicated device/emulator and specify its serial on every adb command. - -Generate the following fixtures with FFmpeg built with libx264, libx265, AAC, and libmp3lame. These recipes reproduce the scenario contracts, not byte-identical encodes across FFmpeg versions. Record the FFmpeg version, source SHA-256s, API/device/encoder, test output, and any captured generated media in the external evidence directory. The multi-audio first track is stereo, non-default and 3 seconds; the default secondary is mono and 6 seconds. HEVC success remains device-dependent and must pass reinspection. - -```sh -SDR_FIXTURES=/absolute/path/to/external/sdr-fixtures -mkdir -p "$SDR_FIXTURES" -ffmpeg -y -f lavfi -i testsrc2=size=1920x1080:rate=30:duration=3 \ - -vf setparams=color_trc=bt709:color_primaries=bt709:colorspace=bt709 \ - -c:v libx264 -profile:v baseline -pix_fmt yuv420p -bf 0 \ - -color_primaries bt709 -color_trc bt709 -colorspace bt709 "$SDR_FIXTURES/no-audio.mp4" -ffmpeg -y -i "$SDR_FIXTURES/no-audio.mp4" \ - -f lavfi -i sine=frequency=440:sample_rate=48000:duration=3 \ - -f lavfi -i sine=frequency=880:sample_rate=48000:duration=6 \ - -filter_complex '[1:a]aformat=channel_layouts=stereo[first]' \ - -map 0:v -map '[first]' -map 2:a -c:v copy -c:a aac \ - -disposition:a:0 0 -disposition:a:1 default "$SDR_FIXTURES/multi-audio.mp4" -ffmpeg -y -i "$SDR_FIXTURES/no-audio.mp4" -c copy \ - -metadata:s:v:0 rotate=90 "$SDR_FIXTURES/rotated.mp4" -ffmpeg -y -i "$SDR_FIXTURES/no-audio.mp4" \ - -f lavfi -i sine=frequency=440:sample_rate=48000:duration=3 \ - -map 0:v -map 1:a -c:v copy -c:a libmp3lame -ac 2 "$SDR_FIXTURES/mp3-stereo.mp4" -ffmpeg -y -f lavfi -i testsrc2=size=512x288:rate=240:duration=2 \ - -vf setparams=color_trc=bt709:color_primaries=bt709:colorspace=bt709 \ - -c:v libx264 -profile:v baseline -pix_fmt yuv420p -bf 0 \ - -color_primaries bt709 -color_trc bt709 -colorspace bt709 "$SDR_FIXTURES/high-rate.mp4" -ffmpeg -y -f lavfi -i testsrc2=size=512x288:rate=60:duration=30 \ - -vf "select='if(lt(t,7.5),1,not(mod(n,3)))',setparams=color_trc=bt709:color_primaries=bt709:colorspace=bt709" -fps_mode vfr \ - -c:v libx264 -profile:v baseline -pix_fmt yuv420p -bf 0 -g 10000 -keyint_min 10000 -sc_threshold 0 -crf 18 \ - -color_primaries bt709 -color_trc bt709 -colorspace bt709 "$SDR_FIXTURES/vfr.mp4" -ffmpeg -y -i "$SDR_FIXTURES/no-audio.mp4" -c:v libx264 -profile:v baseline -pix_fmt yuv420p -bf 0 \ - -vf setparams=color_trc=linear:color_primaries=bt709:colorspace=bt709 \ - -color_primaries bt709 -color_trc linear -colorspace bt709 "$SDR_FIXTURES/linear.mp4" -for SDR_DURATION in 3 12; do - if [ "$SDR_DURATION" = 3 ]; then SDR_HEVC_NAME=hevc-small-open-gop; else SDR_HEVC_NAME=hevc-long-open-gop; fi - ffmpeg -y -f lavfi -i "testsrc2=size=512x288:rate=30:duration=$SDR_DURATION" \ - -vf setparams=color_trc=bt709:color_primaries=bt709:colorspace=bt709 \ - -c:v libx265 -pix_fmt yuv420p -x265-params 'open-gop=1:keyint=30:min-keyint=30:scenecut=0:bframes=0' \ - -color_primaries bt709 -color_trc bt709 -colorspace bt709 "$SDR_FIXTURES/$SDR_HEVC_NAME.mp4" -done -``` - -Build/install the test APK, then stage fixtures. Choose the serial of your dedicated test device; the commands below do not select an arbitrary connected device. - -```sh -SDR_DEVICE=emulator-5556 -./gradlew :library:assembleDebugAndroidTest -adb -s "$SDR_DEVICE" install -r library/build/outputs/apk/androidTest/debug/library-debug-androidTest.apk -adb -s "$SDR_DEVICE" push "$SDR_FIXTURES/." /data/local/tmp/nat547-fixtures/ -adb -s "$SDR_DEVICE" shell run-as com.mux.video.upload.test mkdir -p files/sdr-fixtures -adb -s "$SDR_DEVICE" shell run-as com.mux.video.upload.test cp -r /data/local/tmp/nat547-fixtures/. files/sdr-fixtures/ -adb -s "$SDR_DEVICE" shell am instrument -w -r \ - -e class com.mux.video.upload.internal.standardization.SdrConversionFixtureTests \ - com.mux.video.upload.test/androidx.test.runner.AndroidJUnitRunner -``` - -The H.264 matrix requires accepted, independently reinspected output, including preserved VFR timestamps and first stereo audio selection. LINEAR must fail before export. HEVC may succeed or fail conservatively on an unproven device; a safe failure is not evidence of working HEVC conversion. The cancellation test checks source bytes, tracked output cleanup, and one terminal result. API 23 and physical-device HEVC, quality/performance, and Mux ingest remain separate acceptance gates. diff --git a/library/src/androidTest/java/com/mux/video/upload/internal/standardization/SdrConversionFixtureTests.kt b/library/src/androidTest/java/com/mux/video/upload/internal/standardization/SdrConversionFixtureTests.kt index 3c6426d4..de9ce6f4 100644 --- a/library/src/androidTest/java/com/mux/video/upload/internal/standardization/SdrConversionFixtureTests.kt +++ b/library/src/androidTest/java/com/mux/video/upload/internal/standardization/SdrConversionFixtureTests.kt @@ -13,7 +13,7 @@ import java.util.concurrent.CountDownLatch import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicInteger -/** Stage external fixtures as documented in the adapter README; never package media in the APK. */ +/** Runs the adapter against externally staged fixtures without media in the APK. */ @RunWith(AndroidJUnit4::class) class SdrConversionFixtureTests { private val instrumentation = InstrumentationRegistry.getInstrumentation() diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/README.md b/library/src/main/java/com/mux/video/upload/internal/standardization/README.md deleted file mode 100644 index ad2d3b4c..00000000 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/README.md +++ /dev/null @@ -1,36 +0,0 @@ -# SDR adapter (NAT-547) - -`SdrConversionAdapter.start` accepts the original file, its metadata and complete shared sample/timeline inspection, and the planner's `StandardInputConversion`. These must describe the same unchanged source. It exports on a dedicated `HandlerThread`, validates the generated file with `StandardInputOutputValidator`, and invokes one internal terminal callback on that thread. It does not start transport or emit public upload events. - -- `Completed` transfers an explicitly owned `SdrGeneratedFile` and validated facts to orchestration. Keep it while upload/resume needs it; call its idempotent `delete()` at the terminal lifecycle point. -- `Failed` permits original-file fallback. No partial or rejected output is selected. -- `Cancelled` stops preparation even without a Transformer callback. Orchestration must suppress public results and must not upload the original as a cancellation fallback. -- If deletion fails, `Attempt.pendingCleanup` retains the exact deletion authority for retry. Nothing sweeps `cacheDir/mux-upload`, where customer source copies can live. - -Each attempt allocates only its tracked unique `sdr-*.mp4` file under `cacheDir/mux-upload/standard-input`, after a size-based disk guard. It deletes that file on failure/cancellation, releases Transformer, and quits its thread. A successful file outlives that thread. Persisted payload identity, abandoned-file recovery, and generated resume belong to NAT-549; upload/pause orchestration belongs to NAT-548. - -## Encoding contract - -H.264 stays H.264, HEVC stays HEVC, and another proven SDR source uses H.264. Dimensions are the planner's exact display-oriented even dimensions, with orientation baked into pixels and no upscaling. Portrait encoding is explicitly enabled. Video always encodes, including an identity presentation effect, so bitrate/GOP remediation cannot silently remux the source. - -The 8-bit 4:2:0 path requests H.264 Baseline (no B frames) or HEVC Main, VBR at 6 Mbps through a 2048-pixel long side or 15 Mbps above it, additionally capped to 75% of the plan's average-bitrate ceiling. GOP targets use 90% of the planned interval ceiling, rounded down to whole seconds. For preserved VFR timestamps, the target is also reduced by the nominal rate times the longest measured presentation gap: encoders commonly convert the interval to a nominal-rate frame count. If a positive whole-second target cannot fit, preparation falls back before export. Output validation remains the final bitrate, IDR/closed-GOP, profile, pixel, cadence, and timeline gate. - -Media3's default video factory overrides H.264 profile requests. The adapter therefore supplies Media3 `DefaultCodec` with the exact platform profile, level, dimensions, bitrate, nominal rate, surface input, and GOP interval. Preflight and export share those configuration keys, including Media3 1.11.1's API/SoC performance settings and the Redmi Note 9 Pro high-rate hint workaround. Planned timestamps remain unchanged. Transformer owns decoding, effects, encoding, and muxing. Its default no-output-sample watchdog remains enabled (10 seconds normally, 25 seconds on emulators); timeout errors use the same terminal cleanup as other export failures. Native calls cannot be forcibly interrupted. The AAC delegate disables both encoder and format fallback. Every `onFallbackApplied`, mismatched output MIME, or invalid generated file fails the attempt; a listed encoder never proves a usable path. - -The bounded advertised-format preflight gives cold native enumeration a separate two-second cooperative limit and capability queries a 500 ms limit, checked between calls. These are conservative development guards, not measured hardware performance thresholds. Only matching capability queries count toward the 128-query cap; unrelated inventory entries cannot reject an otherwise supported device merely by exceeding 128 entries. It checks source decode and exact output encode support before creating a file, and pins one supported video encoder. It does not turn advertised capabilities into the planner's device-path proof. - -Preserved cadence keeps source timestamps. Constant-rate reduction is limited to proven constant cadence with an integral source/30 fps ratio; a one-part-per-million allowance covers microsecond-rounded rate measurement. Media3 drops frames at the planned 30 fps and the shared validator checks actual output cadence and duration. LINEAR is correctly classified as SDR by inspection, but this adapter rejects its explicit transfer tags before export until color handling is proven. Frame duplication, fractional-rate resampling, 10-bit SDR encoding, and HDR/tone mapping have no proof in this adapter and fail conservatively. NAT-550 owns tone mapping. - -## Audio decision - -[Shipped Swift v1.2.0](https://github.com/muxinc/swift-upload-sdk/blob/faa440e6ea9339315a9d192ca314c4ef5a8668c9/Sources/MuxUploadSDK/InputStandardization/UploadInputStandardizationWorker.swift#L794) copies AAC and otherwise preserves sample rate, channel count, and available layout while encoding AAC (160 kbps for mono/stereo; 384 kbps for surround). Following that behavior and the Android TDD's exclusion of new surround conversion, this adapter copies known compliant AAC mono/stereo/5.1, converts known non-AAC mono/stereo to AAC-LC at 160 kbps with the same rate/count, and rejects non-AAC surround or unknown layouts. It adds no downmix policy. Output channel layout and reported export sample rate/count must match these targets. - -The asset loader's selector matches the first audio track's container ID, rather than Media3's preferred/default track. Missing identity fails; it never substitutes a secondary track. - -## Evidence and remaining limits - -Committed unit tests cover encoding targets, VFR interval targeting with the shared validator, strict encoder configuration, performance/workaround boundaries, cold/large codec inventories, first-track selection, Media3 timeout/fallback callbacks, and adapter ownership/cancellation. The committed device suite and [fixture recipes](../../../../../../../../androidTest/README.md) cover resize, silent H.264, rotated portrait, MP3 stereo to AAC, HEVC, 240-to-30 fps, preserved VFR, LINEAR fallback, and cancellation. Fixtures are staged outside the APK and remain external to product Git. Run these checks on each Media3 upgrade; unit configuration checks do not prove hardware behavior. - -The distinguishable multi-audio source has 3s video, 3s first stereo AAC (track ID 2, non-default), and 6s secondary mono AAC (ID 3, default). Output contains the first stereo track: its copied AAC bytes have the same SHA-256 as source audio ordinal zero and differ from the default secondary. This proves local selection only; Mux ingest parity remains pending. - -API 36 emulator evidence includes a short accepted HEVC export and a longer export rejected for CRA/open GOP; the software encoder also cannot encode 720p. These observations do not establish a universal HEVC limitation or justify changing codec families. API 23-29 execution, physical hardware, long HEVC closed-GOP support, 10-bit SDR, HDR, high-resolution device coverage, visual quality/performance acceptance, and a live Mux upload remain verification gates. Emulator timings do not establish physical-device performance limits. NAT-548 can use the validated SDR path for the early upload journey; no predecessor merge is required to continue the stack. From 853a71dfeadc7eb27698d728f9a462f5b56af9de Mon Sep 17 00:00:00 2001 From: Eugene Morozov Date: Thu, 8 Oct 2026 14:07:24 -0700 Subject: [PATCH 30/49] Fix listener reentrancy and upload replacement lifecycle --- .../com/mux/video/upload/api/MuxUpload.kt | 73 ++++-- .../mux/video/upload/api/MuxUploadManager.kt | 34 +-- .../video/upload/internal/UploadAttempt.kt | 39 +++- .../video/upload/internal/UploadJobFactory.kt | 20 +- .../upload/internal/UploadPersistence.kt | 63 +++--- .../internal/StandardInputLifecycleTests.kt | 4 +- .../upload/internal/UploadPersistenceTests.kt | 18 +- .../UploadPreparationLifecycleTests.kt | 212 +++++++++++++++++- 8 files changed, 378 insertions(+), 85 deletions(-) diff --git a/library/src/main/java/com/mux/video/upload/api/MuxUpload.kt b/library/src/main/java/com/mux/video/upload/api/MuxUpload.kt index c4b6ac20..78e88090 100644 --- a/library/src/main/java/com/mux/video/upload/api/MuxUpload.kt +++ b/library/src/main/java/com/mux/video/upload/api/MuxUpload.kt @@ -95,6 +95,8 @@ class MuxUpload private constructor( private var observerJob: Job? = null private var cancelled = false private var deliveredResultStatus: UploadStatus? = null + private var deliveredStatus: UploadStatus? = null + private var deliveredProgress: Progress? = null private val currentStatus: UploadStatus get() = uploadInfo.statusFlow?.value ?: lastKnownStatus ?: UploadStatus.Ready private var lastKnownStatus: UploadStatus? = null @@ -135,7 +137,7 @@ class MuxUpload private constructor( forceRestart: Boolean = false, coroutineScope: CoroutineScope = CoroutineScope(Dispatchers.Default) ) { - if (cancelled || uploadInfo.attempt?.isCancelled() == true) return + if (cancelled || uploadInfo.attempt?.isCancelled() == true || uploadInfo.attempt?.replacementFailure() != null) return if (!forceRestart && uploadInfo.uploadJob?.isActive == true && uploadInfo.attempt?.isStopped() != true) { observeUpload(uploadInfo) return @@ -172,7 +174,8 @@ class MuxUpload private constructor( * * If the upload already succeeded, the old result will be returned immediately. * After [cancel], this returns a failure without restarting. Cancellation of the calling - * coroutine still throws [CancellationException]. + * coroutine still throws [CancellationException]. Replacement by a new destination returns + * a failure to existing waiters and ends this handle. */ @Throws @Suppress("unused") @@ -185,11 +188,28 @@ class MuxUpload private constructor( } else { coroutineScope { startInner(coroutineScope = this) - uploadInfo.uploadJob?.await() ?: Result.failure(Exception("Upload failed to start")) + val awaited = uploadInfo + replacementResult(awaited)?.let { return@coroutineScope it } + try { + awaited.uploadJob?.await() ?: Result.failure(Exception("Upload failed to start")) + } catch (e: CancellationException) { + // Replacing the upload ends its job, but must not cancel an active caller. + currentCoroutineContext().ensureActive() + replacementResult(awaited) ?: throw e + } } } } + private fun replacementResult(upload: UploadInfo): Result? { + val replacement = upload.attempt?.replacementFailure() ?: return null + return when (val status = upload.statusFlow?.value) { + is UploadStatus.UploadSuccess -> Result.success(status) + is UploadStatus.UploadFailed -> Result.failure(status.exception) + else -> Result.failure(replacement) + } + } + /** * Pauses the upload. If the upload was already paused, this method has no effect * @@ -197,12 +217,10 @@ class MuxUpload private constructor( */ @Suppress("MemberVisibilityCanBePrivate") fun pause() { - if (cancelled || uploadInfo.attempt?.isCancelled() == true || isPaused) return + if (cancelled || uploadInfo.attempt?.isCancelled() == true || uploadInfo.attempt?.replacementFailure() != null || isPaused) return uploadInfo = if (autoManage) { - observerJob?.cancel("user requested pause") /*uploadInfo =*/ MuxUploadManager.pauseJob(uploadInfo) } else { - observerJob?.cancel("user requested pause") uploadInfo.attempt?.pause { writeUploadState(uploadInfo, it) } uploadInfo.uploadJob?.cancel() uploadInfo @@ -223,7 +241,7 @@ class MuxUpload private constructor( MuxUploadManager.cancelJob(uploadInfo) } else { val current = uploadInfo - if (current.attempt == null) forgetUploadState(current) else current.attempt.cancel { forgetUploadState(current) } + current.attempt?.cancel { forgetUploadState(current) } current.uploadJob?.cancel("user requested cancel") } observerJob?.cancel("user requested cancel") @@ -236,9 +254,10 @@ class MuxUpload private constructor( */ @MainThread fun setProgressListener(listener: UploadEventListener?) { + if (progressListener !== listener) deliveredProgress = null progressListener = listener - if (!cancelled && uploadInfo.attempt?.isCancelled() != true) lastKnownProgress?.let { listener?.onEvent(it) } observeUpload(uploadInfo) + if (!cancelled && uploadInfo.attempt?.isCancelled() != true) lastKnownProgress?.let { notifyProgress(it) } } /** @@ -261,9 +280,10 @@ class MuxUpload private constructor( */ @MainThread fun setStatusListener(listener: UploadEventListener?) { + if (statusListener !== listener) deliveredStatus = null statusListener = listener observeUpload(uploadInfo) - if (!cancelled && uploadInfo.attempt?.isCancelled() != true) listener?.onEvent(currentStatus) + if (!cancelled && uploadInfo.attempt?.isCancelled() != true) notifyStatus(currentStatus) } /** @@ -285,19 +305,19 @@ class MuxUpload private constructor( }.collect { (observed, status) -> if (cancelled || uploadInfo.attempt !== observed.attempt || observed.attempt?.isCancelled() == true) return@collect lastKnownStatus = status - statusListener?.onEvent(status) + notifyStatus(status) // A listener can synchronously pause, cancel, or replace the attempt. if (cancelled || uploadInfo.attempt !== observed.attempt || observed.attempt?.isCancelled() == true || - (observed.attempt?.isStopped() == true && status !is UploadStatus.UploadPaused)) return@collect + (observed.attempt?.isStopped() == true && status !is UploadStatus.UploadPaused && status !is UploadStatus.UploadFailed && status !is UploadStatus.UploadSuccess)) return@collect when (status) { - is UploadStatus.Uploading -> progressListener?.onEvent(status.uploadProgress) - is UploadStatus.UploadPaused -> progressListener?.onEvent(status.uploadProgress) + is UploadStatus.Uploading -> notifyProgress(status.uploadProgress) + is UploadStatus.UploadPaused -> notifyProgress(status.uploadProgress) is UploadStatus.UploadSuccess -> { - progressListener?.onEvent(status.uploadProgress) + notifyProgress(status.uploadProgress) if (!cancelled && observed.attempt?.isCancelled() != true && uploadInfo.attempt === observed.attempt) notifyResult(status) } is UploadStatus.UploadFailed -> { - progressListener?.onEvent(status.uploadProgress) + notifyProgress(status.uploadProgress) if (observed.attempt?.isCancelled() != true && uploadInfo.attempt === observed.attempt) notifyResult(status) } else -> {} @@ -305,6 +325,20 @@ class MuxUpload private constructor( } } + private fun notifyStatus(status: UploadStatus) { + val listener = statusListener ?: return + if (deliveredStatus === status) return + deliveredStatus = status + listener.onEvent(status) + } + + private fun notifyProgress(progress: Progress) { + val listener = progressListener ?: return + if (deliveredProgress == progress) return + deliveredProgress = progress + listener.onEvent(progress) + } + private fun notifyResult(status: UploadStatus) { if (cancelled || uploadInfo.attempt?.isCancelled() == true || resultListener == null || deliveredResultStatus === status) return when (status) { @@ -321,9 +355,12 @@ class MuxUpload private constructor( } private fun observeUpload(uploadInfo: UploadInfo) { - observerJob?.cancel("switching observers") - observerJob = if (!cancelled && (resultListener != null || progressListener != null || statusListener != null)) - newObserveProgressJob(uploadInfo) else null + if (cancelled || (resultListener == null && progressListener == null && statusListener == null)) { + observerJob?.cancel("no listeners") + observerJob = null + } else if (observerJob?.isActive != true) { + observerJob = newObserveProgressJob(uploadInfo) + } } /** diff --git a/library/src/main/java/com/mux/video/upload/api/MuxUploadManager.kt b/library/src/main/java/com/mux/video/upload/api/MuxUploadManager.kt index d9b97c35..d683aa3d 100644 --- a/library/src/main/java/com/mux/video/upload/api/MuxUploadManager.kt +++ b/library/src/main/java/com/mux/video/upload/api/MuxUploadManager.kt @@ -63,20 +63,16 @@ object MuxUploadManager { @JvmSynthetic @MainThread internal fun resumeCachedJobs(includePaused: Boolean): List { - val restored = readAllCachedUploads().filter { upload -> - upload.inputFile.exists().also { if (!it) forgetUploadState(upload) } - }.map { upload -> - if (!includePaused && readUploadResumeState(upload).paused) { + val restored = readCachedUploadSnapshots().filter { cached -> + cached.upload.inputFile.exists().also { if (!it) forgetUploadState(cached.upload) } + }.map { (upload, saved) -> + if (!includePaused && saved.paused) { uploadsByFilename.getOrPut(upload.inputFile.absolutePath) { - val saved = readUploadResumeState(upload) - val preparation = UploadPreparationState().apply { originalSelected = saved.originalSelected } - val attempt = UploadAttempt(preparation, MuxUpload.Progress( - bytesUploaded = if (saved.generatedResumeBlocked) 0 else saved.bytesSent, - totalBytes = upload.inputFile.length()), previousPersistenceOwnerId = saved.attemptId) + val attempt = createUploadAttempt(upload, saved) attempt.pause {} upload.update(attempt = attempt, statusFlow = attempt.status).also { it.session.current.value = it } } - } else startJob(upload, restart = false) + } else startJob(upload, restart = false, resumeState = saved) } notifyListListeners() return restored.map { MuxUpload.create(it) } @@ -108,9 +104,9 @@ object MuxUploadManager { */ @JvmSynthetic @MainThread - internal fun startJob(upload: UploadInfo, restart: Boolean = false): UploadInfo { + internal fun startJob(upload: UploadInfo, restart: Boolean = false, resumeState: UploadResumeState? = null): UploadInfo { assertMainThread() - val updatedInfo = insertOrUpdateUpload(upload, restart) + val updatedInfo = insertOrUpdateUpload(upload, restart, resumeState) notifyListListeners() return updatedInfo } @@ -173,13 +169,14 @@ object MuxUploadManager { upload.uploadJob?.cancel() } - private fun cancelAttempt(upload: UploadInfo) { + private fun cancelAttempt(upload: UploadInfo, forgetUnstarted: Boolean = false) { val attempt = upload.attempt - if (attempt == null) forgetUploadState(upload) else attempt.cancel { forgetUploadState(upload) } + if (attempt != null) attempt.cancel { forgetUploadState(upload) } + else if (forgetUnstarted) forgetUploadState(upload) upload.uploadJob?.cancel() } - private fun insertOrUpdateUpload(upload: UploadInfo, restart: Boolean): UploadInfo { + private fun insertOrUpdateUpload(upload: UploadInfo, restart: Boolean, resumeState: UploadResumeState?): UploadInfo { val filename = upload.inputFile.absolutePath val previous = uploadsByFilename[filename] val newDestination = previous != null && previous.remoteUri != upload.remoteUri @@ -189,13 +186,16 @@ object MuxUploadManager { if (restart || newDestination) { val generatedMayExistRemotely = !newDestination && (source.generatedResumeBlocked || source.attempt?.preparation?.generatedRequestStarted == true || readUploadResumeState(upload).generatedResumeBlocked) - cancelAttempt(source) + if (newDestination) { + source.attempt?.replace { forgetUploadState(source) } + source.uploadJob?.cancel() + } else cancelAttempt(source, forgetUnstarted = true) source = upload.update(attempt = null, uploadJob = source.uploadJob, statusFlow = null, restoredFromOriginal = false, generatedResumeBlocked = generatedMayExistRemotely).also { it.session = if (newDestination) upload.session else source.session } } - val newUpload = startUploadJob(source) + val newUpload = startUploadJob(source, resumeState.takeIf { previous == null && !restart }) uploadsByFilename += upload.inputFile.absolutePath to newUpload observerJobsByFilename[upload.inputFile.absolutePath]?.cancel() observerJobsByFilename += upload.inputFile.absolutePath to newObserveProgressJob(newUpload) diff --git a/library/src/main/java/com/mux/video/upload/internal/UploadAttempt.kt b/library/src/main/java/com/mux/video/upload/internal/UploadAttempt.kt index 8f5305dc..99555091 100644 --- a/library/src/main/java/com/mux/video/upload/internal/UploadAttempt.kt +++ b/library/src/main/java/com/mux/video/upload/internal/UploadAttempt.kt @@ -43,15 +43,18 @@ internal class UploadAttempt( private var terminal = false private var superseded = false private var confirmed = initialProgress + private var transportPayloadGenerated: Boolean? = null + private var replacement: UploadReplacedException? = null @Synchronized fun isStopped() = stopped @Synchronized fun isCancelled() = cancelled + @Synchronized fun replacementFailure() = replacement @Synchronized fun confirmedProgress() = confirmed @Synchronized fun supersede() { superseded = true } @Synchronized fun retainGenerated(output: SdrGeneratedFile) { if (superseded) preparation.discardLateOutput(output) - else if (cancelled) { + else if (cancelled || replacement != null) { preparation.trackCleanup(output) preparation.deleteOwnedFile() } else preparation.verified = output @@ -68,11 +71,13 @@ internal class UploadAttempt( } @Synchronized fun beginTransport(generated: Boolean, persist: () -> Unit) { - if (stopped) throw kotlinx.coroutines.CancellationException("Upload stopped") + if (stopped || terminal || superseded) throw kotlinx.coroutines.CancellationException("Upload stopped") + if (transportPayloadGenerated == generated) return if (generated) preparation.generatedRequestStarted = true else preparation.originalSelected = true - // Persist payload selection before the first request, even without an acknowledgement. + // Claim persistence ownership before the first request of each attempt/payload choice. persist() + transportPayloadGenerated = generated } @Synchronized fun publish(value: UploadStatus) { @@ -103,6 +108,20 @@ internal class UploadAttempt( preparation.deleteOwnedFile() } + fun replace(forget: () -> Unit) { + synchronized(this) { + if (cancelled || replacement != null) return + replacement = UploadReplacedException() + stopped = true + if (!terminal) { + terminal = true + status.value = UploadStatus.UploadFailed(checkNotNull(replacement), confirmed) + } + forget() + } + preparation.deleteOwnedFile() + } + @Synchronized fun finish(value: UploadStatus): Boolean { if (stopped || terminal || superseded) return false terminal = true @@ -111,6 +130,20 @@ internal class UploadAttempt( } } +/** Both paused restoration and transport start with the same progress/ownership rules. */ +internal fun createUploadAttempt(upload: UploadInfo, saved: UploadResumeState): UploadAttempt { + val preparation = upload.attempt?.preparation ?: UploadPreparationState().apply { + originalSelected = upload.restoredFromOriginal || saved.originalSelected + } + val now = System.currentTimeMillis() + return UploadAttempt(preparation, MuxUpload.Progress( + bytesUploaded = if (saved.generatedResumeBlocked) 0 else saved.bytesSent, + totalBytes = upload.inputFile.length(), startTime = now, updatedTime = now), + previousPersistenceOwnerId = saved.attemptId) +} + +internal class UploadReplacedException : IllegalStateException("Upload was replaced by a new destination.") + internal class GeneratedResumeBlockedException : IllegalStateException( "Generated payload identity or server offset is unverified. Create a new Direct Upload." ) diff --git a/library/src/main/java/com/mux/video/upload/internal/UploadJobFactory.kt b/library/src/main/java/com/mux/video/upload/internal/UploadJobFactory.kt index e6f36d3b..5f7b1645 100644 --- a/library/src/main/java/com/mux/video/upload/internal/UploadJobFactory.kt +++ b/library/src/main/java/com/mux/video/upload/internal/UploadJobFactory.kt @@ -15,8 +15,8 @@ import java.io.FileInputStream import java.util.UUID @JvmSynthetic -internal fun startUploadJob(upload: UploadInfo): UploadInfo = MuxUploadSdk.uploadJobFactory() - .createUploadJob(upload, CoroutineScope(Dispatchers.Default)) +internal fun startUploadJob(upload: UploadInfo, saved: UploadResumeState? = null): UploadInfo = MuxUploadSdk.uploadJobFactory() + .createUploadJob(upload, CoroutineScope(Dispatchers.Default), saved) /** Owns payload selection and the transition from cancellable preparation to transport. */ internal class UploadJobFactory internal constructor( @@ -33,16 +33,12 @@ internal class UploadJobFactory internal constructor( ChunkWorker.create(chunk, uploadInfo, "video/*", progressFlow) } - fun createUploadJob(uploadInfo: UploadInfo, outerScope: CoroutineScope): UploadInfo { + fun createUploadJob(uploadInfo: UploadInfo, outerScope: CoroutineScope, resumeState: UploadResumeState? = null): UploadInfo { uploadInfo.attempt?.supersede() - val saved = readUploadResumeState(uploadInfo) - val preparation = uploadInfo.attempt?.preparation ?: UploadPreparationState().apply { - originalSelected = uploadInfo.restoredFromOriginal || saved.originalSelected - } - val startTime = System.currentTimeMillis() - val attempt = UploadAttempt(preparation, MuxUpload.Progress( - bytesUploaded = if (saved.generatedResumeBlocked) 0 else saved.bytesSent, totalBytes = uploadInfo.inputFile.length(), - startTime = startTime, updatedTime = startTime), previousPersistenceOwnerId = saved.attemptId) + val saved = resumeState ?: readUploadResumeState(uploadInfo) + val attempt = createUploadAttempt(uploadInfo, saved) + val preparation = attempt.preparation + val startTime = attempt.confirmedProgress().startTime var runningInfo = uploadInfo.update(attempt = attempt, statusFlow = attempt.status.asStateFlow()) // Lazy startup lets the returned identity be installed before any completion callback. val job = outerScope.async(start = CoroutineStart.LAZY) { @@ -149,7 +145,7 @@ internal class UploadJobFactory internal constructor( } Result.failure(e) } finally { - if (!attempt.isStopped() || attempt.isCancelled()) preparation.deleteOwnedFile() + if (!attempt.isStopped() || attempt.isCancelled() || attempt.replacementFailure() != null) preparation.deleteOwnedFile() } } runningInfo = runningInfo.update(uploadJob = job) diff --git a/library/src/main/java/com/mux/video/upload/internal/UploadPersistence.kt b/library/src/main/java/com/mux/video/upload/internal/UploadPersistence.kt index e9e804d1..c1cb27de 100644 --- a/library/src/main/java/com/mux/video/upload/internal/UploadPersistence.kt +++ b/library/src/main/java/com/mux/video/upload/internal/UploadPersistence.kt @@ -60,26 +60,25 @@ internal fun forgetUploadState(uploadInfo: UploadInfo) { UploadPersistence.removeForFile(uploadInfo) } +internal data class CachedUpload(val upload: UploadInfo, val resumeState: UploadResumeState) + +internal fun readCachedUploadSnapshots(): List = UploadPersistence.readSnapshots() + @JvmSynthetic -internal fun readAllCachedUploads(includePaused: Boolean = true): List { - return UploadPersistence.readEntries() - .map { it.value } - .filter { includePaused || it.state == UploadPersistence.WAS_RUNNING } - .map { - UploadInfo( - inputStandardization = it.inputStandardization, - remoteUri = Uri.parse(it.url), - inputFile = it.file, - chunkSize = it.chunkSize, - retriesPerChunk = it.retriesPerChunk, - optOut = it.optOut, - uploadJob = null, - statusFlow = null, - restoredFromOriginal = it.originalSelected, - generatedResumeBlocked = it.generatedResumeBlocked, - ) - } -} +internal fun readAllCachedUploads(): List = readCachedUploadSnapshots().map { it.upload } + +private fun UploadEntry.toUploadInfo(blocked: Boolean) = UploadInfo( + inputStandardization = inputStandardization, + remoteUri = Uri.parse(url), + inputFile = file, + chunkSize = chunkSize, + retriesPerChunk = retriesPerChunk, + optOut = optOut, + uploadJob = null, + statusFlow = null, + restoredFromOriginal = originalSelected, + generatedResumeBlocked = blocked, +) /** * Datastore for uploads that are paused, are running, or should be running. Internally it models @@ -110,14 +109,7 @@ private object UploadPersistence { fun readState(upload: UploadInfo): UploadResumeState { checkInitialized() val entry = fetchEntries()[upload.inputFile.absolutePath]?.takeIf { it.url == upload.remoteUri.toString() } - return UploadResumeState( - bytesSent = entry?.bytesSent ?: 0, - originalSelected = entry?.originalSelected ?: false, - generatedResumeBlocked = entry?.generatedResumeBlocked == true || - destinationKey(upload.remoteUri.toString()) in readBlocks(), - paused = entry?.state == WAS_PAUSED, - attemptId = entry?.attemptId, - ) + return resumeState(entry, destinationKey(upload.remoteUri.toString()) in readBlocks()) } @Throws @@ -136,13 +128,24 @@ private object UploadPersistence { writeEntries(entries, blocks) } - @Throws @Synchronized - fun readEntries(): MutableMap { + fun readSnapshots(): List { checkInitialized() - return fetchEntries() + val blocks = readBlocks() + return fetchEntries().values.map { entry -> + val saved = resumeState(entry, destinationKey(entry.url) in blocks) + CachedUpload(entry.toUploadInfo(saved.generatedResumeBlocked), saved) + } } + private fun resumeState(entry: UploadEntry?, blocked: Boolean) = UploadResumeState( + bytesSent = entry?.bytesSent ?: 0, + originalSelected = entry?.originalSelected ?: false, + generatedResumeBlocked = entry?.generatedResumeBlocked == true || blocked, + paused = entry?.state == WAS_PAUSED, + attemptId = entry?.attemptId, + ) + @Throws @Synchronized private fun writeEntries(entries: Map, blocks: Set) { diff --git a/library/src/test/java/com/mux/video/upload/internal/StandardInputLifecycleTests.kt b/library/src/test/java/com/mux/video/upload/internal/StandardInputLifecycleTests.kt index fec16c2c..85d1dead 100644 --- a/library/src/test/java/com/mux/video/upload/internal/StandardInputLifecycleTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/StandardInputLifecycleTests.kt @@ -43,8 +43,8 @@ class StandardInputLifecycleTests : AbsRobolectricTest() { val realFactory = UploadJobFactory.create() val factory = mockk() - every { factory.createUploadJob(any(), any()) } answers { - realFactory.createUploadJob(firstArg(), scope) + every { factory.createUploadJob(any(), any(), any()) } answers { + realFactory.createUploadJob(firstArg(), scope, thirdArg()) } mockkObject(MuxUploadSdk) every { MuxUploadSdk.uploadJobFactory() } returns factory diff --git a/library/src/test/java/com/mux/video/upload/internal/UploadPersistenceTests.kt b/library/src/test/java/com/mux/video/upload/internal/UploadPersistenceTests.kt index b6c0769d..2d171b1c 100644 --- a/library/src/test/java/com/mux/video/upload/internal/UploadPersistenceTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/UploadPersistenceTests.kt @@ -128,6 +128,20 @@ class UploadPersistenceTests : AbsRobolectricTest() { ) } + @Test + fun cachedSnapshotsReadEntriesAndBlocksOnceForAllUploads() { + val prefs = mockSharedPreferences() + initializeUploadPersistence(mockContext(prefs)) + writeUploadState(uploadInfo("file/a"), MuxUpload.Progress(bytesUploaded = 5, totalBytes = 10)) + writeUploadState(uploadInfo("file/b"), MuxUpload.Progress(bytesUploaded = 2, totalBytes = 4)) + clearMocks(prefs, answers = false, childMocks = false) + val snapshots = readCachedUploadSnapshots() + assertEquals(listOf(5L, 2L), snapshots.map { it.resumeState.bytesSent }) + assertEquals(listOf(true, true), snapshots.map { it.resumeState.paused }) + verify(exactly = 1) { prefs.getString("uploads", null) } + verify(exactly = 1) { prefs.getString("generated_upload_blocks", null) } + } + private fun uploadInfo(name: String = "a/file") = UploadInfo( inputFile = File(name).absoluteFile, remoteUri = Uri.parse("https://www.mux.com/$name"), @@ -138,10 +152,10 @@ class UploadPersistenceTests : AbsRobolectricTest() { statusFlow = null, ) - private fun mockContext(): Context { + private fun mockContext(prefs: SharedPreferences = mockSharedPreferences()): Context { return mockk { every { applicationContext } returns mockk { - every { getSharedPreferences(any(), any()) } returns mockSharedPreferences() + every { getSharedPreferences(any(), any()) } returns prefs } } } diff --git a/library/src/test/java/com/mux/video/upload/internal/UploadPreparationLifecycleTests.kt b/library/src/test/java/com/mux/video/upload/internal/UploadPreparationLifecycleTests.kt index 969abeb5..3f2bb832 100644 --- a/library/src/test/java/com/mux/video/upload/internal/UploadPreparationLifecycleTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/UploadPreparationLifecycleTests.kt @@ -52,7 +52,7 @@ class UploadPreparationLifecycleTests : AbsRobolectricTest() { mockk { coEvery { upload() } coAnswers { work(chunk, info, flow) } } }) val factory = mockk() - every { factory.createUploadJob(any(), any()) } answers { real.createUploadJob(firstArg(), scope) } + every { factory.createUploadJob(any(), any(), any()) } answers { real.createUploadJob(firstArg(), scope, thirdArg()) } mockkObject(MuxUploadSdk) every { MuxUploadSdk.uploadJobFactory() } returns factory mockkObject(UploadMetrics.Companion) @@ -630,6 +630,216 @@ class UploadPreparationLifecycleTests : AbsRobolectricTest() { assertTrue(results.last().isSuccess) } + @Test fun listenerChangeInsideSuccessCallbackDeliversSuccessOnce() { + val upload = MuxUpload.create(info()) + var deliveries = 0 + var completions = 0 + upload.setResultListener { completions++ } + upload.setStatusListener { status -> + if (status is UploadStatus.UploadSuccess) { + deliveries++ + // Bound this regression if a restarted observer replays success again. + if (deliveries < 3) upload.setProgressListener(null) else upload.clearListeners() + } + } + upload.start(); pump { upload.isSuccessful && completions > 0 } + assertEquals(1, deliveries) + assertEquals(1, completions) + } + + @Test fun listenerRegistrationOnlyDeliversCurrentStateToThatListenerOnce() { + val upload = MuxUpload.create(info()) + var first = 0 + var replacement = 0 + upload.setStatusListener { first++ } + dispatcher.scheduler.runCurrent() + assertEquals(1, first) + upload.setProgressListener { progress += it } + upload.setResultListener { results += it } + dispatcher.scheduler.runCurrent() + assertEquals(1, first) + upload.setStatusListener { replacement++ } + dispatcher.scheduler.runCurrent() + assertEquals(1, replacement) + upload.clearListeners() + upload.setStatusListener { replacement++ } + dispatcher.scheduler.runCurrent() + assertEquals(2, replacement) + } + + @Test fun neverStartedCancelPreservesAnotherAttemptsPausedRecord() { + work = { _, _, _ -> awaitCancellation() } + val managed = MuxUpload.create(info()) + managed.start(); pump { chunks.size == 1 } + managed.pause(); dispatcher.scheduler.runCurrent() + val saved = readUploadResumeState(info()) + for (autoManage in listOf(false, true)) { + MuxUpload.Builder(info().remoteUri, source).manageUploadTask(autoManage).build().cancel() + assertEquals(saved, readUploadResumeState(info())) + assertEquals(1, readAllCachedUploads().size) + assertTrue(managed.isPaused) + } + } + + @Test fun neverStartedCancelPreservesDiskRecordButExplicitRestartClearsOffset() { + writeUploadState(info(), MuxUpload.Progress(bytesUploaded = 8, totalBytes = 16)) + for (autoManage in listOf(false, true)) { + MuxUpload.Builder(info().remoteUri, source).manageUploadTask(autoManage).build().cancel() + assertEquals(8L, readUploadResumeState(info()).bytesSent) + } + work = { _, _, _ -> awaitCancellation() } + val restarted = MuxUpload.create(info()) + restarted.start(forceRestart = true); pump { chunks.size == 1 } + assertEquals(0L, chunks.single().startByte) + } + + @Test fun newDestinationFinishesOldHandleAndExistingWaiterWithReplacementFailure() { + work = { _, _, _ -> awaitCancellation() } + val old = observe(MuxUpload.create(info())) + old.start(); pump { chunks.size == 1 } + var awaited: Result? = null + val waiter = scope.launch { awaited = old.awaitSuccess() } + dispatcher.scheduler.runCurrent() + val replacement = MuxUpload.create(info().copy(remoteUri = Uri.parse("https://example.invalid/replacement"))) + replacement.start(); pump { chunks.size == 2 && waiter.isCompleted && results.size == 1 } + assertFalse(waiter.isCancelled) + assertTrue(awaited!!.exceptionOrNull() is UploadReplacedException) + assertTrue(old.uploadStatus is UploadStatus.UploadFailed) + assertTrue(old.error is UploadReplacedException) + assertTrue(results.single().exceptionOrNull() is UploadReplacedException) + assertFalse(old.isRunning) + old.start(forceRestart = true) + var late: Result? = null + val lateWaiter = scope.launch { late = old.awaitSuccess() } + pump { lateWaiter.isCompleted } + assertFalse(lateWaiter.isCancelled) + assertTrue(late!!.exceptionOrNull() is UploadReplacedException) + assertTrue(replacement.isRunning) + assertEquals(2, chunks.size) + assertEquals(1, results.size) + } + + @Test fun replacementPreservesSuccessThatWasPublishedBeforeListenersObservedIt() { + work = { _, _, _ -> awaitCancellation() } + val old = observe(MuxUpload.create(info())) + old.start(); pump { chunks.size == 1 } + val completed = UploadStatus.UploadSuccess(old.currentProgress) + assertTrue(internalInfo(old).attempt!!.finish(completed)) + // Replace before the callback observer or awaiting caller can consume success. + val replacement = MuxUpload.create(info().copy(remoteUri = Uri.parse("https://example.invalid/replacement"))) + replacement.start(); pump { chunks.size == 2 && results.size == 1 } + assertTrue(results.single().isSuccess) + assertTrue(old.isSuccessful) + var awaited: Result? = null + val waiter = scope.launch { awaited = old.awaitSuccess() } + pump { waiter.isCompleted } + assertSame(completed, awaited!!.getOrThrow()) + assertTrue(replacement.isRunning) + } + + @Test fun replacingPausedDestinationFinishesOldHandleOnce() { + work = { _, _, _ -> awaitCancellation() } + val old = observe(MuxUpload.create(info())) + old.start(); pump { chunks.size == 1 } + old.pause(); dispatcher.scheduler.runCurrent() + val replacement = MuxUpload.create(info().copy(remoteUri = Uri.parse("https://example.invalid/replacement"))) + replacement.start(); pump { chunks.size == 2 && results.size == 1 } + assertFalse(old.isPaused) + assertTrue(old.error is UploadReplacedException) + assertTrue(results.single().isFailure) + assertTrue(replacement.isRunning) + } + + @Test fun replacementWaitsForOldPreparationAndDeletesItsLateOwnedOutput() { + val release = CompletableDeferred() + val generated = generated(8) + prepare = { withContext(NonCancellable) { release.await(); PreparedUpload.Generated(generated) } } + val old = observe(MuxUpload.create(info())) + old.start(); pump { preparations == 1 } + prepare = { PreparedUpload.Original() } + val replacement = MuxUpload.create(info().copy(remoteUri = Uri.parse("https://example.invalid/replacement"))) + replacement.start(); pump { results.size == 1 } + assertTrue(chunks.isEmpty()) + assertTrue(old.error is UploadReplacedException) + release.complete(Unit) + pump { replacement.isSuccessful } + assertFalse(generated.file.exists()) + assertTrue(source.exists()); assertTrue(sibling.exists()) + assertArrayEquals(source.readBytes(), chunks.flatMap { it.sliceData.toList() }.toByteArray()) + assertEquals(1, results.size) + } + + @Test fun callerCancellationStillThrowsInsteadOfReturningReplacementFailure() { + work = { _, _, _ -> awaitCancellation() } + val old = MuxUpload.create(info()) + old.start(); pump { chunks.size == 1 } + var returned = false + val waiter = scope.launch { old.awaitSuccess(); returned = true } + dispatcher.scheduler.runCurrent() + waiter.cancel() + MuxUpload.create(info().copy(remoteUri = Uri.parse("https://example.invalid/replacement"))).start() + pump { chunks.size == 2 && waiter.isCompleted } + assertTrue(waiter.isCancelled) + assertFalse(returned) + } + + @Test fun payloadSelectionIsWrittenOncePerAttemptAndAcknowledgementsRemainPersisted() { + val prefs = spyk(context.getSharedPreferences("mux_upload", 0)) + val persistenceContext = mockk() + every { persistenceContext.applicationContext } returns persistenceContext + every { persistenceContext.getSharedPreferences("mux_upload", 0) } returns prefs + initializeUploadPersistence(persistenceContext) + work = { chunk, upload, _ -> + val saved = readUploadResumeState(upload) + assertEquals(chunk.startByte, saved.bytesSent) + assertEquals(upload.attempt!!.id, saved.attemptId) + assertTrue(saved.originalSelected) + MuxUpload.Progress(bytesUploaded = chunk.contentLength.toLong()) + } + val upload = MuxUpload.create(info()) + upload.start(); pump { upload.isSuccessful } + // Selection once, four acknowledged chunk checkpoints, and terminal record removal. + verify(exactly = 6) { prefs.edit() } + assertTrue(readAllCachedUploads().isEmpty()) + } + + @Test fun resumedAttemptClaimsSavedOwnershipBeforeItsFirstAcknowledgement() { + work = { _, _, _ -> awaitCancellation() } + val upload = MuxUpload.create(info()) + upload.start(); pump { chunks.size == 1 } + val firstOwner = readUploadResumeState(info()).attemptId + upload.pause(); dispatcher.scheduler.runCurrent() + upload.start(); pump { chunks.size == 2 } + val saved = readUploadResumeState(info()) + assertNotEquals(firstOwner, saved.attemptId) + assertEquals(internalInfo(upload).attempt!!.id, saved.attemptId) + assertEquals(0L, saved.bytesSent) + assertTrue(saved.originalSelected) + } + + @Test fun restorationUsesSnapshotsForPausedAndRunningUploadsWithoutExtraStateReads() { + writeUploadState(info(), MuxUpload.Progress(bytesUploaded = 8, totalBytes = 16)) + val running = info().copy(inputFile = sibling) + writeUploadState(running, MuxUpload.Progress(bytesUploaded = 0, totalBytes = sibling.length())) + val prefs = context.getSharedPreferences("mux_upload", 0) + val entries = org.json.JSONArray(prefs.getString("uploads", null)) + entries.getJSONObject(1).getJSONObject("data").put("state", 0) + prefs.edit().putString("uploads", entries.toString()).commit() + mockkStatic(::readUploadResumeState) + every { readUploadResumeState(any()) } throws AssertionError("Restoration must use its cached snapshot") + work = { _, _, _ -> awaitCancellation() } + MuxUploadSdk.initialize(context, true) + pump { chunks.size == 1 } + val paused = MuxUploadManager.findUploadByFile(source)!! + assertTrue(paused.isPaused) + assertEquals(8L, paused.currentProgress.bytesUploaded) + assertEquals(16L, paused.currentProgress.totalBytes) + assertTrue(paused.currentProgress.startTime > 0) + assertTrue(paused.currentProgress.updatedTime >= paused.currentProgress.startTime) + assertTrue(MuxUploadManager.findUploadByFile(sibling)!!.isRunning) + verify(exactly = 0) { readUploadResumeState(any()) } + } + private fun observe(upload: MuxUpload) = upload.apply { setStatusListener { statuses += it } setResultListener { results += it } From 90ad3f59730cba9981829fc8a3564116536b90e1 Mon Sep 17 00:00:00 2001 From: Eugene Morozov Date: Wed, 7 Oct 2026 14:13:22 -0700 Subject: [PATCH 31/49] Remove manual fixture tooling and test-only SDK entry points --- library/build.gradle | 5 - .../SdrConversionFixtureTests.kt | 119 ------------------ .../standardization/Media3SdrExportEngine.kt | 8 +- .../standardization/SdrConversionAdapter.kt | 2 +- .../SdrMedia3ConfigurationTests.kt | 26 +++- 5 files changed, 23 insertions(+), 137 deletions(-) delete mode 100644 library/src/androidTest/java/com/mux/video/upload/internal/standardization/SdrConversionFixtureTests.kt diff --git a/library/build.gradle b/library/build.gradle index 5446e470..d7eff249 100644 --- a/library/build.gradle +++ b/library/build.gradle @@ -35,9 +35,6 @@ android { buildFeatures { buildConfig true } - testOptions { - targetSdk = 36 - } } muxDistribution { @@ -101,6 +98,4 @@ dependencies { testImplementation 'androidx.test.ext:junit:1.2.1' testImplementation "io.mockk:mockk:1.13.9" testImplementation 'org.robolectric:robolectric:4.11.1' - androidTestImplementation 'androidx.test:runner:1.6.2' - androidTestImplementation 'androidx.test.ext:junit:1.2.1' } diff --git a/library/src/androidTest/java/com/mux/video/upload/internal/standardization/SdrConversionFixtureTests.kt b/library/src/androidTest/java/com/mux/video/upload/internal/standardization/SdrConversionFixtureTests.kt deleted file mode 100644 index de9ce6f4..00000000 --- a/library/src/androidTest/java/com/mux/video/upload/internal/standardization/SdrConversionFixtureTests.kt +++ /dev/null @@ -1,119 +0,0 @@ -package com.mux.video.upload.internal.standardization - -import androidx.test.ext.junit.runners.AndroidJUnit4 -import androidx.test.platform.app.InstrumentationRegistry -import com.mux.video.upload.internal.InputStandardization -import com.mux.video.upload.internal.MaximumResolution -import org.junit.Assert.* -import org.junit.Assume.assumeTrue -import org.junit.Test -import org.junit.runner.RunWith -import java.io.File -import java.util.concurrent.CountDownLatch -import java.util.concurrent.TimeUnit -import java.util.concurrent.atomic.AtomicInteger - -/** Runs the adapter against externally staged fixtures without media in the APK. */ -@RunWith(AndroidJUnit4::class) -class SdrConversionFixtureTests { - private val instrumentation = InstrumentationRegistry.getInstrumentation() - private val context = instrumentation.targetContext - private val fixtures = File(instrumentation.context.filesDir, "sdr-fixtures") - - private fun source(name: String): File { - assumeTrue("Stage SDR fixtures before running this suite", fixtures.isDirectory) - val fixture = File(fixtures, name) - assertTrue("Missing fixture $name", fixture.isFile) - return File(context.cacheDir, "mux-upload/source-$name").apply { - parentFile!!.mkdirs(); fixture.copyTo(this, overwrite = true) - } - } - private fun inspect(file: File): Triple { - val metadata = (MediaMetadataInspector().inspect(file) as MetadataInspectionResult.Success).inspection - val samples = StandardInputTimelineInspector().inspect(file, metadata) - assertEquals(SampleScanStatus.Complete, samples.status) - // Direct adapter exercise: these test flags do not establish hardware capability proof. - val capabilities = PlanningCapabilities(true, true, setOf(VideoCodec.H264, VideoCodec.Hevc), PolicyRequirement.entries.toSet(), - canPrepareCompliantAacAudio = true) - val action = StandardInputPlanner().plan(samples.facts, - InputStandardization(true, MaximumResolution.Preset1280x720), capabilities).action - assertTrue("${file.name}: $action\n${samples.facts}\n$metadata", action is StandardInputAction.Convert) - return Triple(metadata, samples, (action as StandardInputAction.Convert).conversion) - } - private fun ownedFiles(): Set = File(context.cacheDir, "mux-upload/standard-input").listFiles() - ?.map { it.name }?.toSet() ?: emptySet() - - private fun export(name: String): SdrConversionResult { - val file = source(name) - val original = file.readBytes() - val (metadata, samples, conversion) = inspect(file) - val before = ownedFiles() - val done = CountDownLatch(1) - val calls = AtomicInteger() - var result: SdrConversionResult? = null - val attempt = SdrConversionAdapter(context).start(file, metadata, samples, conversion) { - result = it; calls.incrementAndGet(); done.countDown() - } - try { - assertTrue("Timed out $name", done.await(90, TimeUnit.SECONDS)) - File(instrumentation.context.filesDir, "sdr-evidence").apply { mkdirs() }.let { area -> - File(area, "$name.txt").writeText("$metadata\n${samples.facts}\n${samples.timeline}\n$conversion\n$result") - } - assertEquals(1, calls.get()) - assertArrayEquals(original, file.readBytes()) - if (result is SdrConversionResult.Completed) { - val completed = result as SdrConversionResult.Completed - assertEquals(MediaFact.Known(conversion.outputCodec), completed.facts.videoCodec) - assertEquals(MediaFact.Known(conversion.outputDimensions), completed.facts.encodedDimensions) - assertEquals(MediaFact.Known(0), completed.facts.rotationDegrees) - assertTrue(StandardInputOutputValidator().validateGeneratedOutput(completed.output.file, samples.facts, - samples.timeline, conversion) is StandardInputOutputValidation.Accepted) - assertTrue(completed.output.delete()) - } - assertEquals(before, ownedFiles()) - return checkNotNull(result) - } finally { - attempt.cancel(); (result as? SdrConversionResult.Completed)?.output?.delete(); file.delete() - } - } - @Test fun h264ResizeSilentPortraitNonAacAndCadenceExportsPassReinspection() { - for (name in listOf("multi-audio.mp4", "no-audio.mp4", "rotated.mp4", "mp3-stereo.mp4", "high-rate.mp4", "vfr.mp4")) { - val result = export(name) - assertTrue("$name: $result", result is SdrConversionResult.Completed) - if (name == "multi-audio.mp4") assertEquals(MediaFact.Known(listOf(AudioTrack(MediaFact.Known( - AudioFormat.Aac(AudioChannelLayout.Stereo))))), (result as SdrConversionResult.Completed).facts.audioTracks) - } - } - @Test fun hevcEitherPassesFullValidationOrRejectsSafelyWithoutChangingCodecFamily() { - for (name in listOf("hevc-small-open-gop.mp4", "hevc-long-open-gop.mp4")) { - val result = export(name) - assertTrue("$name: $result", result is SdrConversionResult.Completed || result in listOf( - SdrConversionResult.Failed(SdrConversionFailure.CapabilityUnavailable), - SdrConversionResult.Failed(SdrConversionFailure.Export), SdrConversionResult.Failed(SdrConversionFailure.OutputInvalid))) - } - } - @Test fun linearTaggedInputFallsBackBeforeExport() { - assertEquals(SdrConversionResult.Failed(SdrConversionFailure.UnsupportedPlan), export("linear.mp4")) - } - @Test fun cancellationRemovesOnlyOwnedPartialAndDoesNotPublishLateResults() { - val file = source("multi-audio.mp4") - val original = file.readBytes() - val (metadata, samples, conversion) = inspect(file) - val before = ownedFiles() - val done = CountDownLatch(1) - val calls = AtomicInteger() - var result: SdrConversionResult? = null - val attempt = SdrConversionAdapter(context).start(file, metadata, samples, conversion) { - result = it; calls.incrementAndGet(); done.countDown() - } - try { - attempt.cancel() - assertTrue(done.await(30, TimeUnit.SECONDS)) - assertEquals(SdrConversionResult.Cancelled, result) - Thread.sleep(500) - assertEquals(1, calls.get()) - assertEquals(before, ownedFiles()) - assertArrayEquals(original, file.readBytes()) - } finally { attempt.cancel(); file.delete() } - } -} diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/Media3SdrExportEngine.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/Media3SdrExportEngine.kt index ff606d0f..2398bd91 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/Media3SdrExportEngine.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/Media3SdrExportEngine.kt @@ -105,10 +105,7 @@ internal class FirstAudioTrackSelector(context: Context, private val trackId: St .setExceedAudioConstraintsIfNecessary(true).setExceedRendererCapabilitiesIfNecessary(true)) } override fun selectAudioTrack(mappedTrackInfo: MappedTrackInfo, rendererFormatSupports: Array>, - rendererMixedMimeTypeAdaptationSupports: IntArray, params: Parameters): Pair? = - selectFirstAudioTrack(mappedTrackInfo) - - internal fun selectFirstAudioTrack(mappedTrackInfo: MappedTrackInfo): Pair? { + rendererMixedMimeTypeAdaptationSupports: IntArray, params: Parameters): Pair? { if (trackId == null) return null for (renderer in 0 until mappedTrackInfo.rendererCount) { if (mappedTrackInfo.getRendererType(renderer) != C.TRACK_TYPE_AUDIO) continue @@ -154,9 +151,6 @@ internal class StrictSdrEncoderFactory( return DefaultCodec(context, exact, media, capability.name, false, null) } - internal fun videoConfiguration(format: Format): MediaFormat = - SdrVideoEncoderConfiguration.create(requestedVideoFormat(format), targets, capability.level) - private fun requestedVideoFormat(format: Format): Format { check(format.sampleMimeType == targets.videoMime && format.width == conversion.outputDimensions.width && format.height == conversion.outputDimensions.height && format.rotationDegrees == 0 && diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt index 81448954..b2eec190 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt @@ -27,7 +27,7 @@ internal class SdrGeneratedFile private constructor(val file: File) { } } -/** Small Media3 boundary so terminal races and cleanup can be tested without a platform encoder. */ +/** Internal boundary for Media3 export and cancellation. */ internal interface SdrExportEngine { fun start(input: File, output: File, completed: (String?, String?) -> Unit, failed: (SdrConversionFailure) -> Unit) diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/SdrMedia3ConfigurationTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/SdrMedia3ConfigurationTests.kt index fed2acee..055d5caf 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/SdrMedia3ConfigurationTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/SdrMedia3ConfigurationTests.kt @@ -3,6 +3,7 @@ package com.mux.video.upload.internal.standardization import android.content.Context import android.media.MediaCodecInfo import android.media.MediaFormat +import android.util.Pair import androidx.media3.common.C import androidx.media3.common.ColorInfo import androidx.media3.common.Format @@ -10,6 +11,7 @@ import androidx.media3.common.MediaItem import androidx.media3.common.TrackGroup import androidx.media3.exoplayer.source.TrackGroupArray import androidx.media3.exoplayer.trackselection.MappingTrackSelector +import androidx.media3.exoplayer.trackselection.ExoTrackSelection import androidx.media3.transformer.Composition import androidx.media3.transformer.EditedMediaItem import androidx.media3.transformer.EditedMediaItemSequence @@ -26,6 +28,7 @@ import org.robolectric.annotation.Config import org.robolectric.annotation.Implementation import org.robolectric.annotation.Implements import org.robolectric.shadows.MediaCodecInfoBuilder +import java.lang.reflect.InvocationTargetException @RunWith(RobolectricTestRunner::class) @Config(manifest = Config.NONE, sdk = [28], shadows = [SupportedFormatShadow::class]) @@ -44,7 +47,7 @@ class SdrMedia3ConfigurationTests { @Test fun strictFactoryRejectsMimeDimensionsRotationAndLinearBeforeCodecCreation() { val factory = StrictSdrEncoderFactory(context, conversion, targets, SdrEncoderCapability("chosen", 4)) - val configuration = factory.videoConfiguration(format) + val configuration = SdrVideoEncoderConfiguration.create(format, targets, 4) assertEquals(targets.platformProfile, configuration.getInteger(MediaFormat.KEY_PROFILE)) assertEquals(4, configuration.getInteger(MediaFormat.KEY_LEVEL)) assertEquals(18, configuration.getInteger(MediaFormat.KEY_I_FRAME_INTERVAL)) @@ -52,7 +55,7 @@ class SdrMedia3ConfigurationTests { for (bad in listOf(format.buildUpon().setSampleMimeType("video/hevc").build(), format.buildUpon().setWidth(640).build(), format.buildUpon().setRotationDegrees(90).build(), format.buildUpon().setColorInfo(ColorInfo.SDR_BT709_LIMITED.buildUpon().setColorTransfer(C.COLOR_TRANSFER_LINEAR).build()).build())) { - assertThrows(IllegalStateException::class.java) { factory.videoConfiguration(bad) } + assertThrows(IllegalStateException::class.java) { factory.createForVideoEncoding(bad, null) } } } @@ -139,10 +142,23 @@ class SdrMedia3ConfigurationTests { every { getRendererType(0) } returns C.TRACK_TYPE_AUDIO every { getTrackGroups(0) } returns groups } - val selection = FirstAudioTrackSelector(context, "2").selectFirstAudioTrack(mapped)!! + val selection = selectAudioTrack(FirstAudioTrackSelector(context, "2"), mapped)!! assertEquals("2", selection.first.group.getFormat(selection.first.tracks.single()).id) - assertNull(FirstAudioTrackSelector(context, null).selectFirstAudioTrack(mapped)) - assertThrows(IllegalStateException::class.java) { FirstAudioTrackSelector(context, "missing").selectFirstAudioTrack(mapped) } + assertNull(selectAudioTrack(FirstAudioTrackSelector(context, null), mapped)) + assertThrows(IllegalStateException::class.java) { selectAudioTrack(FirstAudioTrackSelector(context, "missing"), mapped) } + } + + private fun selectAudioTrack(selector: FirstAudioTrackSelector, + mapped: MappingTrackSelector.MappedTrackInfo): Pair? { + val method = FirstAudioTrackSelector::class.java.declaredMethods.single { it.name == "selectAudioTrack" } + method.isAccessible = true + try { + @Suppress("UNCHECKED_CAST") + return method.invoke(selector, mapped, emptyArray>(), intArrayOf(), selector.parameters) + as Pair? + } catch (error: InvocationTargetException) { + throw error.cause!! + } } @Test fun exportListenerRoutesMuxerWatchdogAndFallbackToFailureAndChecksCopiedAudio() { From 37125bd5753a12549bccc660506cae4987613b92 Mon Sep 17 00:00:00 2001 From: Eugene Morozov Date: Thu, 8 Oct 2026 14:22:21 -0700 Subject: [PATCH 32/49] Follow adopted upload sessions and consolidate lifecycle tests --- .../com/mux/video/upload/api/MuxUpload.kt | 6 +- .../UploadPreparationLifecycleTests.kt | 208 +++++++++--------- .../standardization/UploadPreparationTests.kt | 11 - 3 files changed, 103 insertions(+), 122 deletions(-) diff --git a/library/src/main/java/com/mux/video/upload/api/MuxUpload.kt b/library/src/main/java/com/mux/video/upload/api/MuxUpload.kt index 78e88090..ef8b26c2 100644 --- a/library/src/main/java/com/mux/video/upload/api/MuxUpload.kt +++ b/library/src/main/java/com/mux/video/upload/api/MuxUpload.kt @@ -6,6 +6,7 @@ import com.mux.video.upload.MuxUploadSdk import com.mux.video.upload.api.MuxUpload.Builder import com.mux.video.upload.internal.MaximumResolution import com.mux.video.upload.internal.UploadInfo +import com.mux.video.upload.internal.UploadSession import com.mux.video.upload.internal.update import com.mux.video.upload.internal.writeUploadState import com.mux.video.upload.internal.forgetUploadState @@ -93,6 +94,7 @@ class MuxUpload private constructor( private var progressListener: UploadEventListener? = null private var statusListener: UploadEventListener? = null private var observerJob: Job? = null + private var observedSession: UploadSession? = null private var cancelled = false private var deliveredResultStatus: UploadStatus? = null private var deliveredStatus: UploadStatus? = null @@ -358,7 +360,9 @@ class MuxUpload private constructor( if (cancelled || (resultListener == null && progressListener == null && statusListener == null)) { observerJob?.cancel("no listeners") observerJob = null - } else if (observerJob?.isActive != true) { + } else if (observerJob?.isActive != true || observedSession !== uploadInfo.session) { + observerJob?.cancel("switching upload session") + observedSession = uploadInfo.session observerJob = newObserveProgressJob(uploadInfo) } } diff --git a/library/src/test/java/com/mux/video/upload/internal/UploadPreparationLifecycleTests.kt b/library/src/test/java/com/mux/video/upload/internal/UploadPreparationLifecycleTests.kt index 3f2bb832..e93bff0f 100644 --- a/library/src/test/java/com/mux/video/upload/internal/UploadPreparationLifecycleTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/UploadPreparationLifecycleTests.kt @@ -71,6 +71,7 @@ class UploadPreparationLifecycleTests : AbsRobolectricTest() { @Test fun originalBytesCompleteWithCumulativeProgressAndOneResult() { val upload = observe(MuxUpload.create(info())) + upload.setProgressListener(null) upload.start() pump { upload.isSuccessful } assertArrayEquals(source.readBytes(), chunks.flatMap { it.sliceData.toList() }.toByteArray()) @@ -78,6 +79,7 @@ class UploadPreparationLifecycleTests : AbsRobolectricTest() { assertEquals(16L, upload.currentProgress.bytesUploaded) assertEquals(16L, upload.currentProgress.totalBytes) assertEquals(1, results.size) + assertTrue(statuses.any { it is UploadStatus.UploadSuccess }) assertTrue(readAllCachedUploads().isEmpty()) assertTrue(source.exists()); assertTrue(sibling.exists()) var lateResults = 0 @@ -104,25 +106,23 @@ class UploadPreparationLifecycleTests : AbsRobolectricTest() { assertTrue(source.exists()); assertTrue(sibling.exists()) } - @Test fun automaticLegacyRestorationUsesOriginalOffsetWithoutPreparation() { - writeUploadState(info(), MuxUpload.Progress(bytesUploaded = 8, totalBytes = 16)) - // Remove new fields to model a pre-feature persistence record. - val prefs = context.getSharedPreferences("mux_upload", 0) - val json = org.json.JSONArray(prefs.getString("uploads", null)) - json.getJSONObject(0).getJSONObject("data").remove("generated_resume_blocked") - json.getJSONObject(0).getJSONObject("data").remove("input_standardization") - json.getJSONObject(0).getJSONObject("data").remove("original_selected") - json.getJSONObject(0).getJSONObject("data").put("state", 0) - prefs.edit().putString("uploads", json.toString()).commit() - work = { _, _, _ -> awaitCancellation() } - MuxUploadSdk.initialize(context, true) - pump { chunks.isNotEmpty() } - val restored = MuxUploadManager.resumeAllCachedJobs().single() - assertTrue(restored.isRunning) - assertTrue(restored.uploadStatus is UploadStatus.Uploading) - assertEquals(0, preparations) - assertEquals(8L, chunks.single().startByte) - assertArrayEquals(source.readBytes().sliceArray(8..11), chunks.single().sliceData) + @Test fun automaticLegacyRestorationUsesZeroAndNonzeroOriginalOffsetsWithoutPreparation() { + for (offset in listOf(0L, 8L)) { + val firstChunk = chunks.size + writeUploadState(info(), MuxUpload.Progress(bytesUploaded = offset, totalBytes = 16)) + val prefs = context.getSharedPreferences("mux_upload", 0) + val entries = org.json.JSONArray(prefs.getString("uploads", null)) + val data = entries.getJSONObject(0).getJSONObject("data") + for (field in listOf("generated_resume_blocked", "input_standardization", "original_selected")) data.remove(field) + data.put("state", 0) + prefs.edit().putString("uploads", entries.toString()).commit() + MuxUploadSdk.initialize(context, true) + val restored = MuxUploadManager.findUploadByFile(source)!! + pump { restored.isSuccessful } + assertEquals(0, preparations) + assertEquals(offset, chunks[firstChunk].startByte) + assertArrayEquals(source.readBytes().sliceArray(offset.toInt() until offset.toInt() + 4), chunks[firstChunk].sliceData) + } } @Test fun preparationPauseStopsWorkIsSilentAndResumesOnlyOnStart() { @@ -146,26 +146,6 @@ class UploadPreparationLifecycleTests : AbsRobolectricTest() { assertEquals(2, preparations) } - @Test fun initializationDoesNotAutomaticallyResumeAnExplicitlyPausedRecord() { - work = { _, _, _ -> awaitCancellation() } - val upload = MuxUpload.create(info()) - upload.start() - pump { chunks.size == 1 } - upload.pause() - dispatcher.scheduler.runCurrent() - MuxUploadManager.jobFinished(internalInfo(upload), false) - MuxUploadSdk.initialize(context, true) - dispatcher.scheduler.runCurrent() - assertEquals(1, chunks.size) - assertEquals(1, readAllCachedUploads().size) - val restored = MuxUploadManager.findUploadByFile(source)!! - assertTrue(restored.isPaused) - assertEquals(1, MuxUploadManager.allUploadJobs().size) - restored.start() - pump { chunks.size == 2 } - assertEquals(1, preparations) - } - @Test fun uploadPauseUsesAcknowledgedProgressAndSuppressesLateEvents() { var late: MutableSharedFlow? = null work = { chunk, _, flow -> @@ -181,6 +161,10 @@ class UploadPreparationLifecycleTests : AbsRobolectricTest() { assertEquals(4L, upload.currentProgress.bytesUploaded) assertEquals(4L, readLastByteForFile(info())) val count = statuses.size + val progressCount = progress.size + upload.pause(); dispatcher.scheduler.runCurrent() + assertEquals(count, statuses.size) + assertEquals(progressCount, progress.size) late!!.tryEmit(MuxUpload.Progress(bytesUploaded = 4)) dispatcher.scheduler.runCurrent() assertEquals(count, statuses.size) @@ -334,18 +318,6 @@ class UploadPreparationLifecycleTests : AbsRobolectricTest() { assertTrue(source.exists()); assertTrue(sibling.exists()) } - @Test fun staleCompletionCannotRemoveReplacementManagerJob() { - work = { _, _, _ -> awaitCancellation() } - val upload = MuxUpload.create(info()) - upload.start() - pump { chunks.isNotEmpty() } - val old = internalInfo(upload) - upload.start(forceRestart = true) - pump { chunks.size == 2 } - MuxUploadManager.jobFinished(old) - assertTrue(MuxUploadManager.findUploadByFile(source)!!.isRunning) - } - @Test fun unmanagedBuilderPauseAndCancelHaveSameListenerContract() { work = { _, _, _ -> awaitCancellation() } val upload = observe(MuxUpload.Builder("https://example.invalid/upload", source) @@ -392,7 +364,6 @@ class UploadPreparationLifecycleTests : AbsRobolectricTest() { assertFalse(generated.file.exists()) } - @Test fun unmanagedSuccessClearsResumeStateAndCannotBeAutomaticallyRestored() { val upload = MuxUpload.Builder(info().remoteUri, source).manageUploadTask(false).build() upload.start(); pump { upload.isSuccessful } @@ -463,20 +434,6 @@ class UploadPreparationLifecycleTests : AbsRobolectricTest() { assertEquals(0L, chunks.last().startByte) } - @Test fun oldDestinationHandleCannotPauseCancelOrRestartItsReplacement() { - work = { _, _, _ -> awaitCancellation() } - val old = MuxUpload.create(info()) - old.start(); pump { chunks.size == 1 } - val replacement = MuxUpload.create(info().copy(remoteUri = Uri.parse("https://example.invalid/new-upload"))) - replacement.start(); pump { chunks.size == 2 } - old.pause(); old.start(forceRestart = true); old.cancel() - dispatcher.scheduler.runCurrent() - assertTrue(replacement.isRunning) - assertFalse(internalInfo(replacement).uploadJob!!.isCancelled) - assertTrue(MuxUploadManager.findUploadByFile(source)!!.isRunning) - assertEquals(2, chunks.size) - } - @Test fun existingHandleFollowsResumedAttemptAndItsCompletion() { work = { _, _, _ -> awaitCancellation() } val old = observe(MuxUpload.create(info())) @@ -524,35 +481,25 @@ class UploadPreparationLifecycleTests : AbsRobolectricTest() { assertEquals(4L, readLastByteForFile(current)) } - @Test fun staleCompletionCannotEraseNewAttemptPersistenceAtSameDestination() { - val oldAttempt = UploadAttempt(UploadPreparationState(), MuxUpload.Progress()) - val old = info().update(attempt = oldAttempt, statusFlow = oldAttempt.status) - writeUploadState(old, MuxUpload.Progress(bytesUploaded = 4)) - val newAttempt = UploadAttempt(UploadPreparationState(), MuxUpload.Progress(), oldAttempt.id) - val next = info().update(attempt = newAttempt, statusFlow = newAttempt.status) - writeUploadState(next, MuxUpload.Progress(bytesUploaded = 8)) - MuxUploadManager.jobFinished(old) - assertEquals(8L, readLastByteForFile(next)) - } - - @Test fun repeatedPauseDoesNotReplayStatusOrProgress() { - work = { _, _, _ -> awaitCancellation() } - val upload = observe(MuxUpload.create(info())) + @Test fun staleCompletionPreservesReplacementManagerJobAndItsPersistence() { + work = { _, upload, _ -> + val acknowledged = MuxUpload.Progress(bytesUploaded = 4, totalBytes = 16) + upload.attempt!!.acknowledge(acknowledged) { writeUploadState(upload, acknowledged) } + awaitCancellation() + } + val upload = MuxUpload.create(info()) upload.start(); pump { chunks.size == 1 } + val stale = internalInfo(upload) upload.pause(); dispatcher.scheduler.runCurrent() - val count = statuses.size - val progressCount = progress.size - upload.pause(); dispatcher.scheduler.runCurrent() - assertEquals(count, statuses.size) - assertEquals(progressCount, progress.size) - } - - @Test fun clearingProgressListenerKeepsResultAndStatusListenersActive() { - val upload = observe(MuxUpload.create(info())) - upload.setProgressListener(null) - upload.start(); pump { upload.isSuccessful } - assertEquals(1, results.size) - assertTrue(statuses.any { it is UploadStatus.UploadSuccess }) + work = { _, current, _ -> + val acknowledged = MuxUpload.Progress(bytesUploaded = 8, totalBytes = 16) + current.attempt!!.acknowledge(acknowledged) { writeUploadState(current, acknowledged) } + awaitCancellation() + } + upload.start(); pump { chunks.size == 2 } + MuxUploadManager.jobFinished(stale) + assertTrue(MuxUploadManager.findUploadByFile(source)!!.isRunning) + assertEquals(8L, readLastByteForFile(internalInfo(upload))) } @Test fun transportFailureReportsSafeCategoryAndRetainsPublicException() { @@ -569,20 +516,6 @@ class UploadPreparationLifecycleTests : AbsRobolectricTest() { coVerify { metrics.reportUploadFailed(any(), any(), any(), "Upload failed: Io", any(), any()) } } - - @Test fun legacyZeroByteRecordStillResumesOriginalWithoutPreparation() { - writeUploadState(info(), MuxUpload.Progress(totalBytes = 16)) - val prefs = context.getSharedPreferences("mux_upload", 0) - val json = org.json.JSONArray(prefs.getString("uploads", null)) - json.getJSONObject(0).getJSONObject("data").remove("original_selected") - json.getJSONObject(0).getJSONObject("data").put("state", 0) - prefs.edit().putString("uploads", json.toString()).commit() - val restored = MuxUploadManager.resumeAllCachedJobs().single() - pump { restored.isSuccessful } - assertEquals(0, preparations) - assertEquals(0L, chunks.first().startByte) - } - @Test fun generatedDestinationBlockAlsoRejectsADifferentSourceFile() { val state = UploadPreparationState().apply { generatedRequestStarted = true } val attempt = UploadAttempt(state, MuxUpload.Progress(totalBytes = 16)) @@ -607,7 +540,6 @@ class UploadPreparationLifecycleTests : AbsRobolectricTest() { assertTrue(managed.isRunning) } - @Test fun failedHandleFollowsManagerRetryAndReceivesItsNewResult() { work = { chunk, _, _ -> if (chunk.startByte == 0L) MuxUpload.Progress(bytesUploaded = 4) @@ -708,12 +640,15 @@ class UploadPreparationLifecycleTests : AbsRobolectricTest() { assertTrue(old.error is UploadReplacedException) assertTrue(results.single().exceptionOrNull() is UploadReplacedException) assertFalse(old.isRunning) - old.start(forceRestart = true) + old.pause(); old.start(forceRestart = true) var late: Result? = null val lateWaiter = scope.launch { late = old.awaitSuccess() } pump { lateWaiter.isCompleted } assertFalse(lateWaiter.isCancelled) assertTrue(late!!.exceptionOrNull() is UploadReplacedException) + old.cancel(); dispatcher.scheduler.runCurrent() + assertFalse(internalInfo(replacement).uploadJob!!.isCancelled) + assertTrue(MuxUploadManager.findUploadByFile(source)!!.isRunning) assertTrue(replacement.isRunning) assertEquals(2, chunks.size) assertEquals(1, results.size) @@ -799,7 +734,7 @@ class UploadPreparationLifecycleTests : AbsRobolectricTest() { val upload = MuxUpload.create(info()) upload.start(); pump { upload.isSuccessful } // Selection once, four acknowledged chunk checkpoints, and terminal record removal. - verify(exactly = 6) { prefs.edit() } + verify(atMost = 6) { prefs.edit() } assertTrue(readAllCachedUploads().isEmpty()) } @@ -837,7 +772,60 @@ class UploadPreparationLifecycleTests : AbsRobolectricTest() { assertTrue(paused.currentProgress.startTime > 0) assertTrue(paused.currentProgress.updatedTime >= paused.currentProgress.startTime) assertTrue(MuxUploadManager.findUploadByFile(sibling)!!.isRunning) + assertEquals(sibling.length(), chunks.single().totalFileSize) + assertEquals(2, MuxUploadManager.allUploadJobs().size) verify(exactly = 0) { readUploadResumeState(any()) } + unmockkStatic(::readUploadResumeState) + paused.start(); pump { chunks.size == 2 } + assertTrue(paused.isRunning) + assertEquals(8L, chunks.last().startByte) + } + + @Test fun freshBuilderListenersFollowExistingRunningPausedFailedAndActiveSessions() { + for (existing in listOf("restored running", "restored paused", "failed", "active")) { + val release = CompletableDeferred() + val firstChunk = chunks.size + work = { chunk, _, _ -> release.await(); MuxUpload.Progress(bytesUploaded = chunk.contentLength.toLong()) } + when (existing) { + "restored running", "restored paused" -> { + writeUploadState(info(), MuxUpload.Progress(bytesUploaded = 8, totalBytes = 16)) + if (existing == "restored running") { + val prefs = context.getSharedPreferences("mux_upload", 0) + val entries = org.json.JSONArray(prefs.getString("uploads", null)) + entries.getJSONObject(0).getJSONObject("data").put("state", 0) + prefs.edit().putString("uploads", entries.toString()).commit() + } + MuxUploadSdk.initialize(context, true) + } + "failed" -> { + work = { _, _, _ -> throw java.io.IOException("Test failure") } + val failed = MuxUpload.create(info()) + failed.start(); pump { failed.error != null } + work = { chunk, _, _ -> release.await(); MuxUpload.Progress(bytesUploaded = chunk.contentLength.toLong()) } + } + else -> MuxUpload.create(info()).start() + } + val current = MuxUploadManager.findUploadByFile(source)!! + val completions = mutableListOf>() + val updates = mutableListOf() + val states = mutableListOf() + val fresh = MuxUpload.Builder(info().remoteUri, source).build().apply { + setResultListener { completions += it } + setProgressListener { updates += it } + setStatusListener { states += it } + } + // Ensure the observer is already watching the Builder's original session. + dispatcher.scheduler.runCurrent() + fresh.start(); pump { chunks.size > firstChunk && fresh.isRunning } + release.complete(Unit) + pump { fresh.isSuccessful } + assertEquals(existing, 1, completions.size) + assertTrue(existing, completions.single().isSuccess) + assertEquals(existing, source.length(), updates.last().bytesUploaded) + assertTrue(existing, states.last() is UploadStatus.UploadSuccess) + assertTrue(existing, current.isSuccessful) + assertTrue(MuxUploadManager.allUploadJobs().isEmpty()) + } } private fun observe(upload: MuxUpload) = upload.apply { diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/UploadPreparationTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/UploadPreparationTests.kt index b4fbd208..b41c8cd6 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/UploadPreparationTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/UploadPreparationTests.kt @@ -45,16 +45,6 @@ class UploadPreparationTests : AbsRobolectricTest() { } } - @Test fun unverifiedGeneratedResumeGatesConversionBeforeExportForBothFamilies() = runBlocking { - for (codec in listOf(VideoCodec.H264, VideoCodec.Hevc)) { - val facts = source(codec).copy(averageBitrate = known(9_000_000L)) - val preparation = UploadPreparation(inspectMetadata = { MetadataInspectionResult.Success(metadata(facts)) }, - inspectSamples = { _, _, _ -> MediaSampleInspection(facts, SampleScanStatus.Complete, timeline = timeline) }, - convert = { _, _, _, _, _ -> error("Unverified generated resume must select original") }) - assertEquals(PreparedUpload.Original(PreparationDiagnostic.GeneratedResumeUnavailable), - preparation.prepare(upload(), context)) - } - } @Test fun multipleAudioTracksSelectOriginalEvenWhenGeneratedResumeIsVerified() = runBlocking { for (codec in listOf(VideoCodec.H264, VideoCodec.Hevc)) { val facts = source(codec).copy(averageBitrate = known(9_000_000L), audioTracks = known(listOf( @@ -112,7 +102,6 @@ class UploadPreparationTests : AbsRobolectricTest() { @Test fun adapterCompletionRacingPauseTransfersVerifiedOwnershipBeforeCoroutineDispatch() = bridgeRace(cancel = false) @Test fun adapterCompletionRacingCancelDeletesOnlyItsOwnedFile() = bridgeRace(cancel = true) - @Test fun closedGateSkipsAllMediaInspectionAndExport() = runBlocking { val preparation = UploadPreparation(inspectMetadata = { error("Gate must precede metadata") }, inspectSamples = { _, _, _ -> error("Gate must precede samples") }, From 6e66ea27de97f2278bdb898418225122e69d0743 Mon Sep 17 00:00:00 2001 From: Eugene Morozov Date: Wed, 7 Oct 2026 15:32:41 -0700 Subject: [PATCH 33/49] Use actual VFR GOP windows and reject unproven rate hints --- .../standardization/Media3SdrExportEngine.kt | 10 ++-- .../standardization/SdrConversionAdapter.kt | 3 +- .../standardization/SdrEncodingTargets.kt | 38 +++++++++------ .../SdrConversionAdapterTests.kt | 45 ++++++++++++++++-- .../SdrMedia3ConfigurationTests.kt | 46 ++++++++++++++----- 5 files changed, 105 insertions(+), 37 deletions(-) diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/Media3SdrExportEngine.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/Media3SdrExportEngine.kt index 2398bd91..3d11b082 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/Media3SdrExportEngine.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/Media3SdrExportEngine.kt @@ -133,8 +133,8 @@ internal class StrictSdrEncoderFactory( ) : Codec.EncoderFactory { private val audioDelegate = DefaultEncoderFactory.Builder(context) .setEnableFallback(false).setEnableFormatFallback(false) - .setRequestedAudioEncoderSettings(AudioEncoderSettings.Builder().setBitrate(160_000) - .setProfile(MediaCodecInfo.CodecProfileLevel.AACObjectLC).build()).build() + .setRequestedAudioEncoderSettings(AudioEncoderSettings.Builder().setBitrate(SdrEncodingTargets.AAC_BITRATE) + .setProfile(SdrEncodingTargets.AAC_PROFILE).build()).build() override fun videoNeedsEncoding() = true override fun audioNeedsEncoding() = targets.audioChannels != null && !targets.copyAac @@ -165,6 +165,9 @@ internal object SdrVideoEncoderConfiguration { fun create(format: Format, targets: SdrEncodingTargets, level: Int, sdk: Int = Build.VERSION.SDK_INT, device: String = Build.DEVICE, soc: String = if (Build.VERSION.SDK_INT >= 31) Build.SOC_MODEL else ""): MediaFormat { + // Media3 forces a 30-fps hint on this device. Other rates have unproven GOP/rate + // control here; preflight rejects them instead of changing the planned settings. + check(sdk >= 30 || device != "joyeuse" || format.frameRate == 30f) val media = MediaFormatUtil.createMediaFormatFromFormat(format) media.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface) media.setInteger(MediaFormat.KEY_BITRATE_MODE, MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_VBR) @@ -178,9 +181,6 @@ internal object SdrVideoEncoderConfiguration { val overflow = sdk in 31..34 && soc in setOf("SM8550", "SM7450", "SM6450", "SC9863A", "T612", "T606", "T603") media.setInteger(MediaFormat.KEY_OPERATING_RATE, if (sdk == 26) 30 else if (overflow) 1000 else Int.MAX_VALUE) } - // Redmi Note 9 Pro rejects high KEY_FRAME_RATE. This is a codec hint; timestamps - // and Media3's cadence remain planned. Lowering it only shortens the requested GOP. - if (sdk < 30 && device == "joyeuse" && format.frameRate > 30) media.setFloat(MediaFormat.KEY_FRAME_RATE, 30f) return media } } diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt index b2eec190..3c365baa 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt @@ -4,6 +4,7 @@ import android.content.Context import android.os.Handler import android.os.HandlerThread import android.os.Looper +import androidx.media3.common.MimeTypes import java.io.File import java.util.concurrent.atomic.AtomicBoolean @@ -97,7 +98,7 @@ internal class SdrConversionAdapter( exporter.start(input, owned.file, { videoMime, audioMime -> handler.post { guarded { if (isTerminal()) return@guarded - if (videoMime != targets.videoMime || audioMime != if (targets.audioChannels != null) "audio/mp4a-latm" else null) + if (videoMime != targets.videoMime || audioMime != if (targets.audioChannels != null) MimeTypes.AUDIO_AAC else null) return@guarded fail(SdrConversionFailure.CodecChanged) if (!sourceUnchanged()) return@guarded fail(SdrConversionFailure.SourceChanged) when (val validation = validate(owned.file, source.facts, source.timeline, conversion, cancelled::get)) { diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/SdrEncodingTargets.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/SdrEncodingTargets.kt index 9ce7d088..337a5f40 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/SdrEncodingTargets.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/SdrEncodingTargets.kt @@ -34,6 +34,9 @@ internal data class SdrEncodingTargets( } companion object { + const val AAC_BITRATE = 160_000 + const val AAC_PROFILE = CodecProfileLevel.AACObjectLC + fun from(conversion: StandardInputConversion, metadata: MediaMetadataInspection, source: MediaSampleInspection): SdrEncodingTargets? { val facts = source.facts @@ -85,16 +88,24 @@ internal data class SdrEncodingTargets( if (conversion.outputCadence == OutputCadence.PreserveSourceTimestamps && facts.cadence == MediaFact.Known(Cadence.Variable)) { val times = source.timeline.videoPresentationSeconds.valueOrNull!! - if (times.size < 2 || times.any { !it.isFinite() }) return null - var longestGap = 0.0 + val duration = source.timeline.videoDurationSeconds.valueOrNull!! + if (times.size < 2 || times.any { !it.isFinite() } || !duration.isFinite() || duration <= 0) return null + val end = times.first() + duration + if (!end.isFinite() || end < times.last()) return null for (index in 1 until times.size) { - val gap = times[index] - times[index - 1] - if (gap <= 0) return null - longestGap = maxOf(longestGap, gap) + if (times[index] <= times[index - 1]) return null + } + // Encoders commonly turn seconds into nominal-rate frames. Check every real + // frame window, including the final sample's duration, without retiming gaps. + while (gopSeconds >= 1) { + val frames = ceil(gopSeconds * rate).toInt() + val fits = times.indices.all { index -> + val windowEnd = times.getOrNull(index + frames) ?: end + windowEnd - times[index] <= interval * 0.9 + 2e-6 + } + if (fits) break + gopSeconds-- } - // Encoders commonly turn the interval into nominal-rate frames. Bound their - // elapsed span using the slowest observed gap; validation still proves output. - gopSeconds = minOf(gopSeconds, floor(interval * 0.9 / (rate * longestGap)).toInt()) if (gopSeconds < 1) return null } return SdrEncodingTargets( @@ -118,11 +129,10 @@ internal class SdrCapabilityPreflight( ) { fun videoEncoder(conversion: StandardInputConversion, metadata: MediaMetadataInspection, targets: SdrEncodingTargets, isCancelled: () -> Boolean): SdrEncoderCapability? { - val enumerationStarted = nanoTime() if (isCancelled()) return null val infos = codecInfos() - // Cold native enumeration has its own cooperative bound, not the query budget. - if (isCancelled() || nanoTime() - enumerationStarted >= 2_000_000_000L) return null + // Native enumeration cannot be interrupted; budget subsequent capability queries. + if (isCancelled()) return null val started = nanoTime() var queries = 0 fun withinBudget() = !isCancelled() && nanoTime() - started < 500_000_000L @@ -156,8 +166,8 @@ internal class SdrCapabilityPreflight( val audioMime = metadata.tracks.first { it.sourceTrackId.valueOrNull?.toString() == targets.firstAudioTrackId }.mimeType.valueOrNull ?: return null if (!supportsDecode(audioMime, MediaFormat.createAudioFormat(audioMime, targets.audioSampleRate!!, targets.audioChannels))) return null val aac = MediaFormat.createAudioFormat(MimeTypes.AUDIO_AAC, targets.audioSampleRate, targets.audioChannels) - aac.setInteger(MediaFormat.KEY_BIT_RATE, 160_000) - aac.setInteger(MediaFormat.KEY_AAC_PROFILE, CodecProfileLevel.AACObjectLC) + aac.setInteger(MediaFormat.KEY_BIT_RATE, SdrEncodingTargets.AAC_BITRATE) + aac.setInteger(MediaFormat.KEY_AAC_PROFILE, SdrEncodingTargets.AAC_PROFILE) if (infos.none { withinBudget() && it.isEncoder && it.supportedTypes.contains(MimeTypes.AUDIO_AAC) && supports(it, MimeTypes.AUDIO_AAC, aac) }) return null } @@ -170,8 +180,6 @@ internal class SdrCapabilityPreflight( val caps = info.getCapabilitiesForType(targets.videoMime) val level = caps.profileLevels.filter { it.profile == targets.platformProfile }.maxOfOrNull { it.level } ?: return@runCatching null val size = conversion.outputDimensions - // A device workaround may lower KEY_FRAME_RATE as a configure-time hint. - // Still guard the actual planned size/rate against advertised video limits. if (caps.videoCapabilities?.areSizeAndRateSupported(size.width, size.height, conversion.outputFrameRate) != true) return@runCatching null val requested = Format.Builder().setSampleMimeType(targets.videoMime).setWidth(size.width).setHeight(size.height) diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/SdrConversionAdapterTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/SdrConversionAdapterTests.kt index 29630eff..d47c554d 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/SdrConversionAdapterTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/SdrConversionAdapterTests.kt @@ -37,14 +37,14 @@ class SdrConversionAdapterTests { private fun targets(plan: StandardInputConversion = conversion, input: MediaFacts = facts, tracks: List = listOf(audio())) = SdrEncodingTargets.from(plan, metadata(tracks), samples(input)) - @Test fun vfrGopTargetUsesSlowestPresentationGapAndValidatorStillChecksActualOutput() { + @Test fun vfrGopTargetUsesActualFrameWindowsAndValidatorStillChecksActualOutput() { val times = (0..900).map { if (it <= 450) it / 60.0 else 7.5 + (it - 450) / 20.0 } val variable = facts.copy(cadence = known(Cadence.Variable)) val time = timeline.copy(durationSeconds = known(30.0), videoDurationSeconds = known(30.0), firstAudioDurationSeconds = known(30.0), videoPresentationSeconds = known(times)) val source = samples(variable).copy(timeline = time) val target = SdrEncodingTargets.from(conversion, metadata(), source)!! - assertTrue(target.keyframeIntervalSeconds in 1..12) + assertEquals(12, target.keyframeIntervalSeconds) // Model an encoder that converts seconds to a frame count at the nominal 30 fps. fun exportedGap(seconds: Int): Double { val frames = seconds * 30 @@ -57,6 +57,43 @@ class SdrConversionAdapterTests { assertTrue(validate(target.keyframeIntervalSeconds) is StandardInputOutputValidation.Accepted) } + @Test fun isolatedVfrPausesShortenGopOnlyByTheirActualSpan() { + for ((pause, expected) in listOf(0.05 to 17, 0.2 to 17, 0.61 to 17, 1.0 to 17, 3.0 to 15)) { + val times = (0 until 1800).map { it / 30.0 + if (it >= 900) pause else 0.0 } + assertEquals("pause=$pause", expected, vfrTargets(times, 60.0 + pause)!!.keyframeIntervalSeconds) + } + } + + @Test fun repeatedScreenRecordingPausesUseWorstActualWindow() { + val times = (0 until 1800).map { it / 30.0 + (it / 300) * 3.0 } + assertEquals(12, vfrTargets(times, 75.0)!!.keyframeIntervalSeconds) + } + + @Test fun highResolutionHevcVfrDoesNotRejectAnIsolatedHitch() { + val plan = conversion.copy(sourceCodec = VideoCodec.Hevc, outputCodec = VideoCodec.Hevc, + outputFrameRate = 60.0, outputPolicyLimits = StandardInputPolicyProfile.PublishedMux.highResolution) + val times = (0 until 3600).map { it / 60.0 + if (it >= 1800) 0.1 else 0.0 } + assertEquals(5, vfrTargets(times, 60.1, plan)!!.keyframeIntervalSeconds) + } + + @Test fun vfrGopIncludesLastSampleDurationAndRejectsUnusableWindows() { + val times = (0 until 60).map { it / 30.0 } + assertEquals(1, vfrTargets(times, 19.0)!!.keyframeIntervalSeconds) + assertNull(vfrTargets(times, 20.0)) + assertNull(vfrTargets(listOf(0.0, 0.1, 0.1), 1.0)) + assertNull(vfrTargets(listOf(0.0, Double.NaN), 1.0)) + assertNull(vfrTargets(times, 1.0)) + } + + private fun vfrTargets(times: List, duration: Double, + plan: StandardInputConversion = conversion): SdrEncodingTargets? { + val variable = facts.copy(cadence = known(Cadence.Variable), videoCodec = known(plan.sourceCodec), + frameRate = known(plan.outputFrameRate)) + val time = timeline.copy(durationSeconds = known(duration), videoDurationSeconds = known(duration), + firstAudioDurationSeconds = known(duration), videoPresentationSeconds = known(times)) + return SdrEncodingTargets.from(plan, metadata(), samples(variable).copy(timeline = time)) + } + @Test fun linearTaggedSdrIsRejectedBeforeAllocatingOrStartingExport() { val format = MediaFormat.createVideoFormat("video/avc", 1920, 1080).apply { setInteger(MediaFormat.KEY_COLOR_TRANSFER, MediaFormat.COLOR_TRANSFER_LINEAR) @@ -143,11 +180,11 @@ class SdrConversionAdapterTests { validate = { _, _, _, _, _ -> duringValidation?.invoke(); validation }, createEngine = { _, _, _, _, _ -> engine }) val attempt = adapter.start(sourceFile, inputMetadata, samples(), conversion, results::add) fun idle() { - if (results.isNotEmpty()) return + if (results.isNotEmpty() || !attempt.looper.thread.isAlive) return try { shadowOf(attempt.looper).idle() } catch (error: IllegalStateException) { // PAUSED-mode idle posts a barrier; terminal cleanup can quit before that post. - if (results.isEmpty() || error.message?.contains("handler thread dead") != true) throw error + if (results.isEmpty() || attempt.looper.thread.isAlive) throw error } } fun assertInputSafe() { assertEquals("original", sourceFile.readText()); assertEquals("customer", unrelated.readText()) } diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/SdrMedia3ConfigurationTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/SdrMedia3ConfigurationTests.kt index 055d5caf..041f1d90 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/SdrMedia3ConfigurationTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/SdrMedia3ConfigurationTests.kt @@ -22,17 +22,20 @@ import androidx.test.core.app.ApplicationProvider import io.mockk.* import org.junit.Assert.* import org.junit.Test +import org.junit.After import org.junit.runner.RunWith import org.robolectric.RobolectricTestRunner import org.robolectric.annotation.Config import org.robolectric.annotation.Implementation import org.robolectric.annotation.Implements import org.robolectric.shadows.MediaCodecInfoBuilder +import org.robolectric.shadows.ShadowBuild import java.lang.reflect.InvocationTargetException @RunWith(RobolectricTestRunner::class) @Config(manifest = Config.NONE, sdk = [28], shadows = [SupportedFormatShadow::class]) class SdrMedia3ConfigurationTests { + @After fun resetAdvertisedFormats() { SupportedFormatShadow.supported = { true } } private val context = ApplicationProvider.getApplicationContext() private val facts = compliantFacts().copy(cadence = known(Cadence.Constant)) private val conversion = (StandardInputPlanner().plan(facts.copy(averageBitrate = known(9_000_000L)), @@ -71,14 +74,27 @@ class SdrMedia3ConfigurationTests { .getInteger(MediaFormat.KEY_OPERATING_RATE)) } - @Test fun redmiFrameRateWorkaroundDoesNotIncreaseLowRateGopOrResampleTimestamps() { - val high = format.buildUpon().setFrameRate(120f).build() - assertEquals(30f, SdrVideoEncoderConfiguration.create(high, targets, 4, sdk = 29, device = "joyeuse") - .getFloat(MediaFormat.KEY_FRAME_RATE), 0f) - val low = format.buildUpon().setFrameRate(5f).build() - assertEquals(5f, SdrVideoEncoderConfiguration.create(low, targets, 4, sdk = 29, device = "joyeuse") - .getFloat(MediaFormat.KEY_FRAME_RATE), 0f) - assertEquals(120f, high.frameRate, 0f) + @Test fun redmiRejectsUnprovenRateHintsInsteadOfChangingBitrateOrGop() { + for (rate in listOf(5f, 24f, 60f, 120f)) { + val requested = format.buildUpon().setFrameRate(rate).build() + assertThrows(IllegalStateException::class.java) { + SdrVideoEncoderConfiguration.create(requested, targets, 4, sdk = 29, device = "joyeuse") + } + assertEquals(rate, SdrVideoEncoderConfiguration.create(requested, targets, 4, sdk = 30, device = "joyeuse") + .getFloat(MediaFormat.KEY_FRAME_RATE), 0f) + } + val accepted = SdrVideoEncoderConfiguration.create(format, targets, 4, sdk = 29, device = "joyeuse") + assertEquals(30f, accepted.getFloat(MediaFormat.KEY_FRAME_RATE), 0f) + assertEquals(targets.bitrate, accepted.getInteger(MediaFormat.KEY_BIT_RATE)) + assertEquals(targets.keyframeIntervalSeconds, accepted.getInteger(MediaFormat.KEY_I_FRAME_INTERVAL)) + } + + @Test fun redmiPreflightRejectsNonThirtyFpsBeforeEncoding() { + ShadowBuild.setDevice("joyeuse") + val infos = arrayOf(codec(false, "video/avc"), codec(true, "video/avc")) + assertNull(SdrCapabilityPreflight({ infos }, { 0L }) + .videoEncoder(conversion.copy(outputFrameRate = 60.0), metadata, targets) { false }) + assertNotNull(SdrCapabilityPreflight({ infos }, { 0L }).videoEncoder(conversion, metadata, targets) { false }) } private fun codec(encoder: Boolean, mime: String): MediaCodecInfo { @@ -95,7 +111,7 @@ class SdrMedia3ConfigurationTests { .setIsEncoder(encoder).setCapabilities(capabilities).build() } - @Test fun coldEnumerationHasSeparateBoundAndUnrelatedCodecsDoNotExhaustQueries() { + @Test fun coldEnumerationDoesNotConsumeQueryBudgetAndUnrelatedCodecsDoNotExhaustQueries() { var now = 0L val unrelated = codec(false, "audio/mpeg") val encoder = codec(true, "video/avc") @@ -111,21 +127,27 @@ class SdrMedia3ConfigurationTests { assertEquals(1, queried.last().getInteger(MediaFormat.KEY_PRIORITY)) } - @Test fun preflightStillBoundsColdInitializationQueriesAndCancellation() { + @Test fun preflightAcceptsSlowEnumerationButBoundsQueriesAndCancellation() { var now = 0L val encoder = codec(true, "video/avc") val slowDecoder = codec(false, "video/avc") var queries = 0 - SupportedFormatShadow.supported = { queries++; now += 500_000_000L; true } val infos = arrayOf(slowDecoder, encoder) - assertNull(SdrCapabilityPreflight({ now += 2_000_000_000L; infos }, { now }) + SupportedFormatShadow.supported = { queries++; true } + assertNotNull(SdrCapabilityPreflight({ now += 3_000_000_000L; infos }, { now }) .videoEncoder(conversion, metadata, targets) { false }) + assertEquals(2, queries) + queries = 0 + SupportedFormatShadow.supported = { queries++; now += 500_000_000L; true } now = 0 assertNull(SdrCapabilityPreflight({ infos }, { now }).videoEncoder(conversion, metadata, targets) { false }) assertEquals(1, queries) var enumerated = false assertNull(SdrCapabilityPreflight({ enumerated = true; infos }, { 0L }).videoEncoder(conversion, metadata, targets) { true }) assertFalse(enumerated) + var cancelled = false + assertNull(SdrCapabilityPreflight({ cancelled = true; infos }, { 0L }) + .videoEncoder(conversion, metadata, targets) { cancelled }) val rejected = codec(false, "video/avc") queries = 0 SupportedFormatShadow.supported = { queries++; false } From 0ced0309784ff6ddfb0baa6328c4d26713345038 Mon Sep 17 00:00:00 2001 From: Eugene Morozov Date: Thu, 8 Oct 2026 15:25:42 -0700 Subject: [PATCH 34/49] NAT-548: Return cancellation failure to active await callers --- .../main/java/com/mux/video/upload/api/MuxUpload.kt | 13 ++++++++----- .../internal/UploadPreparationLifecycleTests.kt | 13 +++++++++++++ 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/library/src/main/java/com/mux/video/upload/api/MuxUpload.kt b/library/src/main/java/com/mux/video/upload/api/MuxUpload.kt index ef8b26c2..42462dc6 100644 --- a/library/src/main/java/com/mux/video/upload/api/MuxUpload.kt +++ b/library/src/main/java/com/mux/video/upload/api/MuxUpload.kt @@ -175,9 +175,10 @@ class MuxUpload private constructor( * completes * * If the upload already succeeded, the old result will be returned immediately. - * After [cancel], this returns a failure without restarting. Cancellation of the calling - * coroutine still throws [CancellationException]. Replacement by a new destination returns - * a failure to existing waiters and ends this handle. + * [cancel] returns a failure to existing waiters and later calls without restarting. + * [pause] interrupts existing waiters with [CancellationException]; the upload remains resumable. + * Cancellation of the calling coroutine still throws [CancellationException]. Replacement by + * a new destination returns a failure to existing waiters and ends this handle. */ @Throws @Suppress("unused") @@ -195,9 +196,11 @@ class MuxUpload private constructor( try { awaited.uploadJob?.await() ?: Result.failure(Exception("Upload failed to start")) } catch (e: CancellationException) { - // Replacing the upload ends its job, but must not cancel an active caller. + // Upload cancellation/replacement must not cancel an active caller. currentCoroutineContext().ensureActive() - replacementResult(awaited) ?: throw e + replacementResult(awaited) ?: if (cancelled || awaited.attempt?.isCancelled() == true) { + Result.failure(UploadCancelledException()) + } else throw e } } } diff --git a/library/src/test/java/com/mux/video/upload/internal/UploadPreparationLifecycleTests.kt b/library/src/test/java/com/mux/video/upload/internal/UploadPreparationLifecycleTests.kt index e93bff0f..5decc209 100644 --- a/library/src/test/java/com/mux/video/upload/internal/UploadPreparationLifecycleTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/UploadPreparationLifecycleTests.kt @@ -394,7 +394,20 @@ class UploadPreparationLifecycleTests : AbsRobolectricTest() { val upload = MuxUpload.create(info()) upload.start(); pump { chunks.isNotEmpty() } val before = internalInfo(upload).uploadJob + var waitingResult: Result? = null + val waitingCaller = scope.launch { waitingResult = upload.awaitSuccess() } + var cancelledCallerReturned = false + val cancelledCaller = scope.launch { upload.awaitSuccess(); cancelledCallerReturned = true } + dispatcher.scheduler.runCurrent() + assertTrue(waitingCaller.isActive); assertTrue(cancelledCaller.isActive) + cancelledCaller.cancel() upload.cancel(); upload.start(forceRestart = true) + pump { waitingCaller.isCompleted && cancelledCaller.isCompleted } + assertTrue(cancelledCaller.isCancelled) + assertFalse(cancelledCallerReturned) + assertFalse(waitingCaller.isCancelled) + assertTrue(waitingResult!!.exceptionOrNull() is UploadCancelledException) + assertTrue(results.isEmpty()) assertSame(before, internalInfo(upload).uploadJob) var result: Result? = null val caller = scope.launch { result = upload.awaitSuccess() } From 0f87fed728fac6e89a9b95db28511dfea10b4df2 Mon Sep 17 00:00:00 2001 From: Eugene Morozov Date: Wed, 7 Oct 2026 15:39:36 -0700 Subject: [PATCH 35/49] Synchronize SDR test completion and share AAC layout mapping --- .../standardization/SdrConversionAdapter.kt | 7 +------ .../SdrConversionAdapterTests.kt | 20 +++++++++++++------ 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt index 3c365baa..879bea73 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt @@ -107,12 +107,7 @@ internal class SdrConversionAdapter( // The shared validator checks policy, pixels, cadence, and effective timeline. // Add this adapter's concrete encoder-profile and non-AAC channel contract. val audio = validation.facts.audioTracks.valueOrNull?.singleOrNull()?.format?.valueOrNull as? AudioFormat.Aac - val expectedLayout = when (targets.audioChannels) { - 1 -> AudioChannelLayout.Mono - 2 -> AudioChannelLayout.Stereo - 6 -> AudioChannelLayout.FivePointOne - else -> null - } + val expectedLayout = CodecMetadataReader.aacChannelLayout(targets.audioChannels) if (validation.facts.videoProfile != MediaFact.Known(targets.profile) || (targets.audioChannels != null && audio?.layout != expectedLayout)) fail(SdrConversionFailure.OutputInvalid) else finish(SdrConversionResult.Completed(owned, validation.facts)) diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/SdrConversionAdapterTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/SdrConversionAdapterTests.kt index d47c554d..2dfb4138 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/SdrConversionAdapterTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/SdrConversionAdapterTests.kt @@ -16,6 +16,7 @@ import org.robolectric.Shadows.shadowOf import org.robolectric.annotation.Config import org.robolectric.annotation.LooperMode import java.io.File +import java.util.concurrent.CopyOnWriteArrayList @RunWith(RobolectricTestRunner::class) @Config(manifest = Config.NONE, sdk = [28]) @@ -175,16 +176,23 @@ class SdrConversionAdapterTests { val sourceFile = File.createTempFile("customer-", ".mp4", context.cacheDir).apply { writeText("original") } val unrelated = File(context.cacheDir, "mux-upload/customer-copy.mp4").apply { parentFile!!.mkdirs(); writeText("customer") } val engine = FakeEngine() - val results = mutableListOf() + val results = CopyOnWriteArrayList() val adapter = SdrConversionAdapter(context, preflight = { _, _, _, _ -> capability }, validate = { _, _, _, _, _ -> duringValidation?.invoke(); validation }, createEngine = { _, _, _, _, _ -> engine }) val attempt = adapter.start(sourceFile, inputMetadata, samples(), conversion, results::add) fun idle() { - if (results.isNotEmpty() || !attempt.looper.thread.isAlive) return - try { shadowOf(attempt.looper).idle() } - catch (error: IllegalStateException) { - // PAUSED-mode idle posts a barrier; terminal cleanup can quit before that post. - if (results.isEmpty() || attempt.looper.thread.isAlive) throw error + val thread = attempt.looper.thread + if (results.isEmpty() && thread.isAlive) { + try { shadowOf(attempt.looper).idle() } + catch (error: IllegalStateException) { + // PAUSED-mode idle can race a terminal quit before its barrier is posted. + thread.join(5_000) + if (results.isEmpty() || thread.isAlive) throw error + } + } + if (results.isNotEmpty()) { + thread.join(5_000) + assertFalse("Terminal attempt thread still running", thread.isAlive) } } fun assertInputSafe() { assertEquals("original", sourceFile.readText()); assertEquals("customer", unrelated.readText()) } From 803cdb943a662be792b3deec39a2354f8f4e369d Mon Sep 17 00:00:00 2001 From: Eugene Morozov Date: Thu, 8 Oct 2026 15:47:22 -0700 Subject: [PATCH 36/49] NAT-548: Keep await callers attached across force restart --- .../com/mux/video/upload/api/MuxUpload.kt | 40 +++++++++++++------ .../mux/video/upload/api/MuxUploadManager.kt | 6 +-- .../video/upload/internal/UploadAttempt.kt | 5 ++- .../UploadPreparationLifecycleTests.kt | 29 ++++++++++++-- 4 files changed, 60 insertions(+), 20 deletions(-) diff --git a/library/src/main/java/com/mux/video/upload/api/MuxUpload.kt b/library/src/main/java/com/mux/video/upload/api/MuxUpload.kt index 42462dc6..a2a7ab2a 100644 --- a/library/src/main/java/com/mux/video/upload/api/MuxUpload.kt +++ b/library/src/main/java/com/mux/video/upload/api/MuxUpload.kt @@ -13,6 +13,7 @@ import com.mux.video.upload.internal.forgetUploadState import com.mux.video.upload.internal.readUploadResumeState import com.mux.video.upload.internal.UploadCancelledException import kotlinx.coroutines.flow.filterNotNull +import kotlinx.coroutines.flow.first import kotlinx.coroutines.flow.flatMapLatest import kotlinx.coroutines.flow.flowOf import kotlinx.coroutines.flow.map @@ -155,7 +156,7 @@ class MuxUpload private constructor( readUploadResumeState(uploadInfo).generatedResumeBlocked val previous = uploadInfo if (previous.attempt == null) forgetUploadState(previous) - else previous.attempt.cancel { forgetUploadState(previous) } + else previous.attempt.cancel(forRestart = true) { forgetUploadState(previous) } previous.uploadJob?.cancel() uploadInfo = uploadInfo.update(attempt = null, statusFlow = null, restoredFromOriginal = false, generatedResumeBlocked = blocked) @@ -175,6 +176,7 @@ class MuxUpload private constructor( * completes * * If the upload already succeeded, the old result will be returned immediately. + * If force restart interrupts the attempt, existing waiters follow the restarted upload. * [cancel] returns a failure to existing waiters and later calls without restarting. * [pause] interrupts existing waiters with [CancellationException]; the upload remains resumable. * Cancellation of the calling coroutine still throws [CancellationException]. Replacement by @@ -191,17 +193,31 @@ class MuxUpload private constructor( } else { coroutineScope { startInner(coroutineScope = this) - val awaited = uploadInfo - replacementResult(awaited)?.let { return@coroutineScope it } - try { - awaited.uploadJob?.await() ?: Result.failure(Exception("Upload failed to start")) - } catch (e: CancellationException) { - // Upload cancellation/replacement must not cancel an active caller. - currentCoroutineContext().ensureActive() - replacementResult(awaited) ?: if (cancelled || awaited.attempt?.isCancelled() == true) { - Result.failure(UploadCancelledException()) - } else throw e - } + awaitUploadResult(uploadInfo) + } + } + } + + private suspend fun awaitUploadResult(initial: UploadInfo): Result { + var awaited = initial + while (true) { + replacementResult(awaited)?.let { return it } + try { + return awaited.uploadJob?.await() ?: Result.failure(Exception("Upload failed to start")) + } catch (e: CancellationException) { + // Upload lifecycle changes must not cancel an active caller. + currentCoroutineContext().ensureActive() + replacementResult(awaited)?.let { return it } + val previous = awaited.attempt + if (!cancelled && previous?.isRestarting() == true) { + // Restart marks the old attempt before cancellation. Publication of the new job + // can race this waiter, so wait for its identity in the same destination session. + awaited = awaited.session.current.filterNotNull().first { + it.attempt != null && it.attempt !== previous && it.uploadJob != null + } + } else if (cancelled || previous?.isCancelled() == true) { + return Result.failure(UploadCancelledException()) + } else throw e } } } diff --git a/library/src/main/java/com/mux/video/upload/api/MuxUploadManager.kt b/library/src/main/java/com/mux/video/upload/api/MuxUploadManager.kt index d683aa3d..09a62667 100644 --- a/library/src/main/java/com/mux/video/upload/api/MuxUploadManager.kt +++ b/library/src/main/java/com/mux/video/upload/api/MuxUploadManager.kt @@ -169,9 +169,9 @@ object MuxUploadManager { upload.uploadJob?.cancel() } - private fun cancelAttempt(upload: UploadInfo, forgetUnstarted: Boolean = false) { + private fun cancelAttempt(upload: UploadInfo, forgetUnstarted: Boolean = false, forRestart: Boolean = false) { val attempt = upload.attempt - if (attempt != null) attempt.cancel { forgetUploadState(upload) } + if (attempt != null) attempt.cancel(forRestart = forRestart) { forgetUploadState(upload) } else if (forgetUnstarted) forgetUploadState(upload) upload.uploadJob?.cancel() } @@ -189,7 +189,7 @@ object MuxUploadManager { if (newDestination) { source.attempt?.replace { forgetUploadState(source) } source.uploadJob?.cancel() - } else cancelAttempt(source, forgetUnstarted = true) + } else cancelAttempt(source, forgetUnstarted = true, forRestart = true) source = upload.update(attempt = null, uploadJob = source.uploadJob, statusFlow = null, restoredFromOriginal = false, generatedResumeBlocked = generatedMayExistRemotely).also { it.session = if (newDestination) upload.session else source.session diff --git a/library/src/main/java/com/mux/video/upload/internal/UploadAttempt.kt b/library/src/main/java/com/mux/video/upload/internal/UploadAttempt.kt index 99555091..6f1d087d 100644 --- a/library/src/main/java/com/mux/video/upload/internal/UploadAttempt.kt +++ b/library/src/main/java/com/mux/video/upload/internal/UploadAttempt.kt @@ -40,6 +40,7 @@ internal class UploadAttempt( val status = MutableStateFlow(UploadStatus.Started) private var stopped = false private var cancelled = false + private var restarting = false private var terminal = false private var superseded = false private var confirmed = initialProgress @@ -48,6 +49,7 @@ internal class UploadAttempt( @Synchronized fun isStopped() = stopped @Synchronized fun isCancelled() = cancelled + @Synchronized fun isRestarting() = restarting @Synchronized fun replacementFailure() = replacement @Synchronized fun confirmedProgress() = confirmed @Synchronized fun supersede() { superseded = true } @@ -98,11 +100,12 @@ internal class UploadAttempt( persist(confirmed) } - fun cancel(forget: () -> Unit) { + fun cancel(forRestart: Boolean = false, forget: () -> Unit) { synchronized(this) { if (cancelled) return stopped = true cancelled = true + restarting = forRestart forget() } preparation.deleteOwnedFile() diff --git a/library/src/test/java/com/mux/video/upload/internal/UploadPreparationLifecycleTests.kt b/library/src/test/java/com/mux/video/upload/internal/UploadPreparationLifecycleTests.kt index 5decc209..b888e36a 100644 --- a/library/src/test/java/com/mux/video/upload/internal/UploadPreparationLifecycleTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/UploadPreparationLifecycleTests.kt @@ -30,6 +30,7 @@ class UploadPreparationLifecycleTests : AbsRobolectricTest() { private lateinit var source: File private lateinit var sibling: File private var preparations = 0 + private var beforeCreateJob: () -> Unit = {} private var prepare: suspend (UploadInfo) -> PreparedUpload = { PreparedUpload.Original() } private var work: suspend (ChunkWorker.Chunk, UploadInfo, MutableSharedFlow) -> MuxUpload.Progress = { chunk, _, _ -> MuxUpload.Progress(bytesUploaded = chunk.contentLength.toLong()) } @@ -52,7 +53,7 @@ class UploadPreparationLifecycleTests : AbsRobolectricTest() { mockk { coEvery { upload() } coAnswers { work(chunk, info, flow) } } }) val factory = mockk() - every { factory.createUploadJob(any(), any(), any()) } answers { real.createUploadJob(firstArg(), scope, thirdArg()) } + every { factory.createUploadJob(any(), any(), any()) } answers { beforeCreateJob(); real.createUploadJob(firstArg(), scope, thirdArg()) } mockkObject(MuxUploadSdk) every { MuxUploadSdk.uploadJobFactory() } returns factory mockkObject(UploadMetrics.Companion) @@ -333,17 +334,27 @@ class UploadPreparationLifecycleTests : AbsRobolectricTest() { } @Test fun unmanagedForceRestartStopsPreviousWorkerBeforeStartingAnother() { - work = { _, _, _ -> awaitCancellation() } + val release = CompletableDeferred() + work = { chunk, _, _ -> release.await(); MuxUpload.Progress(bytesUploaded = chunk.contentLength.toLong()) } val upload = MuxUpload.Builder("https://example.invalid/upload", source) .manageUploadTask(false).standardizationRequested(false).build() upload.start() pump { chunks.size == 1 } val old = internalInfo(upload).uploadJob!! + var result: Result? = null + val waiter = scope.launch { result = upload.awaitSuccess() } + dispatcher.scheduler.runCurrent() + // Run the old waiter after teardown but before the new job is published. + beforeCreateJob = { dispatcher.scheduler.runCurrent(); assertTrue(waiter.isActive); assertNull(result) } upload.start(forceRestart = true) pump { chunks.size == 2 } assertTrue(old.isCancelled) assertEquals(0L, chunks.last().startByte) - upload.cancel() + assertTrue(waiter.isActive); assertNull(result) + release.complete(Unit) + pump { waiter.isCompleted && upload.isSuccessful } + assertFalse(waiter.isCancelled) + assertTrue(result!!.getOrThrow() is UploadStatus.UploadSuccess) } @Test fun newDirectUploadUsesOriginalFallbackWithoutOldOffsetOrGeneratedPayload() { @@ -542,15 +553,25 @@ class UploadPreparationLifecycleTests : AbsRobolectricTest() { } @Test fun forceRestartWithoutManagerEntryCancelsItsPreviousJob() { - work = { _, _, _ -> awaitCancellation() } + val release = CompletableDeferred() + work = { chunk, _, _ -> release.await(); MuxUpload.Progress(bytesUploaded = chunk.contentLength.toLong()) } val unmanaged = MuxUpload.Builder(info().remoteUri, source).manageUploadTask(false).build() unmanaged.start(); pump { chunks.size == 1 } val previous = internalInfo(unmanaged).uploadJob!! + var result: Result? = null + val waiter = scope.launch { result = unmanaged.awaitSuccess() } + dispatcher.scheduler.runCurrent() val managed = MuxUpload.create(internalInfo(unmanaged)) + beforeCreateJob = { dispatcher.scheduler.runCurrent(); assertTrue(waiter.isActive); assertNull(result) } managed.start(forceRestart = true) pump { chunks.size == 2 } assertTrue(previous.isCancelled) assertTrue(managed.isRunning) + assertTrue(waiter.isActive); assertNull(result) + release.complete(Unit) + pump { waiter.isCompleted && managed.isSuccessful } + assertFalse(waiter.isCancelled) + assertTrue(result!!.getOrThrow() is UploadStatus.UploadSuccess) } @Test fun failedHandleFollowsManagerRetryAndReceivesItsNewResult() { From 10889a34795c51a35bc87d32dff9470734614bb0 Mon Sep 17 00:00:00 2001 From: Eugene Morozov Date: Fri, 16 Oct 2026 11:29:41 -0700 Subject: [PATCH 37/49] Use normalized platform metadata in SDR conversion --- .../upload/internal/standardization/SdrConversionAdapter.kt | 2 +- .../upload/internal/standardization/SdrEncodingTargets.kt | 3 +-- .../internal/standardization/SdrConversionAdapterTests.kt | 2 +- .../internal/standardization/SdrMedia3ConfigurationTests.kt | 2 +- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt index 879bea73..aac325a9 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt @@ -107,7 +107,7 @@ internal class SdrConversionAdapter( // The shared validator checks policy, pixels, cadence, and effective timeline. // Add this adapter's concrete encoder-profile and non-AAC channel contract. val audio = validation.facts.audioTracks.valueOrNull?.singleOrNull()?.format?.valueOrNull as? AudioFormat.Aac - val expectedLayout = CodecMetadataReader.aacChannelLayout(targets.audioChannels) + val expectedLayout = aacChannelLayout(targets.audioChannels).valueOrNull if (validation.facts.videoProfile != MediaFact.Known(targets.profile) || (targets.audioChannels != null && audio?.layout != expectedLayout)) fail(SdrConversionFailure.OutputInvalid) else finish(SdrConversionResult.Completed(owned, validation.facts)) diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/SdrEncodingTargets.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/SdrEncodingTargets.kt index 337a5f40..4ba96c57 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/SdrEncodingTargets.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/SdrEncodingTargets.kt @@ -42,8 +42,7 @@ internal data class SdrEncodingTargets( val facts = source.facts // LINEAR is SDR, but this surface path has no proof of its transfer handling. if (metadata.tracks.any { it.video?.let { video -> - video.platformColor.transfer.valueOrNull == MediaFormat.COLOR_TRANSFER_LINEAR || - video.configuration.color.transfer.valueOrNull == MediaFormat.COLOR_TRANSFER_LINEAR + video.color.transfer.valueOrNull == MediaFormat.COLOR_TRANSFER_LINEAR } == true }) return null if (source.status != SampleScanStatus.Complete || conversion.toneMapsToSdr || conversion.sourceDynamicRange != DynamicRange.Sdr || facts.dynamicRange != MediaFact.Known(DynamicRange.Sdr) || diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/SdrConversionAdapterTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/SdrConversionAdapterTests.kt index 2dfb4138..0c3b350a 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/SdrConversionAdapterTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/SdrConversionAdapterTests.kt @@ -33,7 +33,7 @@ class SdrConversionAdapterTests { known(if (format.valueOrNull is AudioFormat.Aac) "audio/mp4a-latm" else "audio/mpeg"), known(3.0), MediaFact.Unknown, MediaFact.Unknown, audio = AudioTrackMetadata(format, known(channels), known(48_000), known(0), known(0))) private fun metadata(tracks: List = listOf(audio())) = - MediaMetadataInspection(known(ContainerKind.IsoBaseMedia), tracks, MediaFact.Unknown, facts, 0) + MediaMetadataInspection(known(ContainerKind.IsoBaseMedia), tracks, facts, 0) private fun samples(input: MediaFacts = facts) = MediaSampleInspection(input, SampleScanStatus.Complete, timeline = timeline) private fun targets(plan: StandardInputConversion = conversion, input: MediaFacts = facts, tracks: List = listOf(audio())) = SdrEncodingTargets.from(plan, metadata(tracks), samples(input)) diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/SdrMedia3ConfigurationTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/SdrMedia3ConfigurationTests.kt index 041f1d90..53761dc8 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/SdrMedia3ConfigurationTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/SdrMedia3ConfigurationTests.kt @@ -46,7 +46,7 @@ class SdrMedia3ConfigurationTests { .setHeight(conversion.outputDimensions.height).setFrameRate(30f).setAverageBitrate(targets.bitrate) .setColorInfo(ColorInfo.SDR_BT709_LIMITED).build() private val video = MediaTrackMetadataReader.read(0, MediaFormat.createVideoFormat("video/avc", 1920, 1080)) - private val metadata = MediaMetadataInspection(known(ContainerKind.IsoBaseMedia), listOf(video), MediaFact.Unknown, facts, 0) + private val metadata = MediaMetadataInspection(known(ContainerKind.IsoBaseMedia), listOf(video), facts, 0) @Test fun strictFactoryRejectsMimeDimensionsRotationAndLinearBeforeCodecCreation() { val factory = StrictSdrEncoderFactory(context, conversion, targets, SdrEncoderCapability("chosen", 4)) From 0097f2a466351b94f6a1bee8417db2bf432c6a2d Mon Sep 17 00:00:00 2001 From: Eugene Morozov Date: Fri, 16 Oct 2026 11:30:45 -0700 Subject: [PATCH 38/49] Update upload preparation fixtures for platform metadata --- .../video/upload/internal/UploadPreparationLifecycleTests.kt | 2 +- .../upload/internal/standardization/UploadPreparationTests.kt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/library/src/test/java/com/mux/video/upload/internal/UploadPreparationLifecycleTests.kt b/library/src/test/java/com/mux/video/upload/internal/UploadPreparationLifecycleTests.kt index b888e36a..b744ba9d 100644 --- a/library/src/test/java/com/mux/video/upload/internal/UploadPreparationLifecycleTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/UploadPreparationLifecycleTests.kt @@ -92,7 +92,7 @@ class UploadPreparationLifecycleTests : AbsRobolectricTest() { val facts = compliantFacts().copy(averageBitrate = known(9_000_000L), audioTracks = known(listOf( AudioTrack(known(AudioFormat.Aac(AudioChannelLayout.Stereo))), AudioTrack(known(AudioFormat.Aac(AudioChannelLayout.Mono)))))) - val metadata = MediaMetadataInspection(known(ContainerKind.IsoBaseMedia), emptyList(), MediaFact.Unknown, facts, 0) + val metadata = MediaMetadataInspection(known(ContainerKind.IsoBaseMedia), emptyList(), facts, 0) val preparation = UploadPreparation(inspectMetadata = { MetadataInspectionResult.Success(metadata) }, inspectSamples = { _, _, _ -> MediaSampleInspection(facts, SampleScanStatus.Complete) }, convert = { _, _, _, _, _ -> error("Multi-audio export must never select a replacement track") }) diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/UploadPreparationTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/UploadPreparationTests.kt index b41c8cd6..9f25b712 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/UploadPreparationTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/UploadPreparationTests.kt @@ -24,7 +24,7 @@ class UploadPreparationTests : AbsRobolectricTest() { private fun source(codec: VideoCodec = VideoCodec.H264) = compliantFacts(codec).copy( audioTracks = known(emptyList()), cadence = known(Cadence.Constant)) private fun metadata(facts: MediaFacts) = MediaMetadataInspection(known(ContainerKind.IsoBaseMedia), - emptyList(), MediaFact.Unknown, facts, 0) + emptyList(), facts, 0) private fun upload() = UploadInfo(remoteUri = Uri.parse("https://example.invalid/upload"), inputFile = File("unused"), chunkSize = 4, retriesPerChunk = 3, optOut = true, uploadJob = null, statusFlow = null) From 1cc43867a0f14814d71ea974e99c52b08dc79f18 Mon Sep 17 00:00:00 2001 From: Eugene Morozov Date: Fri, 16 Oct 2026 12:11:27 -0700 Subject: [PATCH 39/49] Use coroutines and a cancellable continuation for SDR conversion --- .../standardization/SdrConversionAdapter.kt | 154 ++++++++++-------- .../SdrConversionAdapterTests.kt | 61 ++++--- 2 files changed, 129 insertions(+), 86 deletions(-) diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt index aac325a9..8512eb1f 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt @@ -1,12 +1,25 @@ package com.mux.video.upload.internal.standardization import android.content.Context -import android.os.Handler -import android.os.HandlerThread import android.os.Looper import androidx.media3.common.MimeTypes +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CoroutineDispatcher +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.CoroutineStart +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.DelicateCoroutinesApi +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.Job +import kotlinx.coroutines.NonCancellable +import kotlinx.coroutines.cancel +import kotlinx.coroutines.ensureActive +import kotlinx.coroutines.launch +import kotlinx.coroutines.suspendCancellableCoroutine +import kotlinx.coroutines.withContext import java.io.File import java.util.concurrent.atomic.AtomicBoolean +import kotlin.coroutines.resume internal sealed interface SdrConversionResult { /** Caller owns retention until upload/resume reaches its terminal point. */ @@ -35,7 +48,7 @@ internal interface SdrExportEngine { fun cancel() } -/** Each attempt owns one thread, one Transformer, and one explicitly tracked output file. */ +/** Each attempt owns a coroutine job, one Transformer, and one explicitly tracked output file. */ internal class SdrConversionAdapter( context: Context, private val preflight: (StandardInputConversion, MediaMetadataInspection, SdrEncodingTargets, () -> Boolean) -> SdrEncoderCapability? = @@ -44,6 +57,7 @@ internal class SdrConversionAdapter( StandardInputOutputValidator()::validateGeneratedOutput, private val createEngine: (Context, Looper, StandardInputConversion, SdrEncodingTargets, SdrEncoderCapability) -> SdrExportEngine = ::Media3SdrExportEngine, + private val ioDispatcher: CoroutineDispatcher = Dispatchers.IO, ) { private val appContext = context.applicationContext @@ -56,10 +70,8 @@ internal class SdrConversionAdapter( private val source: MediaSampleInspection, private val conversion: StandardInputConversion, private val onResult: (SdrConversionResult) -> Unit, ) { - private val thread = HandlerThread("MuxSdrConversion").apply { start() } - internal val looper: Looper = thread.looper - private val handler = Handler(looper) - private val cancelled = AtomicBoolean(false) + private val job = Job() + private val scope = CoroutineScope(job + Dispatchers.Main) private val terminalLock = Any() private var terminal = false private var engine: SdrExportEngine? = null @@ -73,85 +85,93 @@ internal class SdrConversionAdapter( fun cancel() { synchronized(terminalLock) { if (terminal) return - cancelled.set(true) + job.cancel() } - handler.post { finish(SdrConversionResult.Cancelled) } } - internal fun begin() { handler.post { - guarded { - if (cancelled.get()) return@guarded finish(SdrConversionResult.Cancelled) - val targets = SdrEncodingTargets.from(conversion, metadata, source) - ?: return@guarded fail(SdrConversionFailure.UnsupportedPlan) - val capability = preflight(conversion, metadata, targets, cancelled::get) - ?: return@guarded fail(SdrConversionFailure.CapabilityUnavailable) - if (cancelled.get()) return@guarded finish(SdrConversionResult.Cancelled) - if (!sourceUnchanged()) return@guarded fail(SdrConversionFailure.SourceChanged) + @OptIn(DelicateCoroutinesApi::class, ExperimentalCoroutinesApi::class) + internal fun begin() { + // Enter cleanup even when cancellation wins before the initial Main dispatch. + scope.launch(start = CoroutineStart.ATOMIC) { + val result = try { + ensureActive() + convert() + } catch (_: CancellationException) { SdrConversionResult.Cancelled } + catch (_: Exception) { SdrConversionResult.Failed(SdrConversionFailure.Export) } + catch (_: LinkageError) { SdrConversionResult.Failed(SdrConversionFailure.Export) } + withContext(NonCancellable) { finish(result) } + } + } + + private suspend fun convert(): SdrConversionResult { + val targets = withContext(ioDispatcher) { SdrEncodingTargets.from(conversion, metadata, source) } + ?: return SdrConversionResult.Failed(SdrConversionFailure.UnsupportedPlan) + val capability = withContext(ioDispatcher) { preflight(conversion, metadata, targets) { !job.isActive } } + ?: return SdrConversionResult.Failed(SdrConversionFailure.CapabilityUnavailable) + val preparationFailure = withContext(ioDispatcher) { + if (!sourceUnchanged()) return@withContext SdrConversionFailure.SourceChanged val required = source.timeline.durationSeconds.valueOrNull?.let(targets::estimatedOutputBytes) - ?: return@guarded fail(SdrConversionFailure.InsufficientSourceEvidence) - val owned = SdrGeneratedFile.allocate(appContext.cacheDir, required) - ?: return@guarded fail(SdrConversionFailure.DiskSpace) - output = owned - if (cancelled.get()) return@guarded finish(SdrConversionResult.Cancelled) - val exporter = createEngine(appContext, looper, conversion, targets, capability) - engine = exporter + ?: return@withContext SdrConversionFailure.InsufficientSourceEvidence + // Record ownership before returning across a cancellable dispatcher boundary. + output = SdrGeneratedFile.allocate(appContext.cacheDir, required) + if (output == null) SdrConversionFailure.DiskSpace else null + } + if (preparationFailure != null) return SdrConversionResult.Failed(preparationFailure) + val owned = checkNotNull(output) + val exporter = createEngine(appContext, Looper.getMainLooper(), conversion, targets, capability) + engine = exporter + val exportFailure = suspendCancellableCoroutine { continuation -> + val delivered = AtomicBoolean(false) + continuation.invokeOnCancellation { delivered.set(true) } + fun complete(reason: SdrConversionFailure?) { + if (delivered.compareAndSet(false, true)) continuation.resume(reason) + } exporter.start(input, owned.file, { videoMime, audioMime -> - handler.post { guarded { - if (isTerminal()) return@guarded - if (videoMime != targets.videoMime || audioMime != if (targets.audioChannels != null) MimeTypes.AUDIO_AAC else null) - return@guarded fail(SdrConversionFailure.CodecChanged) - if (!sourceUnchanged()) return@guarded fail(SdrConversionFailure.SourceChanged) - when (val validation = validate(owned.file, source.facts, source.timeline, conversion, cancelled::get)) { - is StandardInputOutputValidation.Accepted -> { - if (!sourceUnchanged()) return@guarded fail(SdrConversionFailure.SourceChanged) - // The shared validator checks policy, pixels, cadence, and effective timeline. - // Add this adapter's concrete encoder-profile and non-AAC channel contract. - val audio = validation.facts.audioTracks.valueOrNull?.singleOrNull()?.format?.valueOrNull as? AudioFormat.Aac - val expectedLayout = aacChannelLayout(targets.audioChannels).valueOrNull - if (validation.facts.videoProfile != MediaFact.Known(targets.profile) || - (targets.audioChannels != null && audio?.layout != expectedLayout)) fail(SdrConversionFailure.OutputInvalid) - else finish(SdrConversionResult.Completed(owned, validation.facts)) - } - is StandardInputOutputValidation.Rejected -> fail(SdrConversionFailure.OutputInvalid) - StandardInputOutputValidation.Cancelled -> finish(SdrConversionResult.Cancelled) - } - } } - }, { reason -> handler.post { guarded { if (!isTerminal()) fail(reason) } } }) + complete(if (videoMime != targets.videoMime || + audioMime != if (targets.audioChannels != null) MimeTypes.AUDIO_AAC else null) + SdrConversionFailure.CodecChanged else null) + }, ::complete) + } + if (exportFailure != null) return SdrConversionResult.Failed(exportFailure) + return withContext(ioDispatcher) { + if (!sourceUnchanged()) return@withContext SdrConversionResult.Failed(SdrConversionFailure.SourceChanged) + when (val validation = validate(owned.file, source.facts, source.timeline, conversion) { !job.isActive }) { + is StandardInputOutputValidation.Accepted -> { + if (!sourceUnchanged()) return@withContext SdrConversionResult.Failed(SdrConversionFailure.SourceChanged) + // The shared validator checks policy, pixels, cadence, and effective timeline. + // Add this adapter's concrete encoder-profile and non-AAC channel contract. + val audio = validation.facts.audioTracks.valueOrNull?.singleOrNull()?.format?.valueOrNull as? AudioFormat.Aac + val expectedLayout = aacChannelLayout(targets.audioChannels).valueOrNull + if (validation.facts.videoProfile != MediaFact.Known(targets.profile) || + (targets.audioChannels != null && audio?.layout != expectedLayout)) + SdrConversionResult.Failed(SdrConversionFailure.OutputInvalid) + else SdrConversionResult.Completed(owned, validation.facts) + } + is StandardInputOutputValidation.Rejected -> SdrConversionResult.Failed(SdrConversionFailure.OutputInvalid) + StandardInputOutputValidation.Cancelled -> SdrConversionResult.Cancelled + } } - } } + } private fun sourceUnchanged() = input.isFile && input.canRead() && sourceLength > 0 && input.length() == sourceLength && input.lastModified() == sourceModified - private fun isTerminal() = synchronized(terminalLock) { terminal } - private fun fail(reason: SdrConversionFailure) = finish(SdrConversionResult.Failed(reason)) - private fun guarded(block: () -> Unit) { - if (isTerminal()) return - try { block() } - catch (_: Exception) { fail(SdrConversionFailure.Export) } - catch (_: LinkageError) { fail(SdrConversionFailure.Export) } - } - - private fun finish(proposed: SdrConversionResult) { - val result = synchronized(terminalLock) { + private suspend fun finish(proposed: SdrConversionResult) { + var result = synchronized(terminalLock) { if (terminal) return terminal = true - if (cancelled.get()) SdrConversionResult.Cancelled else proposed + if (job.isCancelled) SdrConversionResult.Cancelled else proposed } try { runCatching { engine?.cancel() } - if (result !is SdrConversionResult.Completed && output?.delete() == false) { + if (result !is SdrConversionResult.Completed && withContext(ioDispatcher) { output?.delete() } == false) { pendingCleanup = output // Keep ownership at its exact path if the OS refuses deletion; never sweep a directory. - if (result !is SdrConversionResult.Cancelled) { - onResult(SdrConversionResult.Failed(SdrConversionFailure.FileOwnership)) - return - } + if (result !is SdrConversionResult.Cancelled) result = SdrConversionResult.Failed(SdrConversionFailure.FileOwnership) } - onResult(result) + runCatching { onResult(result) } } finally { engine = null - handler.removeCallbacksAndMessages(null) - thread.quitSafely() + scope.cancel() } } } diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/SdrConversionAdapterTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/SdrConversionAdapterTests.kt index 0c3b350a..be21e994 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/SdrConversionAdapterTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/SdrConversionAdapterTests.kt @@ -2,17 +2,24 @@ package com.mux.video.upload.internal.standardization import android.content.Context import android.media.MediaFormat +import android.os.Looper import androidx.media3.common.MediaItem import androidx.media3.transformer.Composition import androidx.media3.transformer.EditedMediaItem import androidx.media3.transformer.EditedMediaItemSequence import androidx.media3.transformer.ExportResult import androidx.test.core.app.ApplicationProvider +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.test.StandardTestDispatcher +import kotlinx.coroutines.test.resetMain +import kotlinx.coroutines.test.setMain +import org.junit.After +import org.junit.Before import org.junit.Assert.* import org.junit.Test import org.junit.runner.RunWith import org.robolectric.RobolectricTestRunner -import org.robolectric.Shadows.shadowOf import org.robolectric.annotation.Config import org.robolectric.annotation.LooperMode import java.io.File @@ -21,7 +28,11 @@ import java.util.concurrent.CopyOnWriteArrayList @RunWith(RobolectricTestRunner::class) @Config(manifest = Config.NONE, sdk = [28]) @LooperMode(LooperMode.Mode.PAUSED) +@OptIn(ExperimentalCoroutinesApi::class) class SdrConversionAdapterTests { + private val dispatcher = StandardTestDispatcher() + @Before fun setUpDispatchers() { Dispatchers.setMain(dispatcher) } + @After fun resetDispatchers() { Dispatchers.resetMain() } private val context = ApplicationProvider.getApplicationContext() private val facts = compliantFacts().copy(cadence = known(Cadence.Constant), videoProfile = known(VideoProfile.H264Baseline)) private val conversion = (StandardInputPlanner().plan(facts.copy(averageBitrate = known(9_000_000L)), @@ -160,10 +171,11 @@ class SdrConversionAdapterTests { var starts = 0 var cancellations = 0 override fun start(input: File, output: File, completed: (String?, String?) -> Unit, failed: (SdrConversionFailure) -> Unit) { + assertEquals(Looper.getMainLooper(), Looper.myLooper()) starts++; this.output = output; completions = completed; failures = failed output.writeText("owned partial bytes") } - override fun cancel() { cancellations++ } + override fun cancel() { assertEquals(Looper.getMainLooper(), Looper.myLooper()); cancellations++ } fun complete(video: String? = "video/avc", audio: String? = "audio/mp4a-latm") { completions!!(video, audio) } } private inner class Run( @@ -171,30 +183,18 @@ class SdrConversionAdapterTests { StandardInputPolicyEvaluator().evaluate(facts, selection())), capability: SdrEncoderCapability? = SdrEncoderCapability("fake", 1), duringValidation: (() -> Unit)? = null, + duringPreflight: (() -> Unit)? = null, inputMetadata: MediaMetadataInspection = metadata(), ) { val sourceFile = File.createTempFile("customer-", ".mp4", context.cacheDir).apply { writeText("original") } val unrelated = File(context.cacheDir, "mux-upload/customer-copy.mp4").apply { parentFile!!.mkdirs(); writeText("customer") } val engine = FakeEngine() val results = CopyOnWriteArrayList() - val adapter = SdrConversionAdapter(context, preflight = { _, _, _, _ -> capability }, - validate = { _, _, _, _, _ -> duringValidation?.invoke(); validation }, createEngine = { _, _, _, _, _ -> engine }) + val adapter = SdrConversionAdapter(context, preflight = { _, _, _, _ -> duringPreflight?.invoke(); capability }, + validate = { _, _, _, _, _ -> duringValidation?.invoke(); validation }, createEngine = { _, _, _, _, _ -> engine }, + ioDispatcher = dispatcher) val attempt = adapter.start(sourceFile, inputMetadata, samples(), conversion, results::add) - fun idle() { - val thread = attempt.looper.thread - if (results.isEmpty() && thread.isAlive) { - try { shadowOf(attempt.looper).idle() } - catch (error: IllegalStateException) { - // PAUSED-mode idle can race a terminal quit before its barrier is posted. - thread.join(5_000) - if (results.isEmpty() || thread.isAlive) throw error - } - } - if (results.isNotEmpty()) { - thread.join(5_000) - assertFalse("Terminal attempt thread still running", thread.isAlive) - } - } + fun idle() { dispatcher.scheduler.runCurrent() } fun assertInputSafe() { assertEquals("original", sourceFile.readText()); assertEquals("customer", unrelated.readText()) } } @Test fun successRetainsVerifiedFileUntilOwnerDeletesItAndIgnoresLateEvents() { @@ -234,6 +234,29 @@ class SdrConversionAdapterTests { assertEquals(listOf(SdrConversionResult.Cancelled), run.results) assertEquals(0, run.engine.starts); run.assertInputSafe() } + @Test fun cancelWithoutMedia3CallbackStopsExportAndDeletesPartial() { + val run = Run(duringValidation = { fail("Cancelled export must not validate") }) + run.idle() + run.attempt.cancel() + run.idle() + assertEquals(listOf(SdrConversionResult.Cancelled), run.results) + assertEquals(1, run.engine.cancellations) + assertFalse(run.engine.output!!.exists()) + run.engine.complete() + run.engine.failures!!(SdrConversionFailure.Export) + run.idle() + assertEquals(1, run.results.size) + run.assertInputSafe() + } + @Test fun cancellationDuringPreflightDoesNotAllocateOrStartExport() { + lateinit var run: Run + run = Run(duringPreflight = { run.attempt.cancel() }) + run.idle() + assertEquals(listOf(SdrConversionResult.Cancelled), run.results) + assertNull(run.engine.output) + assertEquals(0, run.engine.starts) + run.assertInputSafe() + } @Test fun forbiddenFallbackWinsOverLaterCompletion() { val run = Run(); run.idle(); run.engine.failures!!(SdrConversionFailure.ForbiddenFallback); run.engine.complete(); run.idle() assertEquals(listOf(SdrConversionResult.Failed(SdrConversionFailure.ForbiddenFallback)), run.results) From 454cc336986519bebe8fd6327b7d05471b07eff5 Mon Sep 17 00:00:00 2001 From: Eugene Morozov Date: Wed, 7 Oct 2026 12:34:09 -0700 Subject: [PATCH 40/49] NAT-547: define SDR targets and bounded codec preflight --- .../standardization/SdrEncodingTargets.kt | 145 ++++++++++++++++++ 1 file changed, 145 insertions(+) create mode 100644 library/src/main/java/com/mux/video/upload/internal/standardization/SdrEncodingTargets.kt diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/SdrEncodingTargets.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/SdrEncodingTargets.kt new file mode 100644 index 00000000..4853a32a --- /dev/null +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/SdrEncodingTargets.kt @@ -0,0 +1,145 @@ +package com.mux.video.upload.internal.standardization + +import android.media.MediaCodecInfo +import android.media.MediaCodecInfo.CodecProfileLevel +import android.media.MediaCodecList +import android.media.MediaFormat +import kotlin.math.abs +import kotlin.math.ceil + +internal enum class SdrConversionFailure { + UnsupportedPlan, InsufficientSourceEvidence, CapabilityUnavailable, DiskSpace, + Export, ForbiddenFallback, CodecChanged, OutputInvalid, SourceChanged, FileOwnership, +} + +/** Encoder targets are deliberately below policy ceilings; output validation still proves compliance. */ +internal data class SdrEncodingTargets( + val videoMime: String, + val profile: VideoProfile, + val platformProfile: Int, + val bitrate: Int, + val keyframeIntervalSeconds: Int, + val audioChannels: Int?, + val audioSampleRate: Int?, + val copyAac: Boolean, + val firstAudioTrackId: String?, +) { + fun estimatedOutputBytes(duration: Double): Long? { + if (!duration.isFinite() || duration <= 0) return null + val bytes = ceil(duration * (bitrate + if (audioChannels != null) 384_000 else 0) / 8 * 1.5) + 8 * 1024 * 1024 + return bytes.takeIf { it.isFinite() && it > 0 && it < Long.MAX_VALUE.toDouble() }?.toLong() + } + + companion object { + fun from(conversion: StandardInputConversion, metadata: MediaMetadataInspection, + source: MediaSampleInspection): SdrEncodingTargets? { + val facts = source.facts + if (source.status != SampleScanStatus.Complete || conversion.toneMapsToSdr || + conversion.sourceDynamicRange != DynamicRange.Sdr || facts.dynamicRange != MediaFact.Known(DynamicRange.Sdr) || + facts.videoCodec != MediaFact.Known(conversion.sourceCodec) || facts.videoTrackCount != MediaFact.Known(1) || + facts.displayDimensions != MediaFact.Known(conversion.sourceDisplayDimensions) || + source.timeline.videoPresentationSeconds.valueOrNull.isNullOrEmpty() || + source.timeline.videoDurationSeconds.valueOrNull == null || + source.timeline.audioVideoStartOffset == MediaFact.Unknown) return null + val expectedCodec = if (conversion.sourceCodec == VideoCodec.Hevc) VideoCodec.Hevc else VideoCodec.H264 + val size = conversion.outputDimensions + val rate = conversion.outputFrameRate + if (conversion.outputCodec != expectedCodec || !size.isValid || size.width % 2 != 0 || size.height % 2 != 0 || + size.width > conversion.sourceDisplayDimensions.width || size.height > conversion.sourceDisplayDimensions.height || + !size.fitsWithin(conversion.selection.generatedOutputDimensions) || + !StandardInputOutputValidator.preservesAspectRatio(conversion.sourceDisplayDimensions, size) || + !rate.isFinite() || rate !in conversion.outputPolicyLimits.frameRateRange || + conversion.outputPixelFormat != PixelFormat(8, ChromaSubsampling.Yuv420)) return null + // This SDR surface path has no proof of 10-bit SDR preservation or frame duplication. + val sourceRate = facts.frameRate.valueOrNull ?: return null + if (conversion.outputCadence == OutputCadence.ConstantFrameRate) { + val ratio = sourceRate / rate + if (facts.cadence != MediaFact.Known(Cadence.Constant) || rate != 30.0 || ratio < 1 || + // Measured rates use microsecond-rounded timestamps; permit only rounding-scale noise. + abs(ratio - kotlin.math.round(ratio)) > maxOf(1.0, ratio) * 1e-6) return null + } else if (sourceRate != rate || facts.cadence == MediaFact.Unknown) return null + val audioFacts = facts.audioTracks.valueOrNull ?: return null + val audio = metadata.tracks.filter { it.kind == TrackKind.Audio }.sortedBy { it.containerIndex.valueOrNull ?: Int.MAX_VALUE } + if (audio.size != audioFacts.size || audio.any { it.containerIndex == MediaFact.Unknown }) return null + if ((conversion.outputAudio == OutputAudio.None) != audio.isEmpty() && + conversion.outputAudio != OutputAudio.AacFromFirstTrackIfPresent) return null + val first = audio.firstOrNull() + val channels = first?.audio?.channelCount?.valueOrNull + val sampleRate = first?.audio?.sampleRate?.valueOrNull + val copy = first?.audio?.format?.valueOrNull is AudioFormat.Aac + val layout = (first?.audio?.format?.valueOrNull as? AudioFormat.Aac)?.layout + if (first != null && (first.audio?.format == MediaFact.Unknown || first.sourceTrackId.valueOrNull == null || channels == null || sampleRate == null || + sampleRate <= 0 || (if (copy) layout !in setOf(AudioChannelLayout.Mono, AudioChannelLayout.Stereo, + AudioChannelLayout.FivePointOne) else channels !in 1..2))) return null + val interval = conversion.outputPolicyLimits.maximumKeyframeIntervals[expectedCodec] ?: return null + if (!interval.isFinite() || interval < 1) return null + return SdrEncodingTargets( + if (expectedCodec == VideoCodec.Hevc) "video/hevc" else "video/avc", + if (expectedCodec == VideoCodec.Hevc) VideoProfile.HevcMain else VideoProfile.H264Baseline, + if (expectedCodec == VideoCodec.Hevc) CodecProfileLevel.HEVCProfileMain else CodecProfileLevel.AVCProfileBaseline, + minOf(6_000_000L.takeIf { size.longSide <= 2048 } ?: 15_000_000L, + conversion.outputPolicyLimits.maximumAverageBitrate * 3 / 4).toInt().takeIf { it > 0 } ?: return null, + maxOf(1, (interval * 0.9).toInt()), channels, sampleRate, copy, first?.sourceTrackId?.valueOrNull?.toString(), + ) + } + } +} + +internal data class SdrEncoderCapability(val name: String, val level: Int) + +/** Bounded advertised-format guard. This does not manufacture planner/device-path proof. */ +internal class SdrCapabilityPreflight { + fun videoEncoder(conversion: StandardInputConversion, metadata: MediaMetadataInspection, + targets: SdrEncodingTargets, isCancelled: () -> Boolean): SdrEncoderCapability? { + val started = System.nanoTime() + fun withinBudget() = !isCancelled() && System.nanoTime() - started < 500_000_000L + val infos = MediaCodecList(MediaCodecList.REGULAR_CODECS).codecInfos + if (infos.size > 128 || !withinBudget()) return null + val video = metadata.tracks.singleOrNull { it.kind == TrackKind.Video } ?: return null + val sourceMime = video.mimeType.valueOrNull ?: return null + val sourceSize = video.video?.encodedDimensions?.valueOrNull ?: return null + fun supportsDecode(mime: String, format: MediaFormat): Boolean = infos.any { info -> + withinBudget() && !info.isEncoder && info.supportedTypes.any { it.equals(mime, true) } && + runCatching { info.getCapabilitiesForType(mime).isFormatSupported(format) }.getOrDefault(false) + } + val sourceFormat = MediaFormat.createVideoFormat(sourceMime, sourceSize.width, sourceSize.height) + val sourceProfile = when (video.video.profile.valueOrNull) { + VideoProfile.H264Baseline -> CodecProfileLevel.AVCProfileBaseline + VideoProfile.H264Main -> CodecProfileLevel.AVCProfileMain + VideoProfile.H264High -> CodecProfileLevel.AVCProfileHigh + VideoProfile.HevcMain -> CodecProfileLevel.HEVCProfileMain + VideoProfile.HevcMain10 -> CodecProfileLevel.HEVCProfileMain10 + else -> null + } + if (sourceProfile != null) sourceFormat.setInteger(MediaFormat.KEY_PROFILE, sourceProfile) + if (!supportsDecode(sourceMime, sourceFormat)) return null + if (targets.audioChannels != null && !targets.copyAac) { + val audioMime = metadata.tracks.first { it.sourceTrackId.valueOrNull?.toString() == targets.firstAudioTrackId }.mimeType.valueOrNull ?: return null + if (!supportsDecode(audioMime, MediaFormat.createAudioFormat(audioMime, targets.audioSampleRate!!, targets.audioChannels))) return null + val aac = MediaFormat.createAudioFormat("audio/mp4a-latm", targets.audioSampleRate, targets.audioChannels) + aac.setInteger(MediaFormat.KEY_BIT_RATE, 160_000) + aac.setInteger(MediaFormat.KEY_AAC_PROFILE, CodecProfileLevel.AACObjectLC) + if (infos.none { withinBudget() && it.isEncoder && it.supportedTypes.contains("audio/mp4a-latm") && + runCatching { it.getCapabilitiesForType("audio/mp4a-latm").isFormatSupported(aac) }.getOrDefault(false) }) return null + } + for (info in infos) { + if (!withinBudget()) return null + if (!info.isEncoder || !info.supportedTypes.contains(targets.videoMime)) continue + val candidate = runCatching { + val caps = info.getCapabilitiesForType(targets.videoMime) + val level = caps.profileLevels.filter { it.profile == targets.platformProfile }.maxOfOrNull { it.level } ?: return@runCatching null + val size = conversion.outputDimensions + val format = MediaFormat.createVideoFormat(targets.videoMime, size.width, size.height) + format.setFloat(MediaFormat.KEY_FRAME_RATE, conversion.outputFrameRate.toFloat()) + format.setInteger(MediaFormat.KEY_BIT_RATE, targets.bitrate) + format.setInteger(MediaFormat.KEY_PROFILE, targets.platformProfile) + format.setInteger(MediaFormat.KEY_LEVEL, level) + format.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface) + format.setInteger(MediaFormat.KEY_BITRATE_MODE, MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_VBR) + if (caps.isFormatSupported(format)) SdrEncoderCapability(info.name, level) else null + }.getOrNull() + if (candidate != null && withinBudget()) return candidate + } + return null + } +} From b9f3e3e576f5a6b9c9be7ee0d0932925b8decc58 Mon Sep 17 00:00:00 2001 From: Eugene Morozov Date: Wed, 7 Oct 2026 12:34:09 -0700 Subject: [PATCH 41/49] NAT-547: own cancellable Media3 SDR export and validated files --- .../standardization/Media3SdrExportEngine.kt | 154 +++++++++++++++++ .../standardization/SdrConversionAdapter.kt | 162 ++++++++++++++++++ 2 files changed, 316 insertions(+) create mode 100644 library/src/main/java/com/mux/video/upload/internal/standardization/Media3SdrExportEngine.kt create mode 100644 library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/Media3SdrExportEngine.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/Media3SdrExportEngine.kt new file mode 100644 index 00000000..027ffade --- /dev/null +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/Media3SdrExportEngine.kt @@ -0,0 +1,154 @@ +package com.mux.video.upload.internal.standardization + +import android.content.Context +import android.media.MediaCodecInfo +import android.media.MediaFormat +import android.media.metrics.LogSessionId +import android.net.Uri +import android.os.Build +import android.os.Looper +import android.util.Pair +import androidx.media3.common.C +import androidx.media3.common.Format +import androidx.media3.common.MediaItem +import androidx.media3.common.util.Clock +import androidx.media3.common.util.MediaFormatUtil +import androidx.media3.common.util.UnstableApi +import androidx.media3.effect.Presentation +import androidx.media3.exoplayer.trackselection.DefaultTrackSelector +import androidx.media3.exoplayer.trackselection.ExoTrackSelection +import androidx.media3.transformer.AudioEncoderSettings +import androidx.media3.transformer.Codec +import androidx.media3.transformer.Composition +import androidx.media3.transformer.DefaultCodec +import androidx.media3.transformer.DefaultDecoderFactory +import androidx.media3.transformer.DefaultEncoderFactory +import androidx.media3.transformer.EditedMediaItem +import androidx.media3.transformer.EditedMediaItemSequence +import androidx.media3.transformer.Effects +import androidx.media3.transformer.ExoPlayerAssetLoader +import androidx.media3.transformer.ExportException +import androidx.media3.transformer.ExportResult +import androidx.media3.transformer.TransformationRequest +import androidx.media3.transformer.Transformer +import java.io.File + +@UnstableApi +internal class Media3SdrExportEngine( + private val context: Context, private val looper: Looper, + private val conversion: StandardInputConversion, private val targets: SdrEncodingTargets, + private val capability: SdrEncoderCapability, +) : SdrExportEngine { + private var transformer: Transformer? = null + + override fun start(input: File, output: File, completed: (String?, String?) -> Unit, + failed: (SdrConversionFailure) -> Unit) { + check(Looper.myLooper() == looper) + val decoder = DefaultDecoderFactory.Builder(context).setEnableDecoderFallback(false).build() + val assetLoader = ExoPlayerAssetLoader.Factory(context, decoder, Clock.DEFAULT, null, + { ctx -> FirstAudioTrackSelector(ctx, targets.firstAudioTrackId) }, null, null) + val item = EditedMediaItem.Builder(MediaItem.fromUri(Uri.fromFile(input))) + .setRemoveAudio(targets.audioChannels == null) + .setEffects(Effects(emptyList(), listOf(Presentation.createForWidthAndHeight( + conversion.outputDimensions.width, conversion.outputDimensions.height, Presentation.LAYOUT_SCALE_TO_FIT)))) + if (conversion.outputCadence == OutputCadence.ConstantFrameRate) item.setFrameRate(30) + val items = listOf(item.build()) + val sequence = if (targets.audioChannels == null) EditedMediaItemSequence.withVideoFrom(items) + else EditedMediaItemSequence.withAudioAndVideoFrom(items) + val composition = Composition.Builder(sequence) + .setHdrMode(Composition.HDR_MODE_KEEP_HDR).build() + val exporter = Transformer.Builder(context) + .setLooper(looper) + .setPortraitEncodingEnabled(true) + .setVideoMimeType(targets.videoMime) + .setAudioMimeType("audio/mp4a-latm") + .setAssetLoaderFactory(assetLoader) + .setEncoderFactory(StrictSdrEncoderFactory(context, conversion, targets, capability)) + .addListener(object : Transformer.Listener { + override fun onCompleted(composition: Composition, exportResult: ExportResult) { + if (targets.audioChannels != null && (exportResult.channelCount != targets.audioChannels || + exportResult.sampleRate != targets.audioSampleRate)) failed(SdrConversionFailure.OutputInvalid) + else completed(exportResult.videoMimeType, exportResult.audioMimeType) + } + override fun onError(composition: Composition, exportResult: ExportResult, exportException: ExportException) { + failed(SdrConversionFailure.Export) + } + override fun onFallbackApplied(composition: Composition, originalTransformationRequest: TransformationRequest, + fallbackTransformationRequest: TransformationRequest) { + failed(SdrConversionFailure.ForbiddenFallback) + } + }).build() + transformer = exporter + exporter.start(composition, output.absolutePath) + } + + override fun cancel() { + check(Looper.myLooper() == looper) + transformer?.cancel() + transformer = null + } +} + +/** MP4 Format.id is the container track ID. Never choose a preferred/default secondary track. */ +@UnstableApi +internal class FirstAudioTrackSelector(context: Context, private val trackId: String?) : DefaultTrackSelector(context) { + init { + setParameters(buildUponParameters().setForceHighestSupportedBitrate(true) + .setExceedAudioConstraintsIfNecessary(true).setExceedRendererCapabilitiesIfNecessary(true)) + } + override fun selectAudioTrack(mappedTrackInfo: MappedTrackInfo, rendererFormatSupports: Array>, + rendererMixedMimeTypeAdaptationSupports: IntArray, params: Parameters): Pair? { + if (trackId == null) return null + for (renderer in 0 until mappedTrackInfo.rendererCount) { + if (mappedTrackInfo.getRendererType(renderer) != C.TRACK_TYPE_AUDIO) continue + val groups = mappedTrackInfo.getTrackGroups(renderer) + for (groupIndex in 0 until groups.length) { + val group = groups[groupIndex] + for (track in 0 until group.length) { + if (group.getFormat(track).id == trackId) return Pair(ExoTrackSelection.Definition(group, track), renderer) + } + } + } + error("First audio track identity unavailable") + } +} + +/** + * Media3's default video factory overrides H.264 profile requests. Supply its DefaultCodec with + * exact platform settings instead; Transformer still owns the entire decode/effect/encode pipeline. + * No alternative encoder, size, profile, MIME, or HDR mode is attempted on configuration failure. + */ +@UnstableApi +internal class StrictSdrEncoderFactory( + private val context: Context, private val conversion: StandardInputConversion, + private val targets: SdrEncodingTargets, private val capability: SdrEncoderCapability, +) : Codec.EncoderFactory { + private val audioDelegate = DefaultEncoderFactory.Builder(context) + .setEnableFallback(false).setEnableFormatFallback(false) + .setRequestedAudioEncoderSettings(AudioEncoderSettings.Builder().setBitrate(160_000) + .setProfile(MediaCodecInfo.CodecProfileLevel.AACObjectLC).build()).build() + + override fun videoNeedsEncoding() = true + override fun audioNeedsEncoding() = targets.audioChannels != null && !targets.copyAac + + override fun createForAudioEncoding(format: Format, logSessionId: LogSessionId?): Codec { + check(format.sampleMimeType == "audio/mp4a-latm" && format.channelCount == targets.audioChannels && + format.sampleRate == targets.audioSampleRate) + return audioDelegate.createForAudioEncoding(format, logSessionId) + } + + override fun createForVideoEncoding(format: Format, logSessionId: LogSessionId?): Codec { + check(format.sampleMimeType == targets.videoMime && format.width == conversion.outputDimensions.width && + format.height == conversion.outputDimensions.height && format.rotationDegrees == 0 && + format.colorInfo?.colorTransfer == C.COLOR_TRANSFER_SDR) + val exact = format.buildUpon().setFrameRate(conversion.outputFrameRate.toFloat()).setAverageBitrate(targets.bitrate).build() + val media = MediaFormatUtil.createMediaFormatFromFormat(exact) + media.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface) + media.setInteger(MediaFormat.KEY_BITRATE_MODE, MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_VBR) + media.setInteger(MediaFormat.KEY_PROFILE, targets.platformProfile) + media.setInteger(MediaFormat.KEY_LEVEL, capability.level) + media.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, targets.keyframeIntervalSeconds) + if (Build.VERSION.SDK_INT >= 29) media.setInteger(MediaFormat.KEY_MAX_B_FRAMES, 0) + return DefaultCodec(context, exact, media, capability.name, false, null) + } +} diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt new file mode 100644 index 00000000..81448954 --- /dev/null +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt @@ -0,0 +1,162 @@ +package com.mux.video.upload.internal.standardization + +import android.content.Context +import android.os.Handler +import android.os.HandlerThread +import android.os.Looper +import java.io.File +import java.util.concurrent.atomic.AtomicBoolean + +internal sealed interface SdrConversionResult { + /** Caller owns retention until upload/resume reaches its terminal point. */ + data class Completed(val output: SdrGeneratedFile, val facts: MediaFacts) : SdrConversionResult + data class Failed(val reason: SdrConversionFailure) : SdrConversionResult + data object Cancelled : SdrConversionResult +} + +/** The only deletion authority returned to orchestration; never accepts an arbitrary path. */ +internal class SdrGeneratedFile private constructor(val file: File) { + fun delete(): Boolean = !file.exists() || file.delete() + + companion object { + fun allocate(cacheDir: File, requiredBytes: Long): SdrGeneratedFile? { + val area = File(cacheDir.canonicalFile, "mux-upload/standard-input") + if ((!area.isDirectory && !area.mkdirs()) || area.canonicalFile != area.absoluteFile || area.usableSpace < requiredBytes) return null + return SdrGeneratedFile(File.createTempFile("sdr-", ".mp4", area)) + } + } +} + +/** Small Media3 boundary so terminal races and cleanup can be tested without a platform encoder. */ +internal interface SdrExportEngine { + fun start(input: File, output: File, completed: (String?, String?) -> Unit, + failed: (SdrConversionFailure) -> Unit) + fun cancel() +} + +/** Each attempt owns one thread, one Transformer, and one explicitly tracked output file. */ +internal class SdrConversionAdapter( + context: Context, + private val preflight: (StandardInputConversion, MediaMetadataInspection, SdrEncodingTargets, () -> Boolean) -> SdrEncoderCapability? = + SdrCapabilityPreflight()::videoEncoder, + private val validate: (File, MediaFacts, StandardInputTimelineFacts, StandardInputConversion, () -> Boolean) -> StandardInputOutputValidation = + StandardInputOutputValidator()::validateGeneratedOutput, + private val createEngine: (Context, Looper, StandardInputConversion, SdrEncodingTargets, SdrEncoderCapability) -> SdrExportEngine = + ::Media3SdrExportEngine, +) { + private val appContext = context.applicationContext + + fun start(input: File, metadata: MediaMetadataInspection, source: MediaSampleInspection, + conversion: StandardInputConversion, onResult: (SdrConversionResult) -> Unit): Attempt = + Attempt(input, metadata, source, conversion, onResult).also { it.begin() } + + inner class Attempt internal constructor( + private val input: File, private val metadata: MediaMetadataInspection, + private val source: MediaSampleInspection, private val conversion: StandardInputConversion, + private val onResult: (SdrConversionResult) -> Unit, + ) { + private val thread = HandlerThread("MuxSdrConversion").apply { start() } + internal val looper: Looper = thread.looper + private val handler = Handler(looper) + private val cancelled = AtomicBoolean(false) + private val terminalLock = Any() + private var terminal = false + private var engine: SdrExportEngine? = null + private var output: SdrGeneratedFile? = null + /** Exact deletion authority retained if the OS refuses cleanup, including cancellation. */ + @Volatile var pendingCleanup: SdrGeneratedFile? = null + private set + private val sourceLength = input.length() + private val sourceModified = input.lastModified() + + fun cancel() { + synchronized(terminalLock) { + if (terminal) return + cancelled.set(true) + } + handler.post { finish(SdrConversionResult.Cancelled) } + } + + internal fun begin() { handler.post { + guarded { + if (cancelled.get()) return@guarded finish(SdrConversionResult.Cancelled) + val targets = SdrEncodingTargets.from(conversion, metadata, source) + ?: return@guarded fail(SdrConversionFailure.UnsupportedPlan) + val capability = preflight(conversion, metadata, targets, cancelled::get) + ?: return@guarded fail(SdrConversionFailure.CapabilityUnavailable) + if (cancelled.get()) return@guarded finish(SdrConversionResult.Cancelled) + if (!sourceUnchanged()) return@guarded fail(SdrConversionFailure.SourceChanged) + val required = source.timeline.durationSeconds.valueOrNull?.let(targets::estimatedOutputBytes) + ?: return@guarded fail(SdrConversionFailure.InsufficientSourceEvidence) + val owned = SdrGeneratedFile.allocate(appContext.cacheDir, required) + ?: return@guarded fail(SdrConversionFailure.DiskSpace) + output = owned + if (cancelled.get()) return@guarded finish(SdrConversionResult.Cancelled) + val exporter = createEngine(appContext, looper, conversion, targets, capability) + engine = exporter + exporter.start(input, owned.file, { videoMime, audioMime -> + handler.post { guarded { + if (isTerminal()) return@guarded + if (videoMime != targets.videoMime || audioMime != if (targets.audioChannels != null) "audio/mp4a-latm" else null) + return@guarded fail(SdrConversionFailure.CodecChanged) + if (!sourceUnchanged()) return@guarded fail(SdrConversionFailure.SourceChanged) + when (val validation = validate(owned.file, source.facts, source.timeline, conversion, cancelled::get)) { + is StandardInputOutputValidation.Accepted -> { + if (!sourceUnchanged()) return@guarded fail(SdrConversionFailure.SourceChanged) + // The shared validator checks policy, pixels, cadence, and effective timeline. + // Add this adapter's concrete encoder-profile and non-AAC channel contract. + val audio = validation.facts.audioTracks.valueOrNull?.singleOrNull()?.format?.valueOrNull as? AudioFormat.Aac + val expectedLayout = when (targets.audioChannels) { + 1 -> AudioChannelLayout.Mono + 2 -> AudioChannelLayout.Stereo + 6 -> AudioChannelLayout.FivePointOne + else -> null + } + if (validation.facts.videoProfile != MediaFact.Known(targets.profile) || + (targets.audioChannels != null && audio?.layout != expectedLayout)) fail(SdrConversionFailure.OutputInvalid) + else finish(SdrConversionResult.Completed(owned, validation.facts)) + } + is StandardInputOutputValidation.Rejected -> fail(SdrConversionFailure.OutputInvalid) + StandardInputOutputValidation.Cancelled -> finish(SdrConversionResult.Cancelled) + } + } } + }, { reason -> handler.post { guarded { if (!isTerminal()) fail(reason) } } }) + } + } } + + private fun sourceUnchanged() = input.isFile && input.canRead() && sourceLength > 0 && + input.length() == sourceLength && input.lastModified() == sourceModified + private fun isTerminal() = synchronized(terminalLock) { terminal } + private fun fail(reason: SdrConversionFailure) = finish(SdrConversionResult.Failed(reason)) + private fun guarded(block: () -> Unit) { + if (isTerminal()) return + try { block() } + catch (_: Exception) { fail(SdrConversionFailure.Export) } + catch (_: LinkageError) { fail(SdrConversionFailure.Export) } + } + + private fun finish(proposed: SdrConversionResult) { + val result = synchronized(terminalLock) { + if (terminal) return + terminal = true + if (cancelled.get()) SdrConversionResult.Cancelled else proposed + } + try { + runCatching { engine?.cancel() } + if (result !is SdrConversionResult.Completed && output?.delete() == false) { + pendingCleanup = output + // Keep ownership at its exact path if the OS refuses deletion; never sweep a directory. + if (result !is SdrConversionResult.Cancelled) { + onResult(SdrConversionResult.Failed(SdrConversionFailure.FileOwnership)) + return + } + } + onResult(result) + } finally { + engine = null + handler.removeCallbacksAndMessages(null) + thread.quitSafely() + } + } + } +} From b710a35a031e9bba925fe6419a7b536bf41bcde5 Mon Sep 17 00:00:00 2001 From: Eugene Morozov Date: Wed, 7 Oct 2026 12:34:09 -0700 Subject: [PATCH 42/49] NAT-547: cover terminal races and document adapter evidence --- .../upload/internal/standardization/README.md | 36 ++++ .../SdrConversionAdapterTests.kt | 193 ++++++++++++++++++ 2 files changed, 229 insertions(+) create mode 100644 library/src/main/java/com/mux/video/upload/internal/standardization/README.md create mode 100644 library/src/test/java/com/mux/video/upload/internal/standardization/SdrConversionAdapterTests.kt diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/README.md b/library/src/main/java/com/mux/video/upload/internal/standardization/README.md new file mode 100644 index 00000000..5585b973 --- /dev/null +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/README.md @@ -0,0 +1,36 @@ +# SDR adapter (NAT-547) + +`SdrConversionAdapter.start` accepts the original file, its metadata and complete shared sample/timeline inspection, and the planner's `StandardInputConversion`. These must describe the same unchanged source. It exports on a dedicated `HandlerThread`, validates the generated file with `StandardInputOutputValidator`, and invokes one internal terminal callback on that thread. It does not start transport or emit public upload events. + +- `Completed` transfers an explicitly owned `SdrGeneratedFile` and validated facts to orchestration. Keep it while upload/resume needs it; call its idempotent `delete()` at the terminal lifecycle point. +- `Failed` permits original-file fallback. No partial or rejected output is selected. +- `Cancelled` stops preparation even without a Transformer callback. Orchestration must suppress public results and must not upload the original as a cancellation fallback. +- If deletion fails, `Attempt.pendingCleanup` retains the exact deletion authority for retry. Nothing sweeps `cacheDir/mux-upload`, where customer source copies can live. + +Each attempt allocates only its tracked unique `sdr-*.mp4` file under `cacheDir/mux-upload/standard-input`, after a size-based disk guard. It deletes that file on failure/cancellation, releases Transformer, and quits its thread. A successful file outlives that thread. Persisted payload identity, abandoned-file recovery, and generated resume belong to NAT-549; upload/pause orchestration belongs to NAT-548. + +## Encoding contract + +H.264 stays H.264, HEVC stays HEVC, and another proven SDR source uses H.264. Dimensions are the planner's exact display-oriented even dimensions, with orientation baked into pixels and no upscaling. Portrait encoding is explicitly enabled. Video always encodes, including an identity presentation effect, so bitrate/GOP remediation cannot silently remux the source. + +The 8-bit 4:2:0 path requests H.264 Baseline (no B frames) or HEVC Main, VBR at 6 Mbps through a 2048-pixel long side or 15 Mbps above it, additionally capped to 75% of the plan's average-bitrate ceiling. GOP targets use 90% of the planned interval ceiling, rounded down to whole seconds, following Swift's headroom convention. Output validation remains the final bitrate, IDR/closed-GOP, profile, pixel, cadence, and timeline gate. + +Media3's default video factory overrides H.264 profile requests. The adapter therefore supplies Media3 `DefaultCodec` with the exact platform profile, level, dimensions, bitrate, nominal rate, surface input, and GOP interval. Transformer owns decoding, effects, encoding, and muxing. The AAC delegate disables both encoder and format fallback. Every `onFallbackApplied`, mismatched output MIME, or invalid generated file fails the attempt; a listed encoder never proves a usable path. + +The bounded advertised-format preflight examines at most 128 codecs and checks a 500 ms budget between platform calls. Native calls cannot be preempted. It checks source decode and exact output encode support before creating a file, and pins one supported video encoder. It does not turn advertised capabilities into the planner's device-path proof. + +Preserved cadence keeps source timestamps. Constant-rate reduction is limited to proven constant cadence with an integral source/30 fps ratio; a one-part-per-million allowance covers microsecond-rounded rate measurement. Media3 drops frames at 30 fps and the shared validator checks actual output cadence and duration. Frame duplication, fractional-rate resampling, 10-bit SDR encoding, and HDR/tone mapping have no proof in this adapter and fail conservatively. NAT-550 owns tone mapping. + +## Audio decision + +[Shipped Swift v1.2.0](https://github.com/muxinc/swift-upload-sdk/blob/faa440e6ea9339315a9d192ca314c4ef5a8668c9/Sources/MuxUploadSDK/InputStandardization/UploadInputStandardizationWorker.swift#L794) copies AAC and otherwise preserves sample rate, channel count, and available layout while encoding AAC (160 kbps for mono/stereo; 384 kbps for surround). Following that behavior and the Android TDD's exclusion of new surround conversion, this adapter copies known compliant AAC mono/stereo/5.1, converts known non-AAC mono/stereo to AAC-LC at 160 kbps with the same rate/count, and rejects non-AAC surround or unknown layouts. It adds no downmix policy. Output channel layout and reported export sample rate/count must match these targets. + +The asset loader's selector matches the first audio track's container ID, rather than Media3's preferred/default track. Missing identity fails; it never substitutes a secondary track. + +## Evidence and remaining limits + +At implementation time, 210 unit tests pass in each debug/release variant; release SDK/sample builds and lint-vital pass. External API 36 emulator checks cover six accepted exports: H.264 resize, silent H.264, rotated portrait, MP3 stereo to AAC, short HEVC Main, and 240-to-30 fps H.264. Two HEVC failures exercise safe rejection: 720p exceeds the emulator's 512-pixel encoder limit; a longer small export emits CRA/open GOP and fails the existing validator. Cancellation removes generated bytes and preserves source copies. These are nine cases across three instrumentation tests, including the grouped export matrix. + +The distinguishable multi-audio source has 3s video, 3s first stereo AAC (track ID 2, non-default), and 6s secondary mono AAC (ID 3, default). Output contains the first stereo track: its copied AAC bytes have the same SHA-256 as source audio ordinal zero and differ from the default secondary. This proves local selection only; Mux ingest parity remains pending. + +Media binaries, fixture harness, output probes, and measurements stay outside product Git in `/private/tmp/nat547-sdr`. API 23-29 execution, physical hardware, long HEVC closed-GOP support, 10-bit SDR, HDR, high-resolution device coverage, visual quality/performance acceptance, and a live Mux upload are unverified. Emulator timings do not establish physical-device performance limits. NAT-548 can use the validated SDR path for the early upload journey; no predecessor merge is required to continue the stack. diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/SdrConversionAdapterTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/SdrConversionAdapterTests.kt new file mode 100644 index 00000000..c6a0d507 --- /dev/null +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/SdrConversionAdapterTests.kt @@ -0,0 +1,193 @@ +package com.mux.video.upload.internal.standardization + +import android.content.Context +import androidx.test.core.app.ApplicationProvider +import org.junit.Assert.* +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.Shadows.shadowOf +import org.robolectric.annotation.Config +import org.robolectric.annotation.LooperMode +import java.io.File + +@RunWith(RobolectricTestRunner::class) +@Config(manifest = Config.NONE, sdk = [28]) +@LooperMode(LooperMode.Mode.PAUSED) +class SdrConversionAdapterTests { + private val context = ApplicationProvider.getApplicationContext() + private val facts = compliantFacts().copy(cadence = known(Cadence.Constant), videoProfile = known(VideoProfile.H264Baseline)) + private val conversion = (StandardInputPlanner().plan(facts.copy(averageBitrate = known(9_000_000L)), + options(), fullCapabilities()).action as StandardInputAction.Convert).conversion + private val timeline = StandardInputTimelineFacts(known(3.0), known(AudioVideoStartOffset.Seconds(0.0)), + known(listOf(0.0, 1.0, 2.0)), known(3.0), known(3.0), known(90_000L)) + private fun audio(channels: Int = 2, format: MediaFact = known(AudioFormat.Aac(AudioChannelLayout.Stereo)), + id: Int = 2, index: Int = 1) = TrackMetadata(index, known(index), known(id), TrackKind.Audio, + known(if (format.valueOrNull is AudioFormat.Aac) "audio/mp4a-latm" else "audio/mpeg"), known(3.0), + MediaFact.Unknown, MediaFact.Unknown, audio = AudioTrackMetadata(format, known(channels), known(48_000), known(0), known(0))) + private fun metadata(tracks: List = listOf(audio())) = + MediaMetadataInspection(known(ContainerKind.IsoBaseMedia), tracks, MediaFact.Unknown, facts, 0) + private fun samples(input: MediaFacts = facts) = MediaSampleInspection(input, SampleScanStatus.Complete, timeline = timeline) + private fun targets(plan: StandardInputConversion = conversion, input: MediaFacts = facts, + tracks: List = listOf(audio())) = SdrEncodingTargets.from(plan, metadata(tracks), samples(input)) + + @Test fun pinsCodecFamilyAndTargetsBelowPublishedCeilings() { + assertEquals("video/avc", targets()!!.videoMime) + assertEquals(6_000_000, targets()!!.bitrate) + assertEquals(18, targets()!!.keyframeIntervalSeconds) + assertEquals(VideoProfile.H264Baseline, targets()!!.profile) + val hevc = facts.copy(videoCodec = known(VideoCodec.Hevc)) + assertEquals("video/hevc", targets(conversion.copy(sourceCodec = VideoCodec.Hevc, outputCodec = VideoCodec.Hevc), hevc)!!.videoMime) + assertNull(targets(conversion.copy(sourceCodec = VideoCodec.Hevc), hevc)) + val other = facts.copy(videoCodec = known(VideoCodec.Other)) + assertEquals("video/avc", targets(conversion.copy(sourceCodec = VideoCodec.Other), other)!!.videoMime) + } + @Test fun rejectsHdrUnknownTimelineUpscalingAndUnprovenTenBitPath() { + assertNull(targets(conversion.copy(toneMapsToSdr = true))) + assertNull(targets(input = facts.copy(dynamicRange = MediaFact.Unknown))) + assertNull(targets(conversion.copy(outputDimensions = Dimensions(1922, 1080)))) + assertNull(targets(conversion.copy(outputPixelFormat = PixelFormat(10, ChromaSubsampling.Yuv420)))) + assertNull(SdrEncodingTargets.from(conversion, metadata(), samples().copy(timeline = StandardInputTimelineFacts()))) + } + @Test fun preservesMonoStereoForNonAacWithoutInventingSurroundDownmix() { + for (channels in listOf(1, 2)) { + val nonAac = facts.copy(audioTracks = known(listOf(AudioTrack(known(AudioFormat.OtherCodec))))) + val target = targets(input = nonAac, tracks = listOf(audio(channels, known(AudioFormat.OtherCodec))))!! + assertEquals(channels, target.audioChannels) + assertFalse(target.copyAac) + } + assertNull(targets(tracks = listOf(audio(6, known(AudioFormat.OtherCodec))))) + assertNull(targets(tracks = listOf(audio(2, MediaFact.Unknown)))) + assertTrue(targets(tracks = listOf(audio(6, known(AudioFormat.Aac(AudioChannelLayout.FivePointOne)))))!!.copyAac) + } + @Test fun choosesContainerFirstTrackEvenWhenExtractorOrderDiffers() { + val multi = facts.copy(audioTracks = known(listOf(AudioTrack(), AudioTrack()))) + assertEquals("42", targets(input = multi, tracks = listOf(audio(id = 99, index = 3), audio(id = 42, index = 1)))!!.firstAudioTrackId) + assertNull(targets(tracks = listOf(audio().copy(sourceTrackId = MediaFact.Unknown)))) + } + @Test fun permitsOnlyProvenIntegralCadenceReductionAndKeepsVfrTimestamps() { + assertNotNull(targets(input = facts.copy(cadence = known(Cadence.Variable)))) + val cfr = conversion.copy(outputCadence = OutputCadence.ConstantFrameRate) + assertNotNull(targets(cfr, facts.copy(frameRate = known(240.0)))) + assertNotNull(targets(cfr, facts.copy(frameRate = known(240.00004008351402)))) + assertNull(targets(cfr, facts.copy(frameRate = known(240.0), cadence = known(Cadence.Variable)))) + assertNull(targets(cfr, facts.copy(frameRate = known(2.0)))) + assertNull(targets(cfr, facts.copy(frameRate = known(239.76)))) + } + + private class FakeEngine : SdrExportEngine { + var output: File? = null + var completions: ((String?, String?) -> Unit)? = null + var failures: ((SdrConversionFailure) -> Unit)? = null + var starts = 0 + var cancellations = 0 + override fun start(input: File, output: File, completed: (String?, String?) -> Unit, failed: (SdrConversionFailure) -> Unit) { + starts++; this.output = output; completions = completed; failures = failed + output.writeText("owned partial bytes") + } + override fun cancel() { cancellations++ } + fun complete(video: String? = "video/avc", audio: String? = "audio/mp4a-latm") { completions!!(video, audio) } + } + private inner class Run( + validation: StandardInputOutputValidation = StandardInputOutputValidation.Accepted(facts, + StandardInputPolicyEvaluator().evaluate(facts, selection())), + capability: SdrEncoderCapability? = SdrEncoderCapability("fake", 1), + duringValidation: (() -> Unit)? = null, + ) { + val sourceFile = File.createTempFile("customer-", ".mp4", context.cacheDir).apply { writeText("original") } + val unrelated = File(context.cacheDir, "mux-upload/customer-copy.mp4").apply { parentFile!!.mkdirs(); writeText("customer") } + val engine = FakeEngine() + val results = mutableListOf() + val adapter = SdrConversionAdapter(context, preflight = { _, _, _, _ -> capability }, + validate = { _, _, _, _, _ -> duringValidation?.invoke(); validation }, createEngine = { _, _, _, _, _ -> engine }) + val attempt = adapter.start(sourceFile, metadata(), samples(), conversion, results::add) + fun idle() { shadowOf(attempt.looper).idle() } + fun assertInputSafe() { assertEquals("original", sourceFile.readText()); assertEquals("customer", unrelated.readText()) } + } + @Test fun successRetainsVerifiedFileUntilOwnerDeletesItAndIgnoresLateEvents() { + val run = Run(); run.idle(); run.engine.complete(); run.idle() + val result = run.results.single() as SdrConversionResult.Completed + assertTrue(result.output.file.exists()); run.assertInputSafe() + run.engine.failures!!(SdrConversionFailure.Export); run.attempt.cancel() + assertEquals(1, run.results.size) + assertTrue(result.output.delete()); assertTrue(result.output.delete()) + assertEquals(1, run.engine.cancellations) + } + @Test fun exportFailureDeletesPartialAndPublishesExactlyOnce() { + val run = Run(); run.idle(); run.engine.failures!!(SdrConversionFailure.Export); run.idle() + assertEquals(listOf(SdrConversionResult.Failed(SdrConversionFailure.Export)), run.results) + assertFalse(run.engine.output!!.exists()); run.assertInputSafe() + run.engine.complete(); assertEquals(1, run.results.size) + } + @Test fun cancelTerminatesWithoutMedia3CallbackAndSuppressesQueuedCompletion() { + val run = Run(); run.idle(); run.engine.complete(); run.attempt.cancel(); run.idle() + assertEquals(listOf(SdrConversionResult.Cancelled), run.results) + assertFalse(run.engine.output!!.exists()); run.assertInputSafe() + } + @Test fun cancelBeforeStartAllocatesNothing() { + val run = Run(); run.attempt.cancel(); run.idle() + assertEquals(listOf(SdrConversionResult.Cancelled), run.results) + assertEquals(0, run.engine.starts); run.assertInputSafe() + } + @Test fun forbiddenFallbackWinsOverLaterCompletion() { + val run = Run(); run.idle(); run.engine.failures!!(SdrConversionFailure.ForbiddenFallback); run.engine.complete(); run.idle() + assertEquals(listOf(SdrConversionResult.Failed(SdrConversionFailure.ForbiddenFallback)), run.results) + assertFalse(run.engine.output!!.exists()) + } + @Test fun codecChangeAndInvalidOutputCannotProduceCompletedResult() { + val changed = Run(); changed.idle(); changed.engine.complete("video/hevc"); changed.idle() + assertEquals(SdrConversionResult.Failed(SdrConversionFailure.CodecChanged), changed.results.single()) + assertFalse(changed.engine.output!!.exists()) + val invalid = Run(StandardInputOutputValidation.Rejected(OutputRejection.EmptyOrUnreadable)) + invalid.idle(); invalid.engine.complete(); invalid.idle() + assertEquals(SdrConversionResult.Failed(SdrConversionFailure.OutputInvalid), invalid.results.single()) + assertFalse(invalid.engine.output!!.exists()) + } + @Test fun unavailableCapabilityDoesNotStartOrAllocate() { + val run = Run(capability = null); run.idle() + assertEquals(SdrConversionResult.Failed(SdrConversionFailure.CapabilityUnavailable), run.results.single()) + assertEquals(0, run.engine.starts); run.assertInputSafe() + } + @Test fun sourceChangeDuringValidationRejectsGeneratedBytes() { + lateinit var run: Run + run = Run(duringValidation = { run.sourceFile.appendText("changed") }) + run.idle(); run.engine.complete(); run.idle() + assertEquals(SdrConversionResult.Failed(SdrConversionFailure.SourceChanged), run.results.single()) + assertFalse(run.engine.output!!.exists()) + } + @Test fun cancellationDuringValidationWinsOverAcceptedOutput() { + lateinit var run: Run + run = Run(duringValidation = { run.attempt.cancel() }) + run.idle(); run.engine.complete(); run.idle() + assertEquals(SdrConversionResult.Cancelled, run.results.single()) + assertFalse(run.engine.output!!.exists()); run.assertInputSafe() + } + @Test fun wrongProfileOrAudioLayoutRejectsEvenAnAcceptedSharedVerdict() { + for (wrong in listOf(facts.copy(videoProfile = known(VideoProfile.H264High)), + facts.copy(audioTracks = known(listOf(AudioTrack(known(AudioFormat.Aac(AudioChannelLayout.Mono)))))))) { + val run = Run(StandardInputOutputValidation.Accepted(wrong, StandardInputPolicyEvaluator().evaluate(wrong, selection()))) + run.idle(); run.engine.complete(); run.idle() + assertEquals(SdrConversionResult.Failed(SdrConversionFailure.OutputInvalid), run.results.single()) + assertFalse(run.engine.output!!.exists()) + } + } + @Test fun failedDeletionRetainsExactCleanupAuthorityAndCannotSweepCustomerParent() { + val run = Run(); run.idle() + val path = run.engine.output!! + assertTrue(path.delete()); assertTrue(path.mkdir()) + val obstacle = File(path, "obstacle").apply { writeText("locked") } + run.attempt.cancel(); run.idle() + assertEquals(SdrConversionResult.Cancelled, run.results.single()) + assertEquals(path, run.attempt.pendingCleanup!!.file) + run.assertInputSafe() + assertTrue(obstacle.delete()); assertTrue(run.attempt.pendingCleanup!!.delete()) + } + @Test fun diskEstimateIsBoundedAndOwnershipCannotDeleteOtherFiles() { + assertNull(targets()!!.estimatedOutputBytes(Double.POSITIVE_INFINITY)) + assertNull(SdrGeneratedFile.allocate(context.cacheDir, Long.MAX_VALUE)) + val customer = File(context.cacheDir, "mux-upload/source.mp4").apply { parentFile!!.mkdirs(); writeText("customer") } + val generated = SdrGeneratedFile.allocate(context.cacheDir, 1)!! + assertEquals("standard-input", generated.file.parentFile!!.name) + assertTrue(generated.delete()); assertTrue(customer.exists()) + } +} From 69327dcccdaffc53ebdac23227ad3eca1b8e1cf0 Mon Sep 17 00:00:00 2001 From: Eugene Morozov Date: Wed, 7 Oct 2026 13:58:02 -0700 Subject: [PATCH 43/49] NAT-547: harden SDR conversion and add reproducible review coverage --- library/build.gradle | 5 + library/src/androidTest/README.md | 59 ++++++ .../SdrConversionFixtureTests.kt | 119 ++++++++++++ .../standardization/Media3SdrExportEngine.kt | 84 ++++++--- .../upload/internal/standardization/README.md | 12 +- .../standardization/SdrEncodingTargets.kt | 81 ++++++-- .../SdrConversionAdapterTests.kt | 66 ++++++- .../SdrMedia3ConfigurationTests.kt | 174 ++++++++++++++++++ 8 files changed, 548 insertions(+), 52 deletions(-) create mode 100644 library/src/androidTest/README.md create mode 100644 library/src/androidTest/java/com/mux/video/upload/internal/standardization/SdrConversionFixtureTests.kt create mode 100644 library/src/test/java/com/mux/video/upload/internal/standardization/SdrMedia3ConfigurationTests.kt diff --git a/library/build.gradle b/library/build.gradle index d7eff249..5446e470 100644 --- a/library/build.gradle +++ b/library/build.gradle @@ -35,6 +35,9 @@ android { buildFeatures { buildConfig true } + testOptions { + targetSdk = 36 + } } muxDistribution { @@ -98,4 +101,6 @@ dependencies { testImplementation 'androidx.test.ext:junit:1.2.1' testImplementation "io.mockk:mockk:1.13.9" testImplementation 'org.robolectric:robolectric:4.11.1' + androidTestImplementation 'androidx.test:runner:1.6.2' + androidTestImplementation 'androidx.test.ext:junit:1.2.1' } diff --git a/library/src/androidTest/README.md b/library/src/androidTest/README.md new file mode 100644 index 00000000..7d6f5e0b --- /dev/null +++ b/library/src/androidTest/README.md @@ -0,0 +1,59 @@ +# SDR conversion fixtures + +`SdrConversionFixtureTests` runs the real Media3 adapter and shared output validator. Media stays outside Git and the test APK. With no staged directory, the suite is skipped; a staged directory with a missing fixture fails. Use a dedicated device/emulator and specify its serial on every adb command. + +Generate the following fixtures with FFmpeg built with libx264, libx265, AAC, and libmp3lame. These recipes reproduce the scenario contracts, not byte-identical encodes across FFmpeg versions. Record the FFmpeg version, source SHA-256s, API/device/encoder, test output, and any captured generated media in the external evidence directory. The multi-audio first track is stereo, non-default and 3 seconds; the default secondary is mono and 6 seconds. HEVC success remains device-dependent and must pass reinspection. + +```sh +SDR_FIXTURES=/absolute/path/to/external/sdr-fixtures +mkdir -p "$SDR_FIXTURES" +ffmpeg -y -f lavfi -i testsrc2=size=1920x1080:rate=30:duration=3 \ + -vf setparams=color_trc=bt709:color_primaries=bt709:colorspace=bt709 \ + -c:v libx264 -profile:v baseline -pix_fmt yuv420p -bf 0 \ + -color_primaries bt709 -color_trc bt709 -colorspace bt709 "$SDR_FIXTURES/no-audio.mp4" +ffmpeg -y -i "$SDR_FIXTURES/no-audio.mp4" \ + -f lavfi -i sine=frequency=440:sample_rate=48000:duration=3 \ + -f lavfi -i sine=frequency=880:sample_rate=48000:duration=6 \ + -filter_complex '[1:a]aformat=channel_layouts=stereo[first]' \ + -map 0:v -map '[first]' -map 2:a -c:v copy -c:a aac \ + -disposition:a:0 0 -disposition:a:1 default "$SDR_FIXTURES/multi-audio.mp4" +ffmpeg -y -i "$SDR_FIXTURES/no-audio.mp4" -c copy \ + -metadata:s:v:0 rotate=90 "$SDR_FIXTURES/rotated.mp4" +ffmpeg -y -i "$SDR_FIXTURES/no-audio.mp4" \ + -f lavfi -i sine=frequency=440:sample_rate=48000:duration=3 \ + -map 0:v -map 1:a -c:v copy -c:a libmp3lame -ac 2 "$SDR_FIXTURES/mp3-stereo.mp4" +ffmpeg -y -f lavfi -i testsrc2=size=512x288:rate=240:duration=2 \ + -vf setparams=color_trc=bt709:color_primaries=bt709:colorspace=bt709 \ + -c:v libx264 -profile:v baseline -pix_fmt yuv420p -bf 0 \ + -color_primaries bt709 -color_trc bt709 -colorspace bt709 "$SDR_FIXTURES/high-rate.mp4" +ffmpeg -y -f lavfi -i testsrc2=size=512x288:rate=60:duration=30 \ + -vf "select='if(lt(t,7.5),1,not(mod(n,3)))',setparams=color_trc=bt709:color_primaries=bt709:colorspace=bt709" -fps_mode vfr \ + -c:v libx264 -profile:v baseline -pix_fmt yuv420p -bf 0 -g 10000 -keyint_min 10000 -sc_threshold 0 -crf 18 \ + -color_primaries bt709 -color_trc bt709 -colorspace bt709 "$SDR_FIXTURES/vfr.mp4" +ffmpeg -y -i "$SDR_FIXTURES/no-audio.mp4" -c:v libx264 -profile:v baseline -pix_fmt yuv420p -bf 0 \ + -vf setparams=color_trc=linear:color_primaries=bt709:colorspace=bt709 \ + -color_primaries bt709 -color_trc linear -colorspace bt709 "$SDR_FIXTURES/linear.mp4" +for SDR_DURATION in 3 12; do + if [ "$SDR_DURATION" = 3 ]; then SDR_HEVC_NAME=hevc-small-open-gop; else SDR_HEVC_NAME=hevc-long-open-gop; fi + ffmpeg -y -f lavfi -i "testsrc2=size=512x288:rate=30:duration=$SDR_DURATION" \ + -vf setparams=color_trc=bt709:color_primaries=bt709:colorspace=bt709 \ + -c:v libx265 -pix_fmt yuv420p -x265-params 'open-gop=1:keyint=30:min-keyint=30:scenecut=0:bframes=0' \ + -color_primaries bt709 -color_trc bt709 -colorspace bt709 "$SDR_FIXTURES/$SDR_HEVC_NAME.mp4" +done +``` + +Build/install the test APK, then stage fixtures. Choose the serial of your dedicated test device; the commands below do not select an arbitrary connected device. + +```sh +SDR_DEVICE=emulator-5556 +./gradlew :library:assembleDebugAndroidTest +adb -s "$SDR_DEVICE" install -r library/build/outputs/apk/androidTest/debug/library-debug-androidTest.apk +adb -s "$SDR_DEVICE" push "$SDR_FIXTURES/." /data/local/tmp/nat547-fixtures/ +adb -s "$SDR_DEVICE" shell run-as com.mux.video.upload.test mkdir -p files/sdr-fixtures +adb -s "$SDR_DEVICE" shell run-as com.mux.video.upload.test cp -r /data/local/tmp/nat547-fixtures/. files/sdr-fixtures/ +adb -s "$SDR_DEVICE" shell am instrument -w -r \ + -e class com.mux.video.upload.internal.standardization.SdrConversionFixtureTests \ + com.mux.video.upload.test/androidx.test.runner.AndroidJUnitRunner +``` + +The H.264 matrix requires accepted, independently reinspected output, including preserved VFR timestamps and first stereo audio selection. LINEAR must fail before export. HEVC may succeed or fail conservatively on an unproven device; a safe failure is not evidence of working HEVC conversion. The cancellation test checks source bytes, tracked output cleanup, and one terminal result. API 23 and physical-device HEVC, quality/performance, and Mux ingest remain separate acceptance gates. diff --git a/library/src/androidTest/java/com/mux/video/upload/internal/standardization/SdrConversionFixtureTests.kt b/library/src/androidTest/java/com/mux/video/upload/internal/standardization/SdrConversionFixtureTests.kt new file mode 100644 index 00000000..3c6426d4 --- /dev/null +++ b/library/src/androidTest/java/com/mux/video/upload/internal/standardization/SdrConversionFixtureTests.kt @@ -0,0 +1,119 @@ +package com.mux.video.upload.internal.standardization + +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import com.mux.video.upload.internal.InputStandardization +import com.mux.video.upload.internal.MaximumResolution +import org.junit.Assert.* +import org.junit.Assume.assumeTrue +import org.junit.Test +import org.junit.runner.RunWith +import java.io.File +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicInteger + +/** Stage external fixtures as documented in the adapter README; never package media in the APK. */ +@RunWith(AndroidJUnit4::class) +class SdrConversionFixtureTests { + private val instrumentation = InstrumentationRegistry.getInstrumentation() + private val context = instrumentation.targetContext + private val fixtures = File(instrumentation.context.filesDir, "sdr-fixtures") + + private fun source(name: String): File { + assumeTrue("Stage SDR fixtures before running this suite", fixtures.isDirectory) + val fixture = File(fixtures, name) + assertTrue("Missing fixture $name", fixture.isFile) + return File(context.cacheDir, "mux-upload/source-$name").apply { + parentFile!!.mkdirs(); fixture.copyTo(this, overwrite = true) + } + } + private fun inspect(file: File): Triple { + val metadata = (MediaMetadataInspector().inspect(file) as MetadataInspectionResult.Success).inspection + val samples = StandardInputTimelineInspector().inspect(file, metadata) + assertEquals(SampleScanStatus.Complete, samples.status) + // Direct adapter exercise: these test flags do not establish hardware capability proof. + val capabilities = PlanningCapabilities(true, true, setOf(VideoCodec.H264, VideoCodec.Hevc), PolicyRequirement.entries.toSet(), + canPrepareCompliantAacAudio = true) + val action = StandardInputPlanner().plan(samples.facts, + InputStandardization(true, MaximumResolution.Preset1280x720), capabilities).action + assertTrue("${file.name}: $action\n${samples.facts}\n$metadata", action is StandardInputAction.Convert) + return Triple(metadata, samples, (action as StandardInputAction.Convert).conversion) + } + private fun ownedFiles(): Set = File(context.cacheDir, "mux-upload/standard-input").listFiles() + ?.map { it.name }?.toSet() ?: emptySet() + + private fun export(name: String): SdrConversionResult { + val file = source(name) + val original = file.readBytes() + val (metadata, samples, conversion) = inspect(file) + val before = ownedFiles() + val done = CountDownLatch(1) + val calls = AtomicInteger() + var result: SdrConversionResult? = null + val attempt = SdrConversionAdapter(context).start(file, metadata, samples, conversion) { + result = it; calls.incrementAndGet(); done.countDown() + } + try { + assertTrue("Timed out $name", done.await(90, TimeUnit.SECONDS)) + File(instrumentation.context.filesDir, "sdr-evidence").apply { mkdirs() }.let { area -> + File(area, "$name.txt").writeText("$metadata\n${samples.facts}\n${samples.timeline}\n$conversion\n$result") + } + assertEquals(1, calls.get()) + assertArrayEquals(original, file.readBytes()) + if (result is SdrConversionResult.Completed) { + val completed = result as SdrConversionResult.Completed + assertEquals(MediaFact.Known(conversion.outputCodec), completed.facts.videoCodec) + assertEquals(MediaFact.Known(conversion.outputDimensions), completed.facts.encodedDimensions) + assertEquals(MediaFact.Known(0), completed.facts.rotationDegrees) + assertTrue(StandardInputOutputValidator().validateGeneratedOutput(completed.output.file, samples.facts, + samples.timeline, conversion) is StandardInputOutputValidation.Accepted) + assertTrue(completed.output.delete()) + } + assertEquals(before, ownedFiles()) + return checkNotNull(result) + } finally { + attempt.cancel(); (result as? SdrConversionResult.Completed)?.output?.delete(); file.delete() + } + } + @Test fun h264ResizeSilentPortraitNonAacAndCadenceExportsPassReinspection() { + for (name in listOf("multi-audio.mp4", "no-audio.mp4", "rotated.mp4", "mp3-stereo.mp4", "high-rate.mp4", "vfr.mp4")) { + val result = export(name) + assertTrue("$name: $result", result is SdrConversionResult.Completed) + if (name == "multi-audio.mp4") assertEquals(MediaFact.Known(listOf(AudioTrack(MediaFact.Known( + AudioFormat.Aac(AudioChannelLayout.Stereo))))), (result as SdrConversionResult.Completed).facts.audioTracks) + } + } + @Test fun hevcEitherPassesFullValidationOrRejectsSafelyWithoutChangingCodecFamily() { + for (name in listOf("hevc-small-open-gop.mp4", "hevc-long-open-gop.mp4")) { + val result = export(name) + assertTrue("$name: $result", result is SdrConversionResult.Completed || result in listOf( + SdrConversionResult.Failed(SdrConversionFailure.CapabilityUnavailable), + SdrConversionResult.Failed(SdrConversionFailure.Export), SdrConversionResult.Failed(SdrConversionFailure.OutputInvalid))) + } + } + @Test fun linearTaggedInputFallsBackBeforeExport() { + assertEquals(SdrConversionResult.Failed(SdrConversionFailure.UnsupportedPlan), export("linear.mp4")) + } + @Test fun cancellationRemovesOnlyOwnedPartialAndDoesNotPublishLateResults() { + val file = source("multi-audio.mp4") + val original = file.readBytes() + val (metadata, samples, conversion) = inspect(file) + val before = ownedFiles() + val done = CountDownLatch(1) + val calls = AtomicInteger() + var result: SdrConversionResult? = null + val attempt = SdrConversionAdapter(context).start(file, metadata, samples, conversion) { + result = it; calls.incrementAndGet(); done.countDown() + } + try { + attempt.cancel() + assertTrue(done.await(30, TimeUnit.SECONDS)) + assertEquals(SdrConversionResult.Cancelled, result) + Thread.sleep(500) + assertEquals(1, calls.get()) + assertEquals(before, ownedFiles()) + assertArrayEquals(original, file.readBytes()) + } finally { attempt.cancel(); file.delete() } + } +} diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/Media3SdrExportEngine.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/Media3SdrExportEngine.kt index 027ffade..ff606d0f 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/Media3SdrExportEngine.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/Media3SdrExportEngine.kt @@ -11,6 +11,7 @@ import android.util.Pair import androidx.media3.common.C import androidx.media3.common.Format import androidx.media3.common.MediaItem +import androidx.media3.common.MimeTypes import androidx.media3.common.util.Clock import androidx.media3.common.util.MediaFormatUtil import androidx.media3.common.util.UnstableApi @@ -51,7 +52,7 @@ internal class Media3SdrExportEngine( .setRemoveAudio(targets.audioChannels == null) .setEffects(Effects(emptyList(), listOf(Presentation.createForWidthAndHeight( conversion.outputDimensions.width, conversion.outputDimensions.height, Presentation.LAYOUT_SCALE_TO_FIT)))) - if (conversion.outputCadence == OutputCadence.ConstantFrameRate) item.setFrameRate(30) + if (conversion.outputCadence == OutputCadence.ConstantFrameRate) item.setFrameRate(conversion.outputFrameRate.toInt()) val items = listOf(item.build()) val sequence = if (targets.audioChannels == null) EditedMediaItemSequence.withVideoFrom(items) else EditedMediaItemSequence.withAudioAndVideoFrom(items) @@ -61,23 +62,11 @@ internal class Media3SdrExportEngine( .setLooper(looper) .setPortraitEncodingEnabled(true) .setVideoMimeType(targets.videoMime) - .setAudioMimeType("audio/mp4a-latm") + .setAudioMimeType(MimeTypes.AUDIO_AAC) .setAssetLoaderFactory(assetLoader) .setEncoderFactory(StrictSdrEncoderFactory(context, conversion, targets, capability)) - .addListener(object : Transformer.Listener { - override fun onCompleted(composition: Composition, exportResult: ExportResult) { - if (targets.audioChannels != null && (exportResult.channelCount != targets.audioChannels || - exportResult.sampleRate != targets.audioSampleRate)) failed(SdrConversionFailure.OutputInvalid) - else completed(exportResult.videoMimeType, exportResult.audioMimeType) - } - override fun onError(composition: Composition, exportResult: ExportResult, exportException: ExportException) { - failed(SdrConversionFailure.Export) - } - override fun onFallbackApplied(composition: Composition, originalTransformationRequest: TransformationRequest, - fallbackTransformationRequest: TransformationRequest) { - failed(SdrConversionFailure.ForbiddenFallback) - } - }).build() + // Retain Media3's default no-output-sample watchdog (10s; 25s on emulators). + .addListener(SdrExportListener(targets, completed, failed)).build() transformer = exporter exporter.start(composition, output.absolutePath) } @@ -89,6 +78,25 @@ internal class Media3SdrExportEngine( } } +@UnstableApi +internal class SdrExportListener( + private val targets: SdrEncodingTargets, private val completed: (String?, String?) -> Unit, + private val failed: (SdrConversionFailure) -> Unit, +) : Transformer.Listener { + override fun onCompleted(composition: Composition, exportResult: ExportResult) { + if (targets.audioChannels != null && (exportResult.channelCount != targets.audioChannels || + exportResult.sampleRate != targets.audioSampleRate)) failed(SdrConversionFailure.OutputInvalid) + else completed(exportResult.videoMimeType, exportResult.audioMimeType) + } + override fun onError(composition: Composition, exportResult: ExportResult, exportException: ExportException) { + failed(SdrConversionFailure.Export) + } + override fun onFallbackApplied(composition: Composition, originalTransformationRequest: TransformationRequest, + fallbackTransformationRequest: TransformationRequest) { + failed(SdrConversionFailure.ForbiddenFallback) + } +} + /** MP4 Format.id is the container track ID. Never choose a preferred/default secondary track. */ @UnstableApi internal class FirstAudioTrackSelector(context: Context, private val trackId: String?) : DefaultTrackSelector(context) { @@ -97,7 +105,10 @@ internal class FirstAudioTrackSelector(context: Context, private val trackId: St .setExceedAudioConstraintsIfNecessary(true).setExceedRendererCapabilitiesIfNecessary(true)) } override fun selectAudioTrack(mappedTrackInfo: MappedTrackInfo, rendererFormatSupports: Array>, - rendererMixedMimeTypeAdaptationSupports: IntArray, params: Parameters): Pair? { + rendererMixedMimeTypeAdaptationSupports: IntArray, params: Parameters): Pair? = + selectFirstAudioTrack(mappedTrackInfo) + + internal fun selectFirstAudioTrack(mappedTrackInfo: MappedTrackInfo): Pair? { if (trackId == null) return null for (renderer in 0 until mappedTrackInfo.rendererCount) { if (mappedTrackInfo.getRendererType(renderer) != C.TRACK_TYPE_AUDIO) continue @@ -132,23 +143,50 @@ internal class StrictSdrEncoderFactory( override fun audioNeedsEncoding() = targets.audioChannels != null && !targets.copyAac override fun createForAudioEncoding(format: Format, logSessionId: LogSessionId?): Codec { - check(format.sampleMimeType == "audio/mp4a-latm" && format.channelCount == targets.audioChannels && + check(format.sampleMimeType == MimeTypes.AUDIO_AAC && format.channelCount == targets.audioChannels && format.sampleRate == targets.audioSampleRate) return audioDelegate.createForAudioEncoding(format, logSessionId) } override fun createForVideoEncoding(format: Format, logSessionId: LogSessionId?): Codec { + val exact = requestedVideoFormat(format) + val media = SdrVideoEncoderConfiguration.create(exact, targets, capability.level) + return DefaultCodec(context, exact, media, capability.name, false, null) + } + + internal fun videoConfiguration(format: Format): MediaFormat = + SdrVideoEncoderConfiguration.create(requestedVideoFormat(format), targets, capability.level) + + private fun requestedVideoFormat(format: Format): Format { check(format.sampleMimeType == targets.videoMime && format.width == conversion.outputDimensions.width && format.height == conversion.outputDimensions.height && format.rotationDegrees == 0 && format.colorInfo?.colorTransfer == C.COLOR_TRANSFER_SDR) - val exact = format.buildUpon().setFrameRate(conversion.outputFrameRate.toFloat()).setAverageBitrate(targets.bitrate).build() - val media = MediaFormatUtil.createMediaFormatFromFormat(exact) + return format.buildUpon().setFrameRate(conversion.outputFrameRate.toFloat()).setAverageBitrate(targets.bitrate).build() + } +} + +/** Shared preflight/export keys; capability metadata remains only an advertised-format guard. */ +@UnstableApi +internal object SdrVideoEncoderConfiguration { + fun create(format: Format, targets: SdrEncodingTargets, level: Int, + sdk: Int = Build.VERSION.SDK_INT, device: String = Build.DEVICE, + soc: String = if (Build.VERSION.SDK_INT >= 31) Build.SOC_MODEL else ""): MediaFormat { + val media = MediaFormatUtil.createMediaFormatFromFormat(format) media.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface) media.setInteger(MediaFormat.KEY_BITRATE_MODE, MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_VBR) media.setInteger(MediaFormat.KEY_PROFILE, targets.platformProfile) - media.setInteger(MediaFormat.KEY_LEVEL, capability.level) + media.setInteger(MediaFormat.KEY_LEVEL, level) media.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, targets.keyframeIntervalSeconds) - if (Build.VERSION.SDK_INT >= 29) media.setInteger(MediaFormat.KEY_MAX_B_FRAMES, 0) - return DefaultCodec(context, exact, media, capability.name, false, null) + if (sdk >= 29) media.setInteger(MediaFormat.KEY_MAX_B_FRAMES, 0) + // Media3 1.11.1's empirical encoder performance settings, retaining its exceptions. + if (sdk >= 25) { + media.setInteger(MediaFormat.KEY_PRIORITY, 1) + val overflow = sdk in 31..34 && soc in setOf("SM8550", "SM7450", "SM6450", "SC9863A", "T612", "T606", "T603") + media.setInteger(MediaFormat.KEY_OPERATING_RATE, if (sdk == 26) 30 else if (overflow) 1000 else Int.MAX_VALUE) + } + // Redmi Note 9 Pro rejects high KEY_FRAME_RATE. This is a codec hint; timestamps + // and Media3's cadence remain planned. Lowering it only shortens the requested GOP. + if (sdk < 30 && device == "joyeuse" && format.frameRate > 30) media.setFloat(MediaFormat.KEY_FRAME_RATE, 30f) + return media } } diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/README.md b/library/src/main/java/com/mux/video/upload/internal/standardization/README.md index 5585b973..ad2d3b4c 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/README.md +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/README.md @@ -13,13 +13,13 @@ Each attempt allocates only its tracked unique `sdr-*.mp4` file under `cacheDir/ H.264 stays H.264, HEVC stays HEVC, and another proven SDR source uses H.264. Dimensions are the planner's exact display-oriented even dimensions, with orientation baked into pixels and no upscaling. Portrait encoding is explicitly enabled. Video always encodes, including an identity presentation effect, so bitrate/GOP remediation cannot silently remux the source. -The 8-bit 4:2:0 path requests H.264 Baseline (no B frames) or HEVC Main, VBR at 6 Mbps through a 2048-pixel long side or 15 Mbps above it, additionally capped to 75% of the plan's average-bitrate ceiling. GOP targets use 90% of the planned interval ceiling, rounded down to whole seconds, following Swift's headroom convention. Output validation remains the final bitrate, IDR/closed-GOP, profile, pixel, cadence, and timeline gate. +The 8-bit 4:2:0 path requests H.264 Baseline (no B frames) or HEVC Main, VBR at 6 Mbps through a 2048-pixel long side or 15 Mbps above it, additionally capped to 75% of the plan's average-bitrate ceiling. GOP targets use 90% of the planned interval ceiling, rounded down to whole seconds. For preserved VFR timestamps, the target is also reduced by the nominal rate times the longest measured presentation gap: encoders commonly convert the interval to a nominal-rate frame count. If a positive whole-second target cannot fit, preparation falls back before export. Output validation remains the final bitrate, IDR/closed-GOP, profile, pixel, cadence, and timeline gate. -Media3's default video factory overrides H.264 profile requests. The adapter therefore supplies Media3 `DefaultCodec` with the exact platform profile, level, dimensions, bitrate, nominal rate, surface input, and GOP interval. Transformer owns decoding, effects, encoding, and muxing. The AAC delegate disables both encoder and format fallback. Every `onFallbackApplied`, mismatched output MIME, or invalid generated file fails the attempt; a listed encoder never proves a usable path. +Media3's default video factory overrides H.264 profile requests. The adapter therefore supplies Media3 `DefaultCodec` with the exact platform profile, level, dimensions, bitrate, nominal rate, surface input, and GOP interval. Preflight and export share those configuration keys, including Media3 1.11.1's API/SoC performance settings and the Redmi Note 9 Pro high-rate hint workaround. Planned timestamps remain unchanged. Transformer owns decoding, effects, encoding, and muxing. Its default no-output-sample watchdog remains enabled (10 seconds normally, 25 seconds on emulators); timeout errors use the same terminal cleanup as other export failures. Native calls cannot be forcibly interrupted. The AAC delegate disables both encoder and format fallback. Every `onFallbackApplied`, mismatched output MIME, or invalid generated file fails the attempt; a listed encoder never proves a usable path. -The bounded advertised-format preflight examines at most 128 codecs and checks a 500 ms budget between platform calls. Native calls cannot be preempted. It checks source decode and exact output encode support before creating a file, and pins one supported video encoder. It does not turn advertised capabilities into the planner's device-path proof. +The bounded advertised-format preflight gives cold native enumeration a separate two-second cooperative limit and capability queries a 500 ms limit, checked between calls. These are conservative development guards, not measured hardware performance thresholds. Only matching capability queries count toward the 128-query cap; unrelated inventory entries cannot reject an otherwise supported device merely by exceeding 128 entries. It checks source decode and exact output encode support before creating a file, and pins one supported video encoder. It does not turn advertised capabilities into the planner's device-path proof. -Preserved cadence keeps source timestamps. Constant-rate reduction is limited to proven constant cadence with an integral source/30 fps ratio; a one-part-per-million allowance covers microsecond-rounded rate measurement. Media3 drops frames at 30 fps and the shared validator checks actual output cadence and duration. Frame duplication, fractional-rate resampling, 10-bit SDR encoding, and HDR/tone mapping have no proof in this adapter and fail conservatively. NAT-550 owns tone mapping. +Preserved cadence keeps source timestamps. Constant-rate reduction is limited to proven constant cadence with an integral source/30 fps ratio; a one-part-per-million allowance covers microsecond-rounded rate measurement. Media3 drops frames at the planned 30 fps and the shared validator checks actual output cadence and duration. LINEAR is correctly classified as SDR by inspection, but this adapter rejects its explicit transfer tags before export until color handling is proven. Frame duplication, fractional-rate resampling, 10-bit SDR encoding, and HDR/tone mapping have no proof in this adapter and fail conservatively. NAT-550 owns tone mapping. ## Audio decision @@ -29,8 +29,8 @@ The asset loader's selector matches the first audio track's container ID, rather ## Evidence and remaining limits -At implementation time, 210 unit tests pass in each debug/release variant; release SDK/sample builds and lint-vital pass. External API 36 emulator checks cover six accepted exports: H.264 resize, silent H.264, rotated portrait, MP3 stereo to AAC, short HEVC Main, and 240-to-30 fps H.264. Two HEVC failures exercise safe rejection: 720p exceeds the emulator's 512-pixel encoder limit; a longer small export emits CRA/open GOP and fails the existing validator. Cancellation removes generated bytes and preserves source copies. These are nine cases across three instrumentation tests, including the grouped export matrix. +Committed unit tests cover encoding targets, VFR interval targeting with the shared validator, strict encoder configuration, performance/workaround boundaries, cold/large codec inventories, first-track selection, Media3 timeout/fallback callbacks, and adapter ownership/cancellation. The committed device suite and [fixture recipes](../../../../../../../../androidTest/README.md) cover resize, silent H.264, rotated portrait, MP3 stereo to AAC, HEVC, 240-to-30 fps, preserved VFR, LINEAR fallback, and cancellation. Fixtures are staged outside the APK and remain external to product Git. Run these checks on each Media3 upgrade; unit configuration checks do not prove hardware behavior. The distinguishable multi-audio source has 3s video, 3s first stereo AAC (track ID 2, non-default), and 6s secondary mono AAC (ID 3, default). Output contains the first stereo track: its copied AAC bytes have the same SHA-256 as source audio ordinal zero and differ from the default secondary. This proves local selection only; Mux ingest parity remains pending. -Media binaries, fixture harness, output probes, and measurements stay outside product Git in `/private/tmp/nat547-sdr`. API 23-29 execution, physical hardware, long HEVC closed-GOP support, 10-bit SDR, HDR, high-resolution device coverage, visual quality/performance acceptance, and a live Mux upload are unverified. Emulator timings do not establish physical-device performance limits. NAT-548 can use the validated SDR path for the early upload journey; no predecessor merge is required to continue the stack. +API 36 emulator evidence includes a short accepted HEVC export and a longer export rejected for CRA/open GOP; the software encoder also cannot encode 720p. These observations do not establish a universal HEVC limitation or justify changing codec families. API 23-29 execution, physical hardware, long HEVC closed-GOP support, 10-bit SDR, HDR, high-resolution device coverage, visual quality/performance acceptance, and a live Mux upload remain verification gates. Emulator timings do not establish physical-device performance limits. NAT-548 can use the validated SDR path for the early upload journey; no predecessor merge is required to continue the stack. diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/SdrEncodingTargets.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/SdrEncodingTargets.kt index 4853a32a..9ce7d088 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/SdrEncodingTargets.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/SdrEncodingTargets.kt @@ -4,8 +4,11 @@ import android.media.MediaCodecInfo import android.media.MediaCodecInfo.CodecProfileLevel import android.media.MediaCodecList import android.media.MediaFormat +import androidx.media3.common.Format +import androidx.media3.common.MimeTypes import kotlin.math.abs import kotlin.math.ceil +import kotlin.math.floor internal enum class SdrConversionFailure { UnsupportedPlan, InsufficientSourceEvidence, CapabilityUnavailable, DiskSpace, @@ -34,6 +37,11 @@ internal data class SdrEncodingTargets( fun from(conversion: StandardInputConversion, metadata: MediaMetadataInspection, source: MediaSampleInspection): SdrEncodingTargets? { val facts = source.facts + // LINEAR is SDR, but this surface path has no proof of its transfer handling. + if (metadata.tracks.any { it.video?.let { video -> + video.platformColor.transfer.valueOrNull == MediaFormat.COLOR_TRANSFER_LINEAR || + video.configuration.color.transfer.valueOrNull == MediaFormat.COLOR_TRANSFER_LINEAR + } == true }) return null if (source.status != SampleScanStatus.Complete || conversion.toneMapsToSdr || conversion.sourceDynamicRange != DynamicRange.Sdr || facts.dynamicRange != MediaFact.Known(DynamicRange.Sdr) || facts.videoCodec != MediaFact.Known(conversion.sourceCodec) || facts.videoTrackCount != MediaFact.Known(1) || @@ -73,13 +81,29 @@ internal data class SdrEncodingTargets( AudioChannelLayout.FivePointOne) else channels !in 1..2))) return null val interval = conversion.outputPolicyLimits.maximumKeyframeIntervals[expectedCodec] ?: return null if (!interval.isFinite() || interval < 1) return null + var gopSeconds = floor(interval * 0.9).toInt() + if (conversion.outputCadence == OutputCadence.PreserveSourceTimestamps && + facts.cadence == MediaFact.Known(Cadence.Variable)) { + val times = source.timeline.videoPresentationSeconds.valueOrNull!! + if (times.size < 2 || times.any { !it.isFinite() }) return null + var longestGap = 0.0 + for (index in 1 until times.size) { + val gap = times[index] - times[index - 1] + if (gap <= 0) return null + longestGap = maxOf(longestGap, gap) + } + // Encoders commonly turn the interval into nominal-rate frames. Bound their + // elapsed span using the slowest observed gap; validation still proves output. + gopSeconds = minOf(gopSeconds, floor(interval * 0.9 / (rate * longestGap)).toInt()) + if (gopSeconds < 1) return null + } return SdrEncodingTargets( - if (expectedCodec == VideoCodec.Hevc) "video/hevc" else "video/avc", + if (expectedCodec == VideoCodec.Hevc) MimeTypes.VIDEO_H265 else MimeTypes.VIDEO_H264, if (expectedCodec == VideoCodec.Hevc) VideoProfile.HevcMain else VideoProfile.H264Baseline, if (expectedCodec == VideoCodec.Hevc) CodecProfileLevel.HEVCProfileMain else CodecProfileLevel.AVCProfileBaseline, minOf(6_000_000L.takeIf { size.longSide <= 2048 } ?: 15_000_000L, conversion.outputPolicyLimits.maximumAverageBitrate * 3 / 4).toInt().takeIf { it > 0 } ?: return null, - maxOf(1, (interval * 0.9).toInt()), channels, sampleRate, copy, first?.sourceTrackId?.valueOrNull?.toString(), + maxOf(1, gopSeconds), channels, sampleRate, copy, first?.sourceTrackId?.valueOrNull?.toString(), ) } } @@ -88,19 +112,34 @@ internal data class SdrEncodingTargets( internal data class SdrEncoderCapability(val name: String, val level: Int) /** Bounded advertised-format guard. This does not manufacture planner/device-path proof. */ -internal class SdrCapabilityPreflight { +internal class SdrCapabilityPreflight( + private val codecInfos: () -> Array = { MediaCodecList(MediaCodecList.REGULAR_CODECS).codecInfos }, + private val nanoTime: () -> Long = System::nanoTime, +) { fun videoEncoder(conversion: StandardInputConversion, metadata: MediaMetadataInspection, targets: SdrEncodingTargets, isCancelled: () -> Boolean): SdrEncoderCapability? { - val started = System.nanoTime() - fun withinBudget() = !isCancelled() && System.nanoTime() - started < 500_000_000L - val infos = MediaCodecList(MediaCodecList.REGULAR_CODECS).codecInfos - if (infos.size > 128 || !withinBudget()) return null + val enumerationStarted = nanoTime() + if (isCancelled()) return null + val infos = codecInfos() + // Cold native enumeration has its own cooperative bound, not the query budget. + if (isCancelled() || nanoTime() - enumerationStarted >= 2_000_000_000L) return null + val started = nanoTime() + var queries = 0 + fun withinBudget() = !isCancelled() && nanoTime() - started < 500_000_000L + fun supports(info: MediaCodecInfo, mime: String, format: MediaFormat): Boolean { + if (!withinBudget() || queries >= 128) return false + queries++ + return runCatching { info.getCapabilitiesForType(mime).isFormatSupported(format) }.getOrDefault(false) + } val video = metadata.tracks.singleOrNull { it.kind == TrackKind.Video } ?: return null val sourceMime = video.mimeType.valueOrNull ?: return null val sourceSize = video.video?.encodedDimensions?.valueOrNull ?: return null - fun supportsDecode(mime: String, format: MediaFormat): Boolean = infos.any { info -> - withinBudget() && !info.isEncoder && info.supportedTypes.any { it.equals(mime, true) } && - runCatching { info.getCapabilitiesForType(mime).isFormatSupported(format) }.getOrDefault(false) + fun supportsDecode(mime: String, format: MediaFormat): Boolean { + for (info in infos) { + if (!withinBudget()) return false + if (!info.isEncoder && info.supportedTypes.any { it.equals(mime, true) } && supports(info, mime, format)) return true + } + return false } val sourceFormat = MediaFormat.createVideoFormat(sourceMime, sourceSize.width, sourceSize.height) val sourceProfile = when (video.video.profile.valueOrNull) { @@ -116,26 +155,28 @@ internal class SdrCapabilityPreflight { if (targets.audioChannels != null && !targets.copyAac) { val audioMime = metadata.tracks.first { it.sourceTrackId.valueOrNull?.toString() == targets.firstAudioTrackId }.mimeType.valueOrNull ?: return null if (!supportsDecode(audioMime, MediaFormat.createAudioFormat(audioMime, targets.audioSampleRate!!, targets.audioChannels))) return null - val aac = MediaFormat.createAudioFormat("audio/mp4a-latm", targets.audioSampleRate, targets.audioChannels) + val aac = MediaFormat.createAudioFormat(MimeTypes.AUDIO_AAC, targets.audioSampleRate, targets.audioChannels) aac.setInteger(MediaFormat.KEY_BIT_RATE, 160_000) aac.setInteger(MediaFormat.KEY_AAC_PROFILE, CodecProfileLevel.AACObjectLC) - if (infos.none { withinBudget() && it.isEncoder && it.supportedTypes.contains("audio/mp4a-latm") && - runCatching { it.getCapabilitiesForType("audio/mp4a-latm").isFormatSupported(aac) }.getOrDefault(false) }) return null + if (infos.none { withinBudget() && it.isEncoder && it.supportedTypes.contains(MimeTypes.AUDIO_AAC) && + supports(it, MimeTypes.AUDIO_AAC, aac) }) return null } for (info in infos) { if (!withinBudget()) return null if (!info.isEncoder || !info.supportedTypes.contains(targets.videoMime)) continue + if (queries >= 128) return null + queries++ val candidate = runCatching { val caps = info.getCapabilitiesForType(targets.videoMime) val level = caps.profileLevels.filter { it.profile == targets.platformProfile }.maxOfOrNull { it.level } ?: return@runCatching null val size = conversion.outputDimensions - val format = MediaFormat.createVideoFormat(targets.videoMime, size.width, size.height) - format.setFloat(MediaFormat.KEY_FRAME_RATE, conversion.outputFrameRate.toFloat()) - format.setInteger(MediaFormat.KEY_BIT_RATE, targets.bitrate) - format.setInteger(MediaFormat.KEY_PROFILE, targets.platformProfile) - format.setInteger(MediaFormat.KEY_LEVEL, level) - format.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface) - format.setInteger(MediaFormat.KEY_BITRATE_MODE, MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_VBR) + // A device workaround may lower KEY_FRAME_RATE as a configure-time hint. + // Still guard the actual planned size/rate against advertised video limits. + if (caps.videoCapabilities?.areSizeAndRateSupported(size.width, size.height, conversion.outputFrameRate) != true) + return@runCatching null + val requested = Format.Builder().setSampleMimeType(targets.videoMime).setWidth(size.width).setHeight(size.height) + .setFrameRate(conversion.outputFrameRate.toFloat()).setAverageBitrate(targets.bitrate).build() + val format = SdrVideoEncoderConfiguration.create(requested, targets, level) if (caps.isFormatSupported(format)) SdrEncoderCapability(info.name, level) else null }.getOrNull() if (candidate != null && withinBudget()) return candidate diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/SdrConversionAdapterTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/SdrConversionAdapterTests.kt index c6a0d507..29630eff 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/SdrConversionAdapterTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/SdrConversionAdapterTests.kt @@ -1,6 +1,12 @@ package com.mux.video.upload.internal.standardization import android.content.Context +import android.media.MediaFormat +import androidx.media3.common.MediaItem +import androidx.media3.transformer.Composition +import androidx.media3.transformer.EditedMediaItem +import androidx.media3.transformer.EditedMediaItemSequence +import androidx.media3.transformer.ExportResult import androidx.test.core.app.ApplicationProvider import org.junit.Assert.* import org.junit.Test @@ -31,6 +37,39 @@ class SdrConversionAdapterTests { private fun targets(plan: StandardInputConversion = conversion, input: MediaFacts = facts, tracks: List = listOf(audio())) = SdrEncodingTargets.from(plan, metadata(tracks), samples(input)) + @Test fun vfrGopTargetUsesSlowestPresentationGapAndValidatorStillChecksActualOutput() { + val times = (0..900).map { if (it <= 450) it / 60.0 else 7.5 + (it - 450) / 20.0 } + val variable = facts.copy(cadence = known(Cadence.Variable)) + val time = timeline.copy(durationSeconds = known(30.0), videoDurationSeconds = known(30.0), + firstAudioDurationSeconds = known(30.0), videoPresentationSeconds = known(times)) + val source = samples(variable).copy(timeline = time) + val target = SdrEncodingTargets.from(conversion, metadata(), source)!! + assertTrue(target.keyframeIntervalSeconds in 1..12) + // Model an encoder that converts seconds to a frame count at the nominal 30 fps. + fun exportedGap(seconds: Int): Double { + val frames = seconds * 30 + return times.indices.filter { it + frames < times.size }.maxOf { times[it + frames] - times[it] } + } + fun validate(seconds: Int) = StandardInputOutputValidator().validateFacts(variable.copy( + maximumKeyframeIntervalSeconds = known(exportedGap(seconds)), encodedDimensions = known(conversion.outputDimensions), + rotationDegrees = known(0)), variable, time, time, conversion) + assertTrue(validate(18) is StandardInputOutputValidation.Rejected) + assertTrue(validate(target.keyframeIntervalSeconds) is StandardInputOutputValidation.Accepted) + } + + @Test fun linearTaggedSdrIsRejectedBeforeAllocatingOrStartingExport() { + val format = MediaFormat.createVideoFormat("video/avc", 1920, 1080).apply { + setInteger(MediaFormat.KEY_COLOR_TRANSFER, MediaFormat.COLOR_TRANSFER_LINEAR) + } + val video = MediaTrackMetadataReader.read(0, format) + val inspection = metadata().copy(tracks = listOf(video, audio())) + assertEquals(known(DynamicRange.Sdr), video.video!!.dynamicRange) + assertNull(SdrEncodingTargets.from(conversion, inspection, samples())) + val run = Run(inputMetadata = inspection); run.idle() + assertEquals(SdrConversionResult.Failed(SdrConversionFailure.UnsupportedPlan), run.results.single()) + assertEquals(0, run.engine.starts); run.assertInputSafe() + } + @Test fun pinsCodecFamilyAndTargetsBelowPublishedCeilings() { assertEquals("video/avc", targets()!!.videoMime) assertEquals(6_000_000, targets()!!.bitrate) @@ -66,7 +105,8 @@ class SdrConversionAdapterTests { assertNull(targets(tracks = listOf(audio().copy(sourceTrackId = MediaFact.Unknown)))) } @Test fun permitsOnlyProvenIntegralCadenceReductionAndKeepsVfrTimestamps() { - assertNotNull(targets(input = facts.copy(cadence = known(Cadence.Variable)))) + assertNotNull(SdrEncodingTargets.from(conversion, metadata(), samples(facts.copy(cadence = known(Cadence.Variable))) + .copy(timeline = timeline.copy(videoPresentationSeconds = known(listOf(0.0, 0.03, 0.08)))))) val cfr = conversion.copy(outputCadence = OutputCadence.ConstantFrameRate) assertNotNull(targets(cfr, facts.copy(frameRate = known(240.0)))) assertNotNull(targets(cfr, facts.copy(frameRate = known(240.00004008351402)))) @@ -93,6 +133,7 @@ class SdrConversionAdapterTests { StandardInputPolicyEvaluator().evaluate(facts, selection())), capability: SdrEncoderCapability? = SdrEncoderCapability("fake", 1), duringValidation: (() -> Unit)? = null, + inputMetadata: MediaMetadataInspection = metadata(), ) { val sourceFile = File.createTempFile("customer-", ".mp4", context.cacheDir).apply { writeText("original") } val unrelated = File(context.cacheDir, "mux-upload/customer-copy.mp4").apply { parentFile!!.mkdirs(); writeText("customer") } @@ -100,8 +141,15 @@ class SdrConversionAdapterTests { val results = mutableListOf() val adapter = SdrConversionAdapter(context, preflight = { _, _, _, _ -> capability }, validate = { _, _, _, _, _ -> duringValidation?.invoke(); validation }, createEngine = { _, _, _, _, _ -> engine }) - val attempt = adapter.start(sourceFile, metadata(), samples(), conversion, results::add) - fun idle() { shadowOf(attempt.looper).idle() } + val attempt = adapter.start(sourceFile, inputMetadata, samples(), conversion, results::add) + fun idle() { + if (results.isNotEmpty()) return + try { shadowOf(attempt.looper).idle() } + catch (error: IllegalStateException) { + // PAUSED-mode idle posts a barrier; terminal cleanup can quit before that post. + if (results.isEmpty() || error.message?.contains("handler thread dead") != true) throw error + } + } fun assertInputSafe() { assertEquals("original", sourceFile.readText()); assertEquals("customer", unrelated.readText()) } } @Test fun successRetainsVerifiedFileUntilOwnerDeletesItAndIgnoresLateEvents() { @@ -119,6 +167,18 @@ class SdrConversionAdapterTests { assertFalse(run.engine.output!!.exists()); run.assertInputSafe() run.engine.complete(); assertEquals(1, run.results.size) } + @Test fun media3MuxerWatchdogErrorUsesTerminalCleanupAndSuppressesLateCompletion() { + val run = Run(); run.idle() + val composition = Composition.Builder(EditedMediaItemSequence.withVideoFrom(listOf( + EditedMediaItem.Builder(MediaItem.fromUri("file:///test.mp4")).build()))).build() + SdrExportListener(targets()!!, run.engine.completions!!, run.engine.failures!!) + .onError(composition, ExportResult.Builder().build(), muxerTimeoutException()) + run.idle() + assertEquals(listOf(SdrConversionResult.Failed(SdrConversionFailure.Export)), run.results) + assertFalse(run.engine.output!!.exists()); run.assertInputSafe() + run.engine.complete(); run.attempt.cancel() + assertEquals(1, run.results.size); assertEquals(1, run.engine.cancellations) + } @Test fun cancelTerminatesWithoutMedia3CallbackAndSuppressesQueuedCompletion() { val run = Run(); run.idle(); run.engine.complete(); run.attempt.cancel(); run.idle() assertEquals(listOf(SdrConversionResult.Cancelled), run.results) diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/SdrMedia3ConfigurationTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/SdrMedia3ConfigurationTests.kt new file mode 100644 index 00000000..fed2acee --- /dev/null +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/SdrMedia3ConfigurationTests.kt @@ -0,0 +1,174 @@ +package com.mux.video.upload.internal.standardization + +import android.content.Context +import android.media.MediaCodecInfo +import android.media.MediaFormat +import androidx.media3.common.C +import androidx.media3.common.ColorInfo +import androidx.media3.common.Format +import androidx.media3.common.MediaItem +import androidx.media3.common.TrackGroup +import androidx.media3.exoplayer.source.TrackGroupArray +import androidx.media3.exoplayer.trackselection.MappingTrackSelector +import androidx.media3.transformer.Composition +import androidx.media3.transformer.EditedMediaItem +import androidx.media3.transformer.EditedMediaItemSequence +import androidx.media3.transformer.ExportException +import androidx.media3.transformer.ExportResult +import androidx.media3.transformer.TransformationRequest +import androidx.test.core.app.ApplicationProvider +import io.mockk.* +import org.junit.Assert.* +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.annotation.Config +import org.robolectric.annotation.Implementation +import org.robolectric.annotation.Implements +import org.robolectric.shadows.MediaCodecInfoBuilder + +@RunWith(RobolectricTestRunner::class) +@Config(manifest = Config.NONE, sdk = [28], shadows = [SupportedFormatShadow::class]) +class SdrMedia3ConfigurationTests { + private val context = ApplicationProvider.getApplicationContext() + private val facts = compliantFacts().copy(cadence = known(Cadence.Constant)) + private val conversion = (StandardInputPlanner().plan(facts.copy(averageBitrate = known(9_000_000L)), + options(), fullCapabilities()).action as StandardInputAction.Convert).conversion + private val targets = SdrEncodingTargets("video/avc", VideoProfile.H264Baseline, + MediaCodecInfo.CodecProfileLevel.AVCProfileBaseline, 6_000_000, 18, null, null, false, null) + private val format = Format.Builder().setSampleMimeType("video/avc").setWidth(conversion.outputDimensions.width) + .setHeight(conversion.outputDimensions.height).setFrameRate(30f).setAverageBitrate(targets.bitrate) + .setColorInfo(ColorInfo.SDR_BT709_LIMITED).build() + private val video = MediaTrackMetadataReader.read(0, MediaFormat.createVideoFormat("video/avc", 1920, 1080)) + private val metadata = MediaMetadataInspection(known(ContainerKind.IsoBaseMedia), listOf(video), MediaFact.Unknown, facts, 0) + + @Test fun strictFactoryRejectsMimeDimensionsRotationAndLinearBeforeCodecCreation() { + val factory = StrictSdrEncoderFactory(context, conversion, targets, SdrEncoderCapability("chosen", 4)) + val configuration = factory.videoConfiguration(format) + assertEquals(targets.platformProfile, configuration.getInteger(MediaFormat.KEY_PROFILE)) + assertEquals(4, configuration.getInteger(MediaFormat.KEY_LEVEL)) + assertEquals(18, configuration.getInteger(MediaFormat.KEY_I_FRAME_INTERVAL)) + assertEquals(1, configuration.getInteger(MediaFormat.KEY_PRIORITY)) + for (bad in listOf(format.buildUpon().setSampleMimeType("video/hevc").build(), + format.buildUpon().setWidth(640).build(), format.buildUpon().setRotationDegrees(90).build(), + format.buildUpon().setColorInfo(ColorInfo.SDR_BT709_LIMITED.buildUpon().setColorTransfer(C.COLOR_TRANSFER_LINEAR).build()).build())) { + assertThrows(IllegalStateException::class.java) { factory.videoConfiguration(bad) } + } + } + + @Test fun performanceSettingsRetainApiAndChipsetExceptionsWithoutChangingProfile() { + val old = SdrVideoEncoderConfiguration.create(format, targets, 4, sdk = 23) + assertFalse(old.containsKey(MediaFormat.KEY_PRIORITY)); assertFalse(old.containsKey(MediaFormat.KEY_OPERATING_RATE)) + assertEquals(30, SdrVideoEncoderConfiguration.create(format, targets, 4, sdk = 26).getInteger(MediaFormat.KEY_OPERATING_RATE)) + val affected = SdrVideoEncoderConfiguration.create(format, targets, 4, sdk = 31, soc = "SM8550") + assertEquals(1000, affected.getInteger(MediaFormat.KEY_OPERATING_RATE)) + assertEquals(targets.platformProfile, affected.getInteger(MediaFormat.KEY_PROFILE)) + assertEquals(0, affected.getInteger(MediaFormat.KEY_MAX_B_FRAMES)) + assertEquals(Int.MAX_VALUE, SdrVideoEncoderConfiguration.create(format, targets, 4, sdk = 35, soc = "SM8550") + .getInteger(MediaFormat.KEY_OPERATING_RATE)) + } + + @Test fun redmiFrameRateWorkaroundDoesNotIncreaseLowRateGopOrResampleTimestamps() { + val high = format.buildUpon().setFrameRate(120f).build() + assertEquals(30f, SdrVideoEncoderConfiguration.create(high, targets, 4, sdk = 29, device = "joyeuse") + .getFloat(MediaFormat.KEY_FRAME_RATE), 0f) + val low = format.buildUpon().setFrameRate(5f).build() + assertEquals(5f, SdrVideoEncoderConfiguration.create(low, targets, 4, sdk = 29, device = "joyeuse") + .getFloat(MediaFormat.KEY_FRAME_RATE), 0f) + assertEquals(120f, high.frameRate, 0f) + } + + private fun codec(encoder: Boolean, mime: String): MediaCodecInfo { + val capabilityFormat = if (mime.startsWith("video/")) MediaFormat.createVideoFormat(mime, 1920, 1080) + else MediaFormat.createAudioFormat(mime, 48_000, 2) + val capabilities = MediaCodecInfoBuilder.CodecCapabilitiesBuilder.newBuilder().setMediaFormat(capabilityFormat) + .setIsEncoder(encoder).apply { + if (mime.startsWith("video/")) setColorFormats(intArrayOf(MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface)) + } + .setProfileLevels(arrayOf(MediaCodecInfo.CodecProfileLevel().apply { + profile = targets.platformProfile; level = MediaCodecInfo.CodecProfileLevel.AVCLevel51 + })).build() + return MediaCodecInfoBuilder.newBuilder().setName(if (encoder) "chosen" else "decoder") + .setIsEncoder(encoder).setCapabilities(capabilities).build() + } + + @Test fun coldEnumerationHasSeparateBoundAndUnrelatedCodecsDoNotExhaustQueries() { + var now = 0L + val unrelated = codec(false, "audio/mpeg") + val encoder = codec(true, "video/avc") + val decoder = codec(false, "video/avc") + val infos = Array(200) { unrelated } + arrayOf(decoder, encoder) + val queried = mutableListOf() + SupportedFormatShadow.supported = { queried.add(it); true } + val preflight = SdrCapabilityPreflight(codecInfos = { now += 800_000_000L; infos }, nanoTime = { now }) + assertEquals(SdrEncoderCapability("chosen", MediaCodecInfo.CodecProfileLevel.AVCLevel51), + preflight.videoEncoder(conversion, metadata, targets) { false }) + assertEquals(2, queried.size) + assertEquals(18, queried.last().getInteger(MediaFormat.KEY_I_FRAME_INTERVAL)) + assertEquals(1, queried.last().getInteger(MediaFormat.KEY_PRIORITY)) + } + + @Test fun preflightStillBoundsColdInitializationQueriesAndCancellation() { + var now = 0L + val encoder = codec(true, "video/avc") + val slowDecoder = codec(false, "video/avc") + var queries = 0 + SupportedFormatShadow.supported = { queries++; now += 500_000_000L; true } + val infos = arrayOf(slowDecoder, encoder) + assertNull(SdrCapabilityPreflight({ now += 2_000_000_000L; infos }, { now }) + .videoEncoder(conversion, metadata, targets) { false }) + now = 0 + assertNull(SdrCapabilityPreflight({ infos }, { now }).videoEncoder(conversion, metadata, targets) { false }) + assertEquals(1, queries) + var enumerated = false + assertNull(SdrCapabilityPreflight({ enumerated = true; infos }, { 0L }).videoEncoder(conversion, metadata, targets) { true }) + assertFalse(enumerated) + val rejected = codec(false, "video/avc") + queries = 0 + SupportedFormatShadow.supported = { queries++; false } + assertNull(SdrCapabilityPreflight({ Array(200) { rejected } }, { 0L }).videoEncoder(conversion, metadata, targets) { false }) + assertEquals(128, queries) + } + + @Test fun firstTrackSelectorIgnoresPreferredSecondaryAndFailsOnMissingIdentity() { + val first = Format.Builder().setId("2").setSampleMimeType("audio/mp4a-latm").setChannelCount(2).build() + val secondary = first.buildUpon().setId("3").setSelectionFlags(C.SELECTION_FLAG_DEFAULT).setChannelCount(1).build() + val groups = TrackGroupArray(TrackGroup("secondary", secondary), TrackGroup("first", first)) + val mapped = mockk { + every { rendererCount } returns 1 + every { getRendererType(0) } returns C.TRACK_TYPE_AUDIO + every { getTrackGroups(0) } returns groups + } + val selection = FirstAudioTrackSelector(context, "2").selectFirstAudioTrack(mapped)!! + assertEquals("2", selection.first.group.getFormat(selection.first.tracks.single()).id) + assertNull(FirstAudioTrackSelector(context, null).selectFirstAudioTrack(mapped)) + assertThrows(IllegalStateException::class.java) { FirstAudioTrackSelector(context, "missing").selectFirstAudioTrack(mapped) } + } + + @Test fun exportListenerRoutesMuxerWatchdogAndFallbackToFailureAndChecksCopiedAudio() { + val composition = Composition.Builder(EditedMediaItemSequence.withVideoFrom(listOf( + EditedMediaItem.Builder(MediaItem.fromUri("file:///test.mp4")).build()))).build() + val failures = mutableListOf() + var completed = false + val listener = SdrExportListener(targets.copy(audioChannels = 2, audioSampleRate = 48_000, copyAac = true), + { _, _ -> completed = true }, failures::add) + listener.onError(composition, ExportResult.Builder().build(), muxerTimeoutException()) + listener.onFallbackApplied(composition, TransformationRequest.Builder().build(), TransformationRequest.Builder().build()) + listener.onCompleted(composition, ExportResult.Builder().setChannelCount(1).setSampleRate(48_000).build()) + assertEquals(listOf(SdrConversionFailure.Export, SdrConversionFailure.ForbiddenFallback, SdrConversionFailure.OutputInvalid), failures) + assertFalse(completed) + listener.onCompleted(composition, ExportResult.Builder().setChannelCount(2).setSampleRate(48_000).build()) + assertTrue(completed) + } +} + +internal fun muxerTimeoutException(): ExportException = ExportException::class.java + .getDeclaredMethod("createForMuxer", Throwable::class.java, Int::class.javaPrimitiveType) + .apply { isAccessible = true }.invoke(null, IllegalStateException("No samples"), ExportException.ERROR_CODE_MUXING_TIMEOUT) as ExportException + +/** Only advertised format answers are faked; codec roles, MIME and profiles use Android objects. */ +@Implements(MediaCodecInfo.CodecCapabilities::class) +class SupportedFormatShadow { + @Implementation fun isFormatSupported(format: MediaFormat): Boolean = supported(format) + companion object { var supported: (MediaFormat) -> Boolean = { true } } +} From d5d1bb8262987a3e2fbfc1c93d15431616d9a6b1 Mon Sep 17 00:00:00 2001 From: Eugene Morozov Date: Wed, 7 Oct 2026 14:05:18 -0700 Subject: [PATCH 44/49] NAT-547: remove internal README notes from product changes --- library/src/androidTest/README.md | 59 ------------------- .../SdrConversionFixtureTests.kt | 2 +- .../upload/internal/standardization/README.md | 36 ----------- 3 files changed, 1 insertion(+), 96 deletions(-) delete mode 100644 library/src/androidTest/README.md delete mode 100644 library/src/main/java/com/mux/video/upload/internal/standardization/README.md diff --git a/library/src/androidTest/README.md b/library/src/androidTest/README.md deleted file mode 100644 index 7d6f5e0b..00000000 --- a/library/src/androidTest/README.md +++ /dev/null @@ -1,59 +0,0 @@ -# SDR conversion fixtures - -`SdrConversionFixtureTests` runs the real Media3 adapter and shared output validator. Media stays outside Git and the test APK. With no staged directory, the suite is skipped; a staged directory with a missing fixture fails. Use a dedicated device/emulator and specify its serial on every adb command. - -Generate the following fixtures with FFmpeg built with libx264, libx265, AAC, and libmp3lame. These recipes reproduce the scenario contracts, not byte-identical encodes across FFmpeg versions. Record the FFmpeg version, source SHA-256s, API/device/encoder, test output, and any captured generated media in the external evidence directory. The multi-audio first track is stereo, non-default and 3 seconds; the default secondary is mono and 6 seconds. HEVC success remains device-dependent and must pass reinspection. - -```sh -SDR_FIXTURES=/absolute/path/to/external/sdr-fixtures -mkdir -p "$SDR_FIXTURES" -ffmpeg -y -f lavfi -i testsrc2=size=1920x1080:rate=30:duration=3 \ - -vf setparams=color_trc=bt709:color_primaries=bt709:colorspace=bt709 \ - -c:v libx264 -profile:v baseline -pix_fmt yuv420p -bf 0 \ - -color_primaries bt709 -color_trc bt709 -colorspace bt709 "$SDR_FIXTURES/no-audio.mp4" -ffmpeg -y -i "$SDR_FIXTURES/no-audio.mp4" \ - -f lavfi -i sine=frequency=440:sample_rate=48000:duration=3 \ - -f lavfi -i sine=frequency=880:sample_rate=48000:duration=6 \ - -filter_complex '[1:a]aformat=channel_layouts=stereo[first]' \ - -map 0:v -map '[first]' -map 2:a -c:v copy -c:a aac \ - -disposition:a:0 0 -disposition:a:1 default "$SDR_FIXTURES/multi-audio.mp4" -ffmpeg -y -i "$SDR_FIXTURES/no-audio.mp4" -c copy \ - -metadata:s:v:0 rotate=90 "$SDR_FIXTURES/rotated.mp4" -ffmpeg -y -i "$SDR_FIXTURES/no-audio.mp4" \ - -f lavfi -i sine=frequency=440:sample_rate=48000:duration=3 \ - -map 0:v -map 1:a -c:v copy -c:a libmp3lame -ac 2 "$SDR_FIXTURES/mp3-stereo.mp4" -ffmpeg -y -f lavfi -i testsrc2=size=512x288:rate=240:duration=2 \ - -vf setparams=color_trc=bt709:color_primaries=bt709:colorspace=bt709 \ - -c:v libx264 -profile:v baseline -pix_fmt yuv420p -bf 0 \ - -color_primaries bt709 -color_trc bt709 -colorspace bt709 "$SDR_FIXTURES/high-rate.mp4" -ffmpeg -y -f lavfi -i testsrc2=size=512x288:rate=60:duration=30 \ - -vf "select='if(lt(t,7.5),1,not(mod(n,3)))',setparams=color_trc=bt709:color_primaries=bt709:colorspace=bt709" -fps_mode vfr \ - -c:v libx264 -profile:v baseline -pix_fmt yuv420p -bf 0 -g 10000 -keyint_min 10000 -sc_threshold 0 -crf 18 \ - -color_primaries bt709 -color_trc bt709 -colorspace bt709 "$SDR_FIXTURES/vfr.mp4" -ffmpeg -y -i "$SDR_FIXTURES/no-audio.mp4" -c:v libx264 -profile:v baseline -pix_fmt yuv420p -bf 0 \ - -vf setparams=color_trc=linear:color_primaries=bt709:colorspace=bt709 \ - -color_primaries bt709 -color_trc linear -colorspace bt709 "$SDR_FIXTURES/linear.mp4" -for SDR_DURATION in 3 12; do - if [ "$SDR_DURATION" = 3 ]; then SDR_HEVC_NAME=hevc-small-open-gop; else SDR_HEVC_NAME=hevc-long-open-gop; fi - ffmpeg -y -f lavfi -i "testsrc2=size=512x288:rate=30:duration=$SDR_DURATION" \ - -vf setparams=color_trc=bt709:color_primaries=bt709:colorspace=bt709 \ - -c:v libx265 -pix_fmt yuv420p -x265-params 'open-gop=1:keyint=30:min-keyint=30:scenecut=0:bframes=0' \ - -color_primaries bt709 -color_trc bt709 -colorspace bt709 "$SDR_FIXTURES/$SDR_HEVC_NAME.mp4" -done -``` - -Build/install the test APK, then stage fixtures. Choose the serial of your dedicated test device; the commands below do not select an arbitrary connected device. - -```sh -SDR_DEVICE=emulator-5556 -./gradlew :library:assembleDebugAndroidTest -adb -s "$SDR_DEVICE" install -r library/build/outputs/apk/androidTest/debug/library-debug-androidTest.apk -adb -s "$SDR_DEVICE" push "$SDR_FIXTURES/." /data/local/tmp/nat547-fixtures/ -adb -s "$SDR_DEVICE" shell run-as com.mux.video.upload.test mkdir -p files/sdr-fixtures -adb -s "$SDR_DEVICE" shell run-as com.mux.video.upload.test cp -r /data/local/tmp/nat547-fixtures/. files/sdr-fixtures/ -adb -s "$SDR_DEVICE" shell am instrument -w -r \ - -e class com.mux.video.upload.internal.standardization.SdrConversionFixtureTests \ - com.mux.video.upload.test/androidx.test.runner.AndroidJUnitRunner -``` - -The H.264 matrix requires accepted, independently reinspected output, including preserved VFR timestamps and first stereo audio selection. LINEAR must fail before export. HEVC may succeed or fail conservatively on an unproven device; a safe failure is not evidence of working HEVC conversion. The cancellation test checks source bytes, tracked output cleanup, and one terminal result. API 23 and physical-device HEVC, quality/performance, and Mux ingest remain separate acceptance gates. diff --git a/library/src/androidTest/java/com/mux/video/upload/internal/standardization/SdrConversionFixtureTests.kt b/library/src/androidTest/java/com/mux/video/upload/internal/standardization/SdrConversionFixtureTests.kt index 3c6426d4..de9ce6f4 100644 --- a/library/src/androidTest/java/com/mux/video/upload/internal/standardization/SdrConversionFixtureTests.kt +++ b/library/src/androidTest/java/com/mux/video/upload/internal/standardization/SdrConversionFixtureTests.kt @@ -13,7 +13,7 @@ import java.util.concurrent.CountDownLatch import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicInteger -/** Stage external fixtures as documented in the adapter README; never package media in the APK. */ +/** Runs the adapter against externally staged fixtures without media in the APK. */ @RunWith(AndroidJUnit4::class) class SdrConversionFixtureTests { private val instrumentation = InstrumentationRegistry.getInstrumentation() diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/README.md b/library/src/main/java/com/mux/video/upload/internal/standardization/README.md deleted file mode 100644 index ad2d3b4c..00000000 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/README.md +++ /dev/null @@ -1,36 +0,0 @@ -# SDR adapter (NAT-547) - -`SdrConversionAdapter.start` accepts the original file, its metadata and complete shared sample/timeline inspection, and the planner's `StandardInputConversion`. These must describe the same unchanged source. It exports on a dedicated `HandlerThread`, validates the generated file with `StandardInputOutputValidator`, and invokes one internal terminal callback on that thread. It does not start transport or emit public upload events. - -- `Completed` transfers an explicitly owned `SdrGeneratedFile` and validated facts to orchestration. Keep it while upload/resume needs it; call its idempotent `delete()` at the terminal lifecycle point. -- `Failed` permits original-file fallback. No partial or rejected output is selected. -- `Cancelled` stops preparation even without a Transformer callback. Orchestration must suppress public results and must not upload the original as a cancellation fallback. -- If deletion fails, `Attempt.pendingCleanup` retains the exact deletion authority for retry. Nothing sweeps `cacheDir/mux-upload`, where customer source copies can live. - -Each attempt allocates only its tracked unique `sdr-*.mp4` file under `cacheDir/mux-upload/standard-input`, after a size-based disk guard. It deletes that file on failure/cancellation, releases Transformer, and quits its thread. A successful file outlives that thread. Persisted payload identity, abandoned-file recovery, and generated resume belong to NAT-549; upload/pause orchestration belongs to NAT-548. - -## Encoding contract - -H.264 stays H.264, HEVC stays HEVC, and another proven SDR source uses H.264. Dimensions are the planner's exact display-oriented even dimensions, with orientation baked into pixels and no upscaling. Portrait encoding is explicitly enabled. Video always encodes, including an identity presentation effect, so bitrate/GOP remediation cannot silently remux the source. - -The 8-bit 4:2:0 path requests H.264 Baseline (no B frames) or HEVC Main, VBR at 6 Mbps through a 2048-pixel long side or 15 Mbps above it, additionally capped to 75% of the plan's average-bitrate ceiling. GOP targets use 90% of the planned interval ceiling, rounded down to whole seconds. For preserved VFR timestamps, the target is also reduced by the nominal rate times the longest measured presentation gap: encoders commonly convert the interval to a nominal-rate frame count. If a positive whole-second target cannot fit, preparation falls back before export. Output validation remains the final bitrate, IDR/closed-GOP, profile, pixel, cadence, and timeline gate. - -Media3's default video factory overrides H.264 profile requests. The adapter therefore supplies Media3 `DefaultCodec` with the exact platform profile, level, dimensions, bitrate, nominal rate, surface input, and GOP interval. Preflight and export share those configuration keys, including Media3 1.11.1's API/SoC performance settings and the Redmi Note 9 Pro high-rate hint workaround. Planned timestamps remain unchanged. Transformer owns decoding, effects, encoding, and muxing. Its default no-output-sample watchdog remains enabled (10 seconds normally, 25 seconds on emulators); timeout errors use the same terminal cleanup as other export failures. Native calls cannot be forcibly interrupted. The AAC delegate disables both encoder and format fallback. Every `onFallbackApplied`, mismatched output MIME, or invalid generated file fails the attempt; a listed encoder never proves a usable path. - -The bounded advertised-format preflight gives cold native enumeration a separate two-second cooperative limit and capability queries a 500 ms limit, checked between calls. These are conservative development guards, not measured hardware performance thresholds. Only matching capability queries count toward the 128-query cap; unrelated inventory entries cannot reject an otherwise supported device merely by exceeding 128 entries. It checks source decode and exact output encode support before creating a file, and pins one supported video encoder. It does not turn advertised capabilities into the planner's device-path proof. - -Preserved cadence keeps source timestamps. Constant-rate reduction is limited to proven constant cadence with an integral source/30 fps ratio; a one-part-per-million allowance covers microsecond-rounded rate measurement. Media3 drops frames at the planned 30 fps and the shared validator checks actual output cadence and duration. LINEAR is correctly classified as SDR by inspection, but this adapter rejects its explicit transfer tags before export until color handling is proven. Frame duplication, fractional-rate resampling, 10-bit SDR encoding, and HDR/tone mapping have no proof in this adapter and fail conservatively. NAT-550 owns tone mapping. - -## Audio decision - -[Shipped Swift v1.2.0](https://github.com/muxinc/swift-upload-sdk/blob/faa440e6ea9339315a9d192ca314c4ef5a8668c9/Sources/MuxUploadSDK/InputStandardization/UploadInputStandardizationWorker.swift#L794) copies AAC and otherwise preserves sample rate, channel count, and available layout while encoding AAC (160 kbps for mono/stereo; 384 kbps for surround). Following that behavior and the Android TDD's exclusion of new surround conversion, this adapter copies known compliant AAC mono/stereo/5.1, converts known non-AAC mono/stereo to AAC-LC at 160 kbps with the same rate/count, and rejects non-AAC surround or unknown layouts. It adds no downmix policy. Output channel layout and reported export sample rate/count must match these targets. - -The asset loader's selector matches the first audio track's container ID, rather than Media3's preferred/default track. Missing identity fails; it never substitutes a secondary track. - -## Evidence and remaining limits - -Committed unit tests cover encoding targets, VFR interval targeting with the shared validator, strict encoder configuration, performance/workaround boundaries, cold/large codec inventories, first-track selection, Media3 timeout/fallback callbacks, and adapter ownership/cancellation. The committed device suite and [fixture recipes](../../../../../../../../androidTest/README.md) cover resize, silent H.264, rotated portrait, MP3 stereo to AAC, HEVC, 240-to-30 fps, preserved VFR, LINEAR fallback, and cancellation. Fixtures are staged outside the APK and remain external to product Git. Run these checks on each Media3 upgrade; unit configuration checks do not prove hardware behavior. - -The distinguishable multi-audio source has 3s video, 3s first stereo AAC (track ID 2, non-default), and 6s secondary mono AAC (ID 3, default). Output contains the first stereo track: its copied AAC bytes have the same SHA-256 as source audio ordinal zero and differ from the default secondary. This proves local selection only; Mux ingest parity remains pending. - -API 36 emulator evidence includes a short accepted HEVC export and a longer export rejected for CRA/open GOP; the software encoder also cannot encode 720p. These observations do not establish a universal HEVC limitation or justify changing codec families. API 23-29 execution, physical hardware, long HEVC closed-GOP support, 10-bit SDR, HDR, high-resolution device coverage, visual quality/performance acceptance, and a live Mux upload remain verification gates. Emulator timings do not establish physical-device performance limits. NAT-548 can use the validated SDR path for the early upload journey; no predecessor merge is required to continue the stack. From 184834ff06ceaac838c6f1a83ee67b00c955b098 Mon Sep 17 00:00:00 2001 From: Eugene Morozov Date: Wed, 7 Oct 2026 14:13:22 -0700 Subject: [PATCH 45/49] Remove manual fixture tooling and test-only SDK entry points --- library/build.gradle | 5 - .../SdrConversionFixtureTests.kt | 119 ------------------ .../standardization/Media3SdrExportEngine.kt | 8 +- .../standardization/SdrConversionAdapter.kt | 2 +- .../SdrMedia3ConfigurationTests.kt | 26 +++- 5 files changed, 23 insertions(+), 137 deletions(-) delete mode 100644 library/src/androidTest/java/com/mux/video/upload/internal/standardization/SdrConversionFixtureTests.kt diff --git a/library/build.gradle b/library/build.gradle index 5446e470..d7eff249 100644 --- a/library/build.gradle +++ b/library/build.gradle @@ -35,9 +35,6 @@ android { buildFeatures { buildConfig true } - testOptions { - targetSdk = 36 - } } muxDistribution { @@ -101,6 +98,4 @@ dependencies { testImplementation 'androidx.test.ext:junit:1.2.1' testImplementation "io.mockk:mockk:1.13.9" testImplementation 'org.robolectric:robolectric:4.11.1' - androidTestImplementation 'androidx.test:runner:1.6.2' - androidTestImplementation 'androidx.test.ext:junit:1.2.1' } diff --git a/library/src/androidTest/java/com/mux/video/upload/internal/standardization/SdrConversionFixtureTests.kt b/library/src/androidTest/java/com/mux/video/upload/internal/standardization/SdrConversionFixtureTests.kt deleted file mode 100644 index de9ce6f4..00000000 --- a/library/src/androidTest/java/com/mux/video/upload/internal/standardization/SdrConversionFixtureTests.kt +++ /dev/null @@ -1,119 +0,0 @@ -package com.mux.video.upload.internal.standardization - -import androidx.test.ext.junit.runners.AndroidJUnit4 -import androidx.test.platform.app.InstrumentationRegistry -import com.mux.video.upload.internal.InputStandardization -import com.mux.video.upload.internal.MaximumResolution -import org.junit.Assert.* -import org.junit.Assume.assumeTrue -import org.junit.Test -import org.junit.runner.RunWith -import java.io.File -import java.util.concurrent.CountDownLatch -import java.util.concurrent.TimeUnit -import java.util.concurrent.atomic.AtomicInteger - -/** Runs the adapter against externally staged fixtures without media in the APK. */ -@RunWith(AndroidJUnit4::class) -class SdrConversionFixtureTests { - private val instrumentation = InstrumentationRegistry.getInstrumentation() - private val context = instrumentation.targetContext - private val fixtures = File(instrumentation.context.filesDir, "sdr-fixtures") - - private fun source(name: String): File { - assumeTrue("Stage SDR fixtures before running this suite", fixtures.isDirectory) - val fixture = File(fixtures, name) - assertTrue("Missing fixture $name", fixture.isFile) - return File(context.cacheDir, "mux-upload/source-$name").apply { - parentFile!!.mkdirs(); fixture.copyTo(this, overwrite = true) - } - } - private fun inspect(file: File): Triple { - val metadata = (MediaMetadataInspector().inspect(file) as MetadataInspectionResult.Success).inspection - val samples = StandardInputTimelineInspector().inspect(file, metadata) - assertEquals(SampleScanStatus.Complete, samples.status) - // Direct adapter exercise: these test flags do not establish hardware capability proof. - val capabilities = PlanningCapabilities(true, true, setOf(VideoCodec.H264, VideoCodec.Hevc), PolicyRequirement.entries.toSet(), - canPrepareCompliantAacAudio = true) - val action = StandardInputPlanner().plan(samples.facts, - InputStandardization(true, MaximumResolution.Preset1280x720), capabilities).action - assertTrue("${file.name}: $action\n${samples.facts}\n$metadata", action is StandardInputAction.Convert) - return Triple(metadata, samples, (action as StandardInputAction.Convert).conversion) - } - private fun ownedFiles(): Set = File(context.cacheDir, "mux-upload/standard-input").listFiles() - ?.map { it.name }?.toSet() ?: emptySet() - - private fun export(name: String): SdrConversionResult { - val file = source(name) - val original = file.readBytes() - val (metadata, samples, conversion) = inspect(file) - val before = ownedFiles() - val done = CountDownLatch(1) - val calls = AtomicInteger() - var result: SdrConversionResult? = null - val attempt = SdrConversionAdapter(context).start(file, metadata, samples, conversion) { - result = it; calls.incrementAndGet(); done.countDown() - } - try { - assertTrue("Timed out $name", done.await(90, TimeUnit.SECONDS)) - File(instrumentation.context.filesDir, "sdr-evidence").apply { mkdirs() }.let { area -> - File(area, "$name.txt").writeText("$metadata\n${samples.facts}\n${samples.timeline}\n$conversion\n$result") - } - assertEquals(1, calls.get()) - assertArrayEquals(original, file.readBytes()) - if (result is SdrConversionResult.Completed) { - val completed = result as SdrConversionResult.Completed - assertEquals(MediaFact.Known(conversion.outputCodec), completed.facts.videoCodec) - assertEquals(MediaFact.Known(conversion.outputDimensions), completed.facts.encodedDimensions) - assertEquals(MediaFact.Known(0), completed.facts.rotationDegrees) - assertTrue(StandardInputOutputValidator().validateGeneratedOutput(completed.output.file, samples.facts, - samples.timeline, conversion) is StandardInputOutputValidation.Accepted) - assertTrue(completed.output.delete()) - } - assertEquals(before, ownedFiles()) - return checkNotNull(result) - } finally { - attempt.cancel(); (result as? SdrConversionResult.Completed)?.output?.delete(); file.delete() - } - } - @Test fun h264ResizeSilentPortraitNonAacAndCadenceExportsPassReinspection() { - for (name in listOf("multi-audio.mp4", "no-audio.mp4", "rotated.mp4", "mp3-stereo.mp4", "high-rate.mp4", "vfr.mp4")) { - val result = export(name) - assertTrue("$name: $result", result is SdrConversionResult.Completed) - if (name == "multi-audio.mp4") assertEquals(MediaFact.Known(listOf(AudioTrack(MediaFact.Known( - AudioFormat.Aac(AudioChannelLayout.Stereo))))), (result as SdrConversionResult.Completed).facts.audioTracks) - } - } - @Test fun hevcEitherPassesFullValidationOrRejectsSafelyWithoutChangingCodecFamily() { - for (name in listOf("hevc-small-open-gop.mp4", "hevc-long-open-gop.mp4")) { - val result = export(name) - assertTrue("$name: $result", result is SdrConversionResult.Completed || result in listOf( - SdrConversionResult.Failed(SdrConversionFailure.CapabilityUnavailable), - SdrConversionResult.Failed(SdrConversionFailure.Export), SdrConversionResult.Failed(SdrConversionFailure.OutputInvalid))) - } - } - @Test fun linearTaggedInputFallsBackBeforeExport() { - assertEquals(SdrConversionResult.Failed(SdrConversionFailure.UnsupportedPlan), export("linear.mp4")) - } - @Test fun cancellationRemovesOnlyOwnedPartialAndDoesNotPublishLateResults() { - val file = source("multi-audio.mp4") - val original = file.readBytes() - val (metadata, samples, conversion) = inspect(file) - val before = ownedFiles() - val done = CountDownLatch(1) - val calls = AtomicInteger() - var result: SdrConversionResult? = null - val attempt = SdrConversionAdapter(context).start(file, metadata, samples, conversion) { - result = it; calls.incrementAndGet(); done.countDown() - } - try { - attempt.cancel() - assertTrue(done.await(30, TimeUnit.SECONDS)) - assertEquals(SdrConversionResult.Cancelled, result) - Thread.sleep(500) - assertEquals(1, calls.get()) - assertEquals(before, ownedFiles()) - assertArrayEquals(original, file.readBytes()) - } finally { attempt.cancel(); file.delete() } - } -} diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/Media3SdrExportEngine.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/Media3SdrExportEngine.kt index ff606d0f..2398bd91 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/Media3SdrExportEngine.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/Media3SdrExportEngine.kt @@ -105,10 +105,7 @@ internal class FirstAudioTrackSelector(context: Context, private val trackId: St .setExceedAudioConstraintsIfNecessary(true).setExceedRendererCapabilitiesIfNecessary(true)) } override fun selectAudioTrack(mappedTrackInfo: MappedTrackInfo, rendererFormatSupports: Array>, - rendererMixedMimeTypeAdaptationSupports: IntArray, params: Parameters): Pair? = - selectFirstAudioTrack(mappedTrackInfo) - - internal fun selectFirstAudioTrack(mappedTrackInfo: MappedTrackInfo): Pair? { + rendererMixedMimeTypeAdaptationSupports: IntArray, params: Parameters): Pair? { if (trackId == null) return null for (renderer in 0 until mappedTrackInfo.rendererCount) { if (mappedTrackInfo.getRendererType(renderer) != C.TRACK_TYPE_AUDIO) continue @@ -154,9 +151,6 @@ internal class StrictSdrEncoderFactory( return DefaultCodec(context, exact, media, capability.name, false, null) } - internal fun videoConfiguration(format: Format): MediaFormat = - SdrVideoEncoderConfiguration.create(requestedVideoFormat(format), targets, capability.level) - private fun requestedVideoFormat(format: Format): Format { check(format.sampleMimeType == targets.videoMime && format.width == conversion.outputDimensions.width && format.height == conversion.outputDimensions.height && format.rotationDegrees == 0 && diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt index 81448954..b2eec190 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt @@ -27,7 +27,7 @@ internal class SdrGeneratedFile private constructor(val file: File) { } } -/** Small Media3 boundary so terminal races and cleanup can be tested without a platform encoder. */ +/** Internal boundary for Media3 export and cancellation. */ internal interface SdrExportEngine { fun start(input: File, output: File, completed: (String?, String?) -> Unit, failed: (SdrConversionFailure) -> Unit) diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/SdrMedia3ConfigurationTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/SdrMedia3ConfigurationTests.kt index fed2acee..055d5caf 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/SdrMedia3ConfigurationTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/SdrMedia3ConfigurationTests.kt @@ -3,6 +3,7 @@ package com.mux.video.upload.internal.standardization import android.content.Context import android.media.MediaCodecInfo import android.media.MediaFormat +import android.util.Pair import androidx.media3.common.C import androidx.media3.common.ColorInfo import androidx.media3.common.Format @@ -10,6 +11,7 @@ import androidx.media3.common.MediaItem import androidx.media3.common.TrackGroup import androidx.media3.exoplayer.source.TrackGroupArray import androidx.media3.exoplayer.trackselection.MappingTrackSelector +import androidx.media3.exoplayer.trackselection.ExoTrackSelection import androidx.media3.transformer.Composition import androidx.media3.transformer.EditedMediaItem import androidx.media3.transformer.EditedMediaItemSequence @@ -26,6 +28,7 @@ import org.robolectric.annotation.Config import org.robolectric.annotation.Implementation import org.robolectric.annotation.Implements import org.robolectric.shadows.MediaCodecInfoBuilder +import java.lang.reflect.InvocationTargetException @RunWith(RobolectricTestRunner::class) @Config(manifest = Config.NONE, sdk = [28], shadows = [SupportedFormatShadow::class]) @@ -44,7 +47,7 @@ class SdrMedia3ConfigurationTests { @Test fun strictFactoryRejectsMimeDimensionsRotationAndLinearBeforeCodecCreation() { val factory = StrictSdrEncoderFactory(context, conversion, targets, SdrEncoderCapability("chosen", 4)) - val configuration = factory.videoConfiguration(format) + val configuration = SdrVideoEncoderConfiguration.create(format, targets, 4) assertEquals(targets.platformProfile, configuration.getInteger(MediaFormat.KEY_PROFILE)) assertEquals(4, configuration.getInteger(MediaFormat.KEY_LEVEL)) assertEquals(18, configuration.getInteger(MediaFormat.KEY_I_FRAME_INTERVAL)) @@ -52,7 +55,7 @@ class SdrMedia3ConfigurationTests { for (bad in listOf(format.buildUpon().setSampleMimeType("video/hevc").build(), format.buildUpon().setWidth(640).build(), format.buildUpon().setRotationDegrees(90).build(), format.buildUpon().setColorInfo(ColorInfo.SDR_BT709_LIMITED.buildUpon().setColorTransfer(C.COLOR_TRANSFER_LINEAR).build()).build())) { - assertThrows(IllegalStateException::class.java) { factory.videoConfiguration(bad) } + assertThrows(IllegalStateException::class.java) { factory.createForVideoEncoding(bad, null) } } } @@ -139,10 +142,23 @@ class SdrMedia3ConfigurationTests { every { getRendererType(0) } returns C.TRACK_TYPE_AUDIO every { getTrackGroups(0) } returns groups } - val selection = FirstAudioTrackSelector(context, "2").selectFirstAudioTrack(mapped)!! + val selection = selectAudioTrack(FirstAudioTrackSelector(context, "2"), mapped)!! assertEquals("2", selection.first.group.getFormat(selection.first.tracks.single()).id) - assertNull(FirstAudioTrackSelector(context, null).selectFirstAudioTrack(mapped)) - assertThrows(IllegalStateException::class.java) { FirstAudioTrackSelector(context, "missing").selectFirstAudioTrack(mapped) } + assertNull(selectAudioTrack(FirstAudioTrackSelector(context, null), mapped)) + assertThrows(IllegalStateException::class.java) { selectAudioTrack(FirstAudioTrackSelector(context, "missing"), mapped) } + } + + private fun selectAudioTrack(selector: FirstAudioTrackSelector, + mapped: MappingTrackSelector.MappedTrackInfo): Pair? { + val method = FirstAudioTrackSelector::class.java.declaredMethods.single { it.name == "selectAudioTrack" } + method.isAccessible = true + try { + @Suppress("UNCHECKED_CAST") + return method.invoke(selector, mapped, emptyArray>(), intArrayOf(), selector.parameters) + as Pair? + } catch (error: InvocationTargetException) { + throw error.cause!! + } } @Test fun exportListenerRoutesMuxerWatchdogAndFallbackToFailureAndChecksCopiedAudio() { From fcae48c3df440b75c10ef58a6b0b49507ccae4b8 Mon Sep 17 00:00:00 2001 From: Eugene Morozov Date: Wed, 7 Oct 2026 15:32:41 -0700 Subject: [PATCH 46/49] Use actual VFR GOP windows and reject unproven rate hints --- .../standardization/Media3SdrExportEngine.kt | 10 ++-- .../standardization/SdrConversionAdapter.kt | 3 +- .../standardization/SdrEncodingTargets.kt | 38 +++++++++------ .../SdrConversionAdapterTests.kt | 45 ++++++++++++++++-- .../SdrMedia3ConfigurationTests.kt | 46 ++++++++++++++----- 5 files changed, 105 insertions(+), 37 deletions(-) diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/Media3SdrExportEngine.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/Media3SdrExportEngine.kt index 2398bd91..3d11b082 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/Media3SdrExportEngine.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/Media3SdrExportEngine.kt @@ -133,8 +133,8 @@ internal class StrictSdrEncoderFactory( ) : Codec.EncoderFactory { private val audioDelegate = DefaultEncoderFactory.Builder(context) .setEnableFallback(false).setEnableFormatFallback(false) - .setRequestedAudioEncoderSettings(AudioEncoderSettings.Builder().setBitrate(160_000) - .setProfile(MediaCodecInfo.CodecProfileLevel.AACObjectLC).build()).build() + .setRequestedAudioEncoderSettings(AudioEncoderSettings.Builder().setBitrate(SdrEncodingTargets.AAC_BITRATE) + .setProfile(SdrEncodingTargets.AAC_PROFILE).build()).build() override fun videoNeedsEncoding() = true override fun audioNeedsEncoding() = targets.audioChannels != null && !targets.copyAac @@ -165,6 +165,9 @@ internal object SdrVideoEncoderConfiguration { fun create(format: Format, targets: SdrEncodingTargets, level: Int, sdk: Int = Build.VERSION.SDK_INT, device: String = Build.DEVICE, soc: String = if (Build.VERSION.SDK_INT >= 31) Build.SOC_MODEL else ""): MediaFormat { + // Media3 forces a 30-fps hint on this device. Other rates have unproven GOP/rate + // control here; preflight rejects them instead of changing the planned settings. + check(sdk >= 30 || device != "joyeuse" || format.frameRate == 30f) val media = MediaFormatUtil.createMediaFormatFromFormat(format) media.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface) media.setInteger(MediaFormat.KEY_BITRATE_MODE, MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_VBR) @@ -178,9 +181,6 @@ internal object SdrVideoEncoderConfiguration { val overflow = sdk in 31..34 && soc in setOf("SM8550", "SM7450", "SM6450", "SC9863A", "T612", "T606", "T603") media.setInteger(MediaFormat.KEY_OPERATING_RATE, if (sdk == 26) 30 else if (overflow) 1000 else Int.MAX_VALUE) } - // Redmi Note 9 Pro rejects high KEY_FRAME_RATE. This is a codec hint; timestamps - // and Media3's cadence remain planned. Lowering it only shortens the requested GOP. - if (sdk < 30 && device == "joyeuse" && format.frameRate > 30) media.setFloat(MediaFormat.KEY_FRAME_RATE, 30f) return media } } diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt index b2eec190..3c365baa 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt @@ -4,6 +4,7 @@ import android.content.Context import android.os.Handler import android.os.HandlerThread import android.os.Looper +import androidx.media3.common.MimeTypes import java.io.File import java.util.concurrent.atomic.AtomicBoolean @@ -97,7 +98,7 @@ internal class SdrConversionAdapter( exporter.start(input, owned.file, { videoMime, audioMime -> handler.post { guarded { if (isTerminal()) return@guarded - if (videoMime != targets.videoMime || audioMime != if (targets.audioChannels != null) "audio/mp4a-latm" else null) + if (videoMime != targets.videoMime || audioMime != if (targets.audioChannels != null) MimeTypes.AUDIO_AAC else null) return@guarded fail(SdrConversionFailure.CodecChanged) if (!sourceUnchanged()) return@guarded fail(SdrConversionFailure.SourceChanged) when (val validation = validate(owned.file, source.facts, source.timeline, conversion, cancelled::get)) { diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/SdrEncodingTargets.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/SdrEncodingTargets.kt index 9ce7d088..337a5f40 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/SdrEncodingTargets.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/SdrEncodingTargets.kt @@ -34,6 +34,9 @@ internal data class SdrEncodingTargets( } companion object { + const val AAC_BITRATE = 160_000 + const val AAC_PROFILE = CodecProfileLevel.AACObjectLC + fun from(conversion: StandardInputConversion, metadata: MediaMetadataInspection, source: MediaSampleInspection): SdrEncodingTargets? { val facts = source.facts @@ -85,16 +88,24 @@ internal data class SdrEncodingTargets( if (conversion.outputCadence == OutputCadence.PreserveSourceTimestamps && facts.cadence == MediaFact.Known(Cadence.Variable)) { val times = source.timeline.videoPresentationSeconds.valueOrNull!! - if (times.size < 2 || times.any { !it.isFinite() }) return null - var longestGap = 0.0 + val duration = source.timeline.videoDurationSeconds.valueOrNull!! + if (times.size < 2 || times.any { !it.isFinite() } || !duration.isFinite() || duration <= 0) return null + val end = times.first() + duration + if (!end.isFinite() || end < times.last()) return null for (index in 1 until times.size) { - val gap = times[index] - times[index - 1] - if (gap <= 0) return null - longestGap = maxOf(longestGap, gap) + if (times[index] <= times[index - 1]) return null + } + // Encoders commonly turn seconds into nominal-rate frames. Check every real + // frame window, including the final sample's duration, without retiming gaps. + while (gopSeconds >= 1) { + val frames = ceil(gopSeconds * rate).toInt() + val fits = times.indices.all { index -> + val windowEnd = times.getOrNull(index + frames) ?: end + windowEnd - times[index] <= interval * 0.9 + 2e-6 + } + if (fits) break + gopSeconds-- } - // Encoders commonly turn the interval into nominal-rate frames. Bound their - // elapsed span using the slowest observed gap; validation still proves output. - gopSeconds = minOf(gopSeconds, floor(interval * 0.9 / (rate * longestGap)).toInt()) if (gopSeconds < 1) return null } return SdrEncodingTargets( @@ -118,11 +129,10 @@ internal class SdrCapabilityPreflight( ) { fun videoEncoder(conversion: StandardInputConversion, metadata: MediaMetadataInspection, targets: SdrEncodingTargets, isCancelled: () -> Boolean): SdrEncoderCapability? { - val enumerationStarted = nanoTime() if (isCancelled()) return null val infos = codecInfos() - // Cold native enumeration has its own cooperative bound, not the query budget. - if (isCancelled() || nanoTime() - enumerationStarted >= 2_000_000_000L) return null + // Native enumeration cannot be interrupted; budget subsequent capability queries. + if (isCancelled()) return null val started = nanoTime() var queries = 0 fun withinBudget() = !isCancelled() && nanoTime() - started < 500_000_000L @@ -156,8 +166,8 @@ internal class SdrCapabilityPreflight( val audioMime = metadata.tracks.first { it.sourceTrackId.valueOrNull?.toString() == targets.firstAudioTrackId }.mimeType.valueOrNull ?: return null if (!supportsDecode(audioMime, MediaFormat.createAudioFormat(audioMime, targets.audioSampleRate!!, targets.audioChannels))) return null val aac = MediaFormat.createAudioFormat(MimeTypes.AUDIO_AAC, targets.audioSampleRate, targets.audioChannels) - aac.setInteger(MediaFormat.KEY_BIT_RATE, 160_000) - aac.setInteger(MediaFormat.KEY_AAC_PROFILE, CodecProfileLevel.AACObjectLC) + aac.setInteger(MediaFormat.KEY_BIT_RATE, SdrEncodingTargets.AAC_BITRATE) + aac.setInteger(MediaFormat.KEY_AAC_PROFILE, SdrEncodingTargets.AAC_PROFILE) if (infos.none { withinBudget() && it.isEncoder && it.supportedTypes.contains(MimeTypes.AUDIO_AAC) && supports(it, MimeTypes.AUDIO_AAC, aac) }) return null } @@ -170,8 +180,6 @@ internal class SdrCapabilityPreflight( val caps = info.getCapabilitiesForType(targets.videoMime) val level = caps.profileLevels.filter { it.profile == targets.platformProfile }.maxOfOrNull { it.level } ?: return@runCatching null val size = conversion.outputDimensions - // A device workaround may lower KEY_FRAME_RATE as a configure-time hint. - // Still guard the actual planned size/rate against advertised video limits. if (caps.videoCapabilities?.areSizeAndRateSupported(size.width, size.height, conversion.outputFrameRate) != true) return@runCatching null val requested = Format.Builder().setSampleMimeType(targets.videoMime).setWidth(size.width).setHeight(size.height) diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/SdrConversionAdapterTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/SdrConversionAdapterTests.kt index 29630eff..d47c554d 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/SdrConversionAdapterTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/SdrConversionAdapterTests.kt @@ -37,14 +37,14 @@ class SdrConversionAdapterTests { private fun targets(plan: StandardInputConversion = conversion, input: MediaFacts = facts, tracks: List = listOf(audio())) = SdrEncodingTargets.from(plan, metadata(tracks), samples(input)) - @Test fun vfrGopTargetUsesSlowestPresentationGapAndValidatorStillChecksActualOutput() { + @Test fun vfrGopTargetUsesActualFrameWindowsAndValidatorStillChecksActualOutput() { val times = (0..900).map { if (it <= 450) it / 60.0 else 7.5 + (it - 450) / 20.0 } val variable = facts.copy(cadence = known(Cadence.Variable)) val time = timeline.copy(durationSeconds = known(30.0), videoDurationSeconds = known(30.0), firstAudioDurationSeconds = known(30.0), videoPresentationSeconds = known(times)) val source = samples(variable).copy(timeline = time) val target = SdrEncodingTargets.from(conversion, metadata(), source)!! - assertTrue(target.keyframeIntervalSeconds in 1..12) + assertEquals(12, target.keyframeIntervalSeconds) // Model an encoder that converts seconds to a frame count at the nominal 30 fps. fun exportedGap(seconds: Int): Double { val frames = seconds * 30 @@ -57,6 +57,43 @@ class SdrConversionAdapterTests { assertTrue(validate(target.keyframeIntervalSeconds) is StandardInputOutputValidation.Accepted) } + @Test fun isolatedVfrPausesShortenGopOnlyByTheirActualSpan() { + for ((pause, expected) in listOf(0.05 to 17, 0.2 to 17, 0.61 to 17, 1.0 to 17, 3.0 to 15)) { + val times = (0 until 1800).map { it / 30.0 + if (it >= 900) pause else 0.0 } + assertEquals("pause=$pause", expected, vfrTargets(times, 60.0 + pause)!!.keyframeIntervalSeconds) + } + } + + @Test fun repeatedScreenRecordingPausesUseWorstActualWindow() { + val times = (0 until 1800).map { it / 30.0 + (it / 300) * 3.0 } + assertEquals(12, vfrTargets(times, 75.0)!!.keyframeIntervalSeconds) + } + + @Test fun highResolutionHevcVfrDoesNotRejectAnIsolatedHitch() { + val plan = conversion.copy(sourceCodec = VideoCodec.Hevc, outputCodec = VideoCodec.Hevc, + outputFrameRate = 60.0, outputPolicyLimits = StandardInputPolicyProfile.PublishedMux.highResolution) + val times = (0 until 3600).map { it / 60.0 + if (it >= 1800) 0.1 else 0.0 } + assertEquals(5, vfrTargets(times, 60.1, plan)!!.keyframeIntervalSeconds) + } + + @Test fun vfrGopIncludesLastSampleDurationAndRejectsUnusableWindows() { + val times = (0 until 60).map { it / 30.0 } + assertEquals(1, vfrTargets(times, 19.0)!!.keyframeIntervalSeconds) + assertNull(vfrTargets(times, 20.0)) + assertNull(vfrTargets(listOf(0.0, 0.1, 0.1), 1.0)) + assertNull(vfrTargets(listOf(0.0, Double.NaN), 1.0)) + assertNull(vfrTargets(times, 1.0)) + } + + private fun vfrTargets(times: List, duration: Double, + plan: StandardInputConversion = conversion): SdrEncodingTargets? { + val variable = facts.copy(cadence = known(Cadence.Variable), videoCodec = known(plan.sourceCodec), + frameRate = known(plan.outputFrameRate)) + val time = timeline.copy(durationSeconds = known(duration), videoDurationSeconds = known(duration), + firstAudioDurationSeconds = known(duration), videoPresentationSeconds = known(times)) + return SdrEncodingTargets.from(plan, metadata(), samples(variable).copy(timeline = time)) + } + @Test fun linearTaggedSdrIsRejectedBeforeAllocatingOrStartingExport() { val format = MediaFormat.createVideoFormat("video/avc", 1920, 1080).apply { setInteger(MediaFormat.KEY_COLOR_TRANSFER, MediaFormat.COLOR_TRANSFER_LINEAR) @@ -143,11 +180,11 @@ class SdrConversionAdapterTests { validate = { _, _, _, _, _ -> duringValidation?.invoke(); validation }, createEngine = { _, _, _, _, _ -> engine }) val attempt = adapter.start(sourceFile, inputMetadata, samples(), conversion, results::add) fun idle() { - if (results.isNotEmpty()) return + if (results.isNotEmpty() || !attempt.looper.thread.isAlive) return try { shadowOf(attempt.looper).idle() } catch (error: IllegalStateException) { // PAUSED-mode idle posts a barrier; terminal cleanup can quit before that post. - if (results.isEmpty() || error.message?.contains("handler thread dead") != true) throw error + if (results.isEmpty() || attempt.looper.thread.isAlive) throw error } } fun assertInputSafe() { assertEquals("original", sourceFile.readText()); assertEquals("customer", unrelated.readText()) } diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/SdrMedia3ConfigurationTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/SdrMedia3ConfigurationTests.kt index 055d5caf..041f1d90 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/SdrMedia3ConfigurationTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/SdrMedia3ConfigurationTests.kt @@ -22,17 +22,20 @@ import androidx.test.core.app.ApplicationProvider import io.mockk.* import org.junit.Assert.* import org.junit.Test +import org.junit.After import org.junit.runner.RunWith import org.robolectric.RobolectricTestRunner import org.robolectric.annotation.Config import org.robolectric.annotation.Implementation import org.robolectric.annotation.Implements import org.robolectric.shadows.MediaCodecInfoBuilder +import org.robolectric.shadows.ShadowBuild import java.lang.reflect.InvocationTargetException @RunWith(RobolectricTestRunner::class) @Config(manifest = Config.NONE, sdk = [28], shadows = [SupportedFormatShadow::class]) class SdrMedia3ConfigurationTests { + @After fun resetAdvertisedFormats() { SupportedFormatShadow.supported = { true } } private val context = ApplicationProvider.getApplicationContext() private val facts = compliantFacts().copy(cadence = known(Cadence.Constant)) private val conversion = (StandardInputPlanner().plan(facts.copy(averageBitrate = known(9_000_000L)), @@ -71,14 +74,27 @@ class SdrMedia3ConfigurationTests { .getInteger(MediaFormat.KEY_OPERATING_RATE)) } - @Test fun redmiFrameRateWorkaroundDoesNotIncreaseLowRateGopOrResampleTimestamps() { - val high = format.buildUpon().setFrameRate(120f).build() - assertEquals(30f, SdrVideoEncoderConfiguration.create(high, targets, 4, sdk = 29, device = "joyeuse") - .getFloat(MediaFormat.KEY_FRAME_RATE), 0f) - val low = format.buildUpon().setFrameRate(5f).build() - assertEquals(5f, SdrVideoEncoderConfiguration.create(low, targets, 4, sdk = 29, device = "joyeuse") - .getFloat(MediaFormat.KEY_FRAME_RATE), 0f) - assertEquals(120f, high.frameRate, 0f) + @Test fun redmiRejectsUnprovenRateHintsInsteadOfChangingBitrateOrGop() { + for (rate in listOf(5f, 24f, 60f, 120f)) { + val requested = format.buildUpon().setFrameRate(rate).build() + assertThrows(IllegalStateException::class.java) { + SdrVideoEncoderConfiguration.create(requested, targets, 4, sdk = 29, device = "joyeuse") + } + assertEquals(rate, SdrVideoEncoderConfiguration.create(requested, targets, 4, sdk = 30, device = "joyeuse") + .getFloat(MediaFormat.KEY_FRAME_RATE), 0f) + } + val accepted = SdrVideoEncoderConfiguration.create(format, targets, 4, sdk = 29, device = "joyeuse") + assertEquals(30f, accepted.getFloat(MediaFormat.KEY_FRAME_RATE), 0f) + assertEquals(targets.bitrate, accepted.getInteger(MediaFormat.KEY_BIT_RATE)) + assertEquals(targets.keyframeIntervalSeconds, accepted.getInteger(MediaFormat.KEY_I_FRAME_INTERVAL)) + } + + @Test fun redmiPreflightRejectsNonThirtyFpsBeforeEncoding() { + ShadowBuild.setDevice("joyeuse") + val infos = arrayOf(codec(false, "video/avc"), codec(true, "video/avc")) + assertNull(SdrCapabilityPreflight({ infos }, { 0L }) + .videoEncoder(conversion.copy(outputFrameRate = 60.0), metadata, targets) { false }) + assertNotNull(SdrCapabilityPreflight({ infos }, { 0L }).videoEncoder(conversion, metadata, targets) { false }) } private fun codec(encoder: Boolean, mime: String): MediaCodecInfo { @@ -95,7 +111,7 @@ class SdrMedia3ConfigurationTests { .setIsEncoder(encoder).setCapabilities(capabilities).build() } - @Test fun coldEnumerationHasSeparateBoundAndUnrelatedCodecsDoNotExhaustQueries() { + @Test fun coldEnumerationDoesNotConsumeQueryBudgetAndUnrelatedCodecsDoNotExhaustQueries() { var now = 0L val unrelated = codec(false, "audio/mpeg") val encoder = codec(true, "video/avc") @@ -111,21 +127,27 @@ class SdrMedia3ConfigurationTests { assertEquals(1, queried.last().getInteger(MediaFormat.KEY_PRIORITY)) } - @Test fun preflightStillBoundsColdInitializationQueriesAndCancellation() { + @Test fun preflightAcceptsSlowEnumerationButBoundsQueriesAndCancellation() { var now = 0L val encoder = codec(true, "video/avc") val slowDecoder = codec(false, "video/avc") var queries = 0 - SupportedFormatShadow.supported = { queries++; now += 500_000_000L; true } val infos = arrayOf(slowDecoder, encoder) - assertNull(SdrCapabilityPreflight({ now += 2_000_000_000L; infos }, { now }) + SupportedFormatShadow.supported = { queries++; true } + assertNotNull(SdrCapabilityPreflight({ now += 3_000_000_000L; infos }, { now }) .videoEncoder(conversion, metadata, targets) { false }) + assertEquals(2, queries) + queries = 0 + SupportedFormatShadow.supported = { queries++; now += 500_000_000L; true } now = 0 assertNull(SdrCapabilityPreflight({ infos }, { now }).videoEncoder(conversion, metadata, targets) { false }) assertEquals(1, queries) var enumerated = false assertNull(SdrCapabilityPreflight({ enumerated = true; infos }, { 0L }).videoEncoder(conversion, metadata, targets) { true }) assertFalse(enumerated) + var cancelled = false + assertNull(SdrCapabilityPreflight({ cancelled = true; infos }, { 0L }) + .videoEncoder(conversion, metadata, targets) { cancelled }) val rejected = codec(false, "video/avc") queries = 0 SupportedFormatShadow.supported = { queries++; false } From 707f47cb85b7507d4d7501297a9222d9941880d9 Mon Sep 17 00:00:00 2001 From: Eugene Morozov Date: Wed, 7 Oct 2026 15:39:36 -0700 Subject: [PATCH 47/49] Synchronize SDR test completion and share AAC layout mapping --- .../standardization/SdrConversionAdapter.kt | 7 +------ .../SdrConversionAdapterTests.kt | 20 +++++++++++++------ 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt index 3c365baa..879bea73 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt @@ -107,12 +107,7 @@ internal class SdrConversionAdapter( // The shared validator checks policy, pixels, cadence, and effective timeline. // Add this adapter's concrete encoder-profile and non-AAC channel contract. val audio = validation.facts.audioTracks.valueOrNull?.singleOrNull()?.format?.valueOrNull as? AudioFormat.Aac - val expectedLayout = when (targets.audioChannels) { - 1 -> AudioChannelLayout.Mono - 2 -> AudioChannelLayout.Stereo - 6 -> AudioChannelLayout.FivePointOne - else -> null - } + val expectedLayout = CodecMetadataReader.aacChannelLayout(targets.audioChannels) if (validation.facts.videoProfile != MediaFact.Known(targets.profile) || (targets.audioChannels != null && audio?.layout != expectedLayout)) fail(SdrConversionFailure.OutputInvalid) else finish(SdrConversionResult.Completed(owned, validation.facts)) diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/SdrConversionAdapterTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/SdrConversionAdapterTests.kt index d47c554d..2dfb4138 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/SdrConversionAdapterTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/SdrConversionAdapterTests.kt @@ -16,6 +16,7 @@ import org.robolectric.Shadows.shadowOf import org.robolectric.annotation.Config import org.robolectric.annotation.LooperMode import java.io.File +import java.util.concurrent.CopyOnWriteArrayList @RunWith(RobolectricTestRunner::class) @Config(manifest = Config.NONE, sdk = [28]) @@ -175,16 +176,23 @@ class SdrConversionAdapterTests { val sourceFile = File.createTempFile("customer-", ".mp4", context.cacheDir).apply { writeText("original") } val unrelated = File(context.cacheDir, "mux-upload/customer-copy.mp4").apply { parentFile!!.mkdirs(); writeText("customer") } val engine = FakeEngine() - val results = mutableListOf() + val results = CopyOnWriteArrayList() val adapter = SdrConversionAdapter(context, preflight = { _, _, _, _ -> capability }, validate = { _, _, _, _, _ -> duringValidation?.invoke(); validation }, createEngine = { _, _, _, _, _ -> engine }) val attempt = adapter.start(sourceFile, inputMetadata, samples(), conversion, results::add) fun idle() { - if (results.isNotEmpty() || !attempt.looper.thread.isAlive) return - try { shadowOf(attempt.looper).idle() } - catch (error: IllegalStateException) { - // PAUSED-mode idle posts a barrier; terminal cleanup can quit before that post. - if (results.isEmpty() || attempt.looper.thread.isAlive) throw error + val thread = attempt.looper.thread + if (results.isEmpty() && thread.isAlive) { + try { shadowOf(attempt.looper).idle() } + catch (error: IllegalStateException) { + // PAUSED-mode idle can race a terminal quit before its barrier is posted. + thread.join(5_000) + if (results.isEmpty() || thread.isAlive) throw error + } + } + if (results.isNotEmpty()) { + thread.join(5_000) + assertFalse("Terminal attempt thread still running", thread.isAlive) } } fun assertInputSafe() { assertEquals("original", sourceFile.readText()); assertEquals("customer", unrelated.readText()) } From d39b5ed29fabd34f9e0e9b543d9b2398b47e85a2 Mon Sep 17 00:00:00 2001 From: Eugene Morozov Date: Fri, 16 Oct 2026 11:29:41 -0700 Subject: [PATCH 48/49] Use normalized platform metadata in SDR conversion --- .../upload/internal/standardization/SdrConversionAdapter.kt | 2 +- .../upload/internal/standardization/SdrEncodingTargets.kt | 3 +-- .../internal/standardization/SdrConversionAdapterTests.kt | 2 +- .../internal/standardization/SdrMedia3ConfigurationTests.kt | 2 +- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt index 879bea73..aac325a9 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt @@ -107,7 +107,7 @@ internal class SdrConversionAdapter( // The shared validator checks policy, pixels, cadence, and effective timeline. // Add this adapter's concrete encoder-profile and non-AAC channel contract. val audio = validation.facts.audioTracks.valueOrNull?.singleOrNull()?.format?.valueOrNull as? AudioFormat.Aac - val expectedLayout = CodecMetadataReader.aacChannelLayout(targets.audioChannels) + val expectedLayout = aacChannelLayout(targets.audioChannels).valueOrNull if (validation.facts.videoProfile != MediaFact.Known(targets.profile) || (targets.audioChannels != null && audio?.layout != expectedLayout)) fail(SdrConversionFailure.OutputInvalid) else finish(SdrConversionResult.Completed(owned, validation.facts)) diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/SdrEncodingTargets.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/SdrEncodingTargets.kt index 337a5f40..4ba96c57 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/SdrEncodingTargets.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/SdrEncodingTargets.kt @@ -42,8 +42,7 @@ internal data class SdrEncodingTargets( val facts = source.facts // LINEAR is SDR, but this surface path has no proof of its transfer handling. if (metadata.tracks.any { it.video?.let { video -> - video.platformColor.transfer.valueOrNull == MediaFormat.COLOR_TRANSFER_LINEAR || - video.configuration.color.transfer.valueOrNull == MediaFormat.COLOR_TRANSFER_LINEAR + video.color.transfer.valueOrNull == MediaFormat.COLOR_TRANSFER_LINEAR } == true }) return null if (source.status != SampleScanStatus.Complete || conversion.toneMapsToSdr || conversion.sourceDynamicRange != DynamicRange.Sdr || facts.dynamicRange != MediaFact.Known(DynamicRange.Sdr) || diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/SdrConversionAdapterTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/SdrConversionAdapterTests.kt index 2dfb4138..0c3b350a 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/SdrConversionAdapterTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/SdrConversionAdapterTests.kt @@ -33,7 +33,7 @@ class SdrConversionAdapterTests { known(if (format.valueOrNull is AudioFormat.Aac) "audio/mp4a-latm" else "audio/mpeg"), known(3.0), MediaFact.Unknown, MediaFact.Unknown, audio = AudioTrackMetadata(format, known(channels), known(48_000), known(0), known(0))) private fun metadata(tracks: List = listOf(audio())) = - MediaMetadataInspection(known(ContainerKind.IsoBaseMedia), tracks, MediaFact.Unknown, facts, 0) + MediaMetadataInspection(known(ContainerKind.IsoBaseMedia), tracks, facts, 0) private fun samples(input: MediaFacts = facts) = MediaSampleInspection(input, SampleScanStatus.Complete, timeline = timeline) private fun targets(plan: StandardInputConversion = conversion, input: MediaFacts = facts, tracks: List = listOf(audio())) = SdrEncodingTargets.from(plan, metadata(tracks), samples(input)) diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/SdrMedia3ConfigurationTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/SdrMedia3ConfigurationTests.kt index 041f1d90..53761dc8 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/SdrMedia3ConfigurationTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/SdrMedia3ConfigurationTests.kt @@ -46,7 +46,7 @@ class SdrMedia3ConfigurationTests { .setHeight(conversion.outputDimensions.height).setFrameRate(30f).setAverageBitrate(targets.bitrate) .setColorInfo(ColorInfo.SDR_BT709_LIMITED).build() private val video = MediaTrackMetadataReader.read(0, MediaFormat.createVideoFormat("video/avc", 1920, 1080)) - private val metadata = MediaMetadataInspection(known(ContainerKind.IsoBaseMedia), listOf(video), MediaFact.Unknown, facts, 0) + private val metadata = MediaMetadataInspection(known(ContainerKind.IsoBaseMedia), listOf(video), facts, 0) @Test fun strictFactoryRejectsMimeDimensionsRotationAndLinearBeforeCodecCreation() { val factory = StrictSdrEncoderFactory(context, conversion, targets, SdrEncoderCapability("chosen", 4)) From fff4029d51a9645a8ff06770db1696d173c1e205 Mon Sep 17 00:00:00 2001 From: Eugene Morozov Date: Fri, 16 Oct 2026 12:11:27 -0700 Subject: [PATCH 49/49] Use coroutines and a cancellable continuation for SDR conversion --- .../standardization/SdrConversionAdapter.kt | 154 ++++++++++-------- .../SdrConversionAdapterTests.kt | 61 ++++--- 2 files changed, 129 insertions(+), 86 deletions(-) diff --git a/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt b/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt index aac325a9..8512eb1f 100644 --- a/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt +++ b/library/src/main/java/com/mux/video/upload/internal/standardization/SdrConversionAdapter.kt @@ -1,12 +1,25 @@ package com.mux.video.upload.internal.standardization import android.content.Context -import android.os.Handler -import android.os.HandlerThread import android.os.Looper import androidx.media3.common.MimeTypes +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CoroutineDispatcher +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.CoroutineStart +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.DelicateCoroutinesApi +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.Job +import kotlinx.coroutines.NonCancellable +import kotlinx.coroutines.cancel +import kotlinx.coroutines.ensureActive +import kotlinx.coroutines.launch +import kotlinx.coroutines.suspendCancellableCoroutine +import kotlinx.coroutines.withContext import java.io.File import java.util.concurrent.atomic.AtomicBoolean +import kotlin.coroutines.resume internal sealed interface SdrConversionResult { /** Caller owns retention until upload/resume reaches its terminal point. */ @@ -35,7 +48,7 @@ internal interface SdrExportEngine { fun cancel() } -/** Each attempt owns one thread, one Transformer, and one explicitly tracked output file. */ +/** Each attempt owns a coroutine job, one Transformer, and one explicitly tracked output file. */ internal class SdrConversionAdapter( context: Context, private val preflight: (StandardInputConversion, MediaMetadataInspection, SdrEncodingTargets, () -> Boolean) -> SdrEncoderCapability? = @@ -44,6 +57,7 @@ internal class SdrConversionAdapter( StandardInputOutputValidator()::validateGeneratedOutput, private val createEngine: (Context, Looper, StandardInputConversion, SdrEncodingTargets, SdrEncoderCapability) -> SdrExportEngine = ::Media3SdrExportEngine, + private val ioDispatcher: CoroutineDispatcher = Dispatchers.IO, ) { private val appContext = context.applicationContext @@ -56,10 +70,8 @@ internal class SdrConversionAdapter( private val source: MediaSampleInspection, private val conversion: StandardInputConversion, private val onResult: (SdrConversionResult) -> Unit, ) { - private val thread = HandlerThread("MuxSdrConversion").apply { start() } - internal val looper: Looper = thread.looper - private val handler = Handler(looper) - private val cancelled = AtomicBoolean(false) + private val job = Job() + private val scope = CoroutineScope(job + Dispatchers.Main) private val terminalLock = Any() private var terminal = false private var engine: SdrExportEngine? = null @@ -73,85 +85,93 @@ internal class SdrConversionAdapter( fun cancel() { synchronized(terminalLock) { if (terminal) return - cancelled.set(true) + job.cancel() } - handler.post { finish(SdrConversionResult.Cancelled) } } - internal fun begin() { handler.post { - guarded { - if (cancelled.get()) return@guarded finish(SdrConversionResult.Cancelled) - val targets = SdrEncodingTargets.from(conversion, metadata, source) - ?: return@guarded fail(SdrConversionFailure.UnsupportedPlan) - val capability = preflight(conversion, metadata, targets, cancelled::get) - ?: return@guarded fail(SdrConversionFailure.CapabilityUnavailable) - if (cancelled.get()) return@guarded finish(SdrConversionResult.Cancelled) - if (!sourceUnchanged()) return@guarded fail(SdrConversionFailure.SourceChanged) + @OptIn(DelicateCoroutinesApi::class, ExperimentalCoroutinesApi::class) + internal fun begin() { + // Enter cleanup even when cancellation wins before the initial Main dispatch. + scope.launch(start = CoroutineStart.ATOMIC) { + val result = try { + ensureActive() + convert() + } catch (_: CancellationException) { SdrConversionResult.Cancelled } + catch (_: Exception) { SdrConversionResult.Failed(SdrConversionFailure.Export) } + catch (_: LinkageError) { SdrConversionResult.Failed(SdrConversionFailure.Export) } + withContext(NonCancellable) { finish(result) } + } + } + + private suspend fun convert(): SdrConversionResult { + val targets = withContext(ioDispatcher) { SdrEncodingTargets.from(conversion, metadata, source) } + ?: return SdrConversionResult.Failed(SdrConversionFailure.UnsupportedPlan) + val capability = withContext(ioDispatcher) { preflight(conversion, metadata, targets) { !job.isActive } } + ?: return SdrConversionResult.Failed(SdrConversionFailure.CapabilityUnavailable) + val preparationFailure = withContext(ioDispatcher) { + if (!sourceUnchanged()) return@withContext SdrConversionFailure.SourceChanged val required = source.timeline.durationSeconds.valueOrNull?.let(targets::estimatedOutputBytes) - ?: return@guarded fail(SdrConversionFailure.InsufficientSourceEvidence) - val owned = SdrGeneratedFile.allocate(appContext.cacheDir, required) - ?: return@guarded fail(SdrConversionFailure.DiskSpace) - output = owned - if (cancelled.get()) return@guarded finish(SdrConversionResult.Cancelled) - val exporter = createEngine(appContext, looper, conversion, targets, capability) - engine = exporter + ?: return@withContext SdrConversionFailure.InsufficientSourceEvidence + // Record ownership before returning across a cancellable dispatcher boundary. + output = SdrGeneratedFile.allocate(appContext.cacheDir, required) + if (output == null) SdrConversionFailure.DiskSpace else null + } + if (preparationFailure != null) return SdrConversionResult.Failed(preparationFailure) + val owned = checkNotNull(output) + val exporter = createEngine(appContext, Looper.getMainLooper(), conversion, targets, capability) + engine = exporter + val exportFailure = suspendCancellableCoroutine { continuation -> + val delivered = AtomicBoolean(false) + continuation.invokeOnCancellation { delivered.set(true) } + fun complete(reason: SdrConversionFailure?) { + if (delivered.compareAndSet(false, true)) continuation.resume(reason) + } exporter.start(input, owned.file, { videoMime, audioMime -> - handler.post { guarded { - if (isTerminal()) return@guarded - if (videoMime != targets.videoMime || audioMime != if (targets.audioChannels != null) MimeTypes.AUDIO_AAC else null) - return@guarded fail(SdrConversionFailure.CodecChanged) - if (!sourceUnchanged()) return@guarded fail(SdrConversionFailure.SourceChanged) - when (val validation = validate(owned.file, source.facts, source.timeline, conversion, cancelled::get)) { - is StandardInputOutputValidation.Accepted -> { - if (!sourceUnchanged()) return@guarded fail(SdrConversionFailure.SourceChanged) - // The shared validator checks policy, pixels, cadence, and effective timeline. - // Add this adapter's concrete encoder-profile and non-AAC channel contract. - val audio = validation.facts.audioTracks.valueOrNull?.singleOrNull()?.format?.valueOrNull as? AudioFormat.Aac - val expectedLayout = aacChannelLayout(targets.audioChannels).valueOrNull - if (validation.facts.videoProfile != MediaFact.Known(targets.profile) || - (targets.audioChannels != null && audio?.layout != expectedLayout)) fail(SdrConversionFailure.OutputInvalid) - else finish(SdrConversionResult.Completed(owned, validation.facts)) - } - is StandardInputOutputValidation.Rejected -> fail(SdrConversionFailure.OutputInvalid) - StandardInputOutputValidation.Cancelled -> finish(SdrConversionResult.Cancelled) - } - } } - }, { reason -> handler.post { guarded { if (!isTerminal()) fail(reason) } } }) + complete(if (videoMime != targets.videoMime || + audioMime != if (targets.audioChannels != null) MimeTypes.AUDIO_AAC else null) + SdrConversionFailure.CodecChanged else null) + }, ::complete) + } + if (exportFailure != null) return SdrConversionResult.Failed(exportFailure) + return withContext(ioDispatcher) { + if (!sourceUnchanged()) return@withContext SdrConversionResult.Failed(SdrConversionFailure.SourceChanged) + when (val validation = validate(owned.file, source.facts, source.timeline, conversion) { !job.isActive }) { + is StandardInputOutputValidation.Accepted -> { + if (!sourceUnchanged()) return@withContext SdrConversionResult.Failed(SdrConversionFailure.SourceChanged) + // The shared validator checks policy, pixels, cadence, and effective timeline. + // Add this adapter's concrete encoder-profile and non-AAC channel contract. + val audio = validation.facts.audioTracks.valueOrNull?.singleOrNull()?.format?.valueOrNull as? AudioFormat.Aac + val expectedLayout = aacChannelLayout(targets.audioChannels).valueOrNull + if (validation.facts.videoProfile != MediaFact.Known(targets.profile) || + (targets.audioChannels != null && audio?.layout != expectedLayout)) + SdrConversionResult.Failed(SdrConversionFailure.OutputInvalid) + else SdrConversionResult.Completed(owned, validation.facts) + } + is StandardInputOutputValidation.Rejected -> SdrConversionResult.Failed(SdrConversionFailure.OutputInvalid) + StandardInputOutputValidation.Cancelled -> SdrConversionResult.Cancelled + } } - } } + } private fun sourceUnchanged() = input.isFile && input.canRead() && sourceLength > 0 && input.length() == sourceLength && input.lastModified() == sourceModified - private fun isTerminal() = synchronized(terminalLock) { terminal } - private fun fail(reason: SdrConversionFailure) = finish(SdrConversionResult.Failed(reason)) - private fun guarded(block: () -> Unit) { - if (isTerminal()) return - try { block() } - catch (_: Exception) { fail(SdrConversionFailure.Export) } - catch (_: LinkageError) { fail(SdrConversionFailure.Export) } - } - - private fun finish(proposed: SdrConversionResult) { - val result = synchronized(terminalLock) { + private suspend fun finish(proposed: SdrConversionResult) { + var result = synchronized(terminalLock) { if (terminal) return terminal = true - if (cancelled.get()) SdrConversionResult.Cancelled else proposed + if (job.isCancelled) SdrConversionResult.Cancelled else proposed } try { runCatching { engine?.cancel() } - if (result !is SdrConversionResult.Completed && output?.delete() == false) { + if (result !is SdrConversionResult.Completed && withContext(ioDispatcher) { output?.delete() } == false) { pendingCleanup = output // Keep ownership at its exact path if the OS refuses deletion; never sweep a directory. - if (result !is SdrConversionResult.Cancelled) { - onResult(SdrConversionResult.Failed(SdrConversionFailure.FileOwnership)) - return - } + if (result !is SdrConversionResult.Cancelled) result = SdrConversionResult.Failed(SdrConversionFailure.FileOwnership) } - onResult(result) + runCatching { onResult(result) } } finally { engine = null - handler.removeCallbacksAndMessages(null) - thread.quitSafely() + scope.cancel() } } } diff --git a/library/src/test/java/com/mux/video/upload/internal/standardization/SdrConversionAdapterTests.kt b/library/src/test/java/com/mux/video/upload/internal/standardization/SdrConversionAdapterTests.kt index 0c3b350a..be21e994 100644 --- a/library/src/test/java/com/mux/video/upload/internal/standardization/SdrConversionAdapterTests.kt +++ b/library/src/test/java/com/mux/video/upload/internal/standardization/SdrConversionAdapterTests.kt @@ -2,17 +2,24 @@ package com.mux.video.upload.internal.standardization import android.content.Context import android.media.MediaFormat +import android.os.Looper import androidx.media3.common.MediaItem import androidx.media3.transformer.Composition import androidx.media3.transformer.EditedMediaItem import androidx.media3.transformer.EditedMediaItemSequence import androidx.media3.transformer.ExportResult import androidx.test.core.app.ApplicationProvider +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.test.StandardTestDispatcher +import kotlinx.coroutines.test.resetMain +import kotlinx.coroutines.test.setMain +import org.junit.After +import org.junit.Before import org.junit.Assert.* import org.junit.Test import org.junit.runner.RunWith import org.robolectric.RobolectricTestRunner -import org.robolectric.Shadows.shadowOf import org.robolectric.annotation.Config import org.robolectric.annotation.LooperMode import java.io.File @@ -21,7 +28,11 @@ import java.util.concurrent.CopyOnWriteArrayList @RunWith(RobolectricTestRunner::class) @Config(manifest = Config.NONE, sdk = [28]) @LooperMode(LooperMode.Mode.PAUSED) +@OptIn(ExperimentalCoroutinesApi::class) class SdrConversionAdapterTests { + private val dispatcher = StandardTestDispatcher() + @Before fun setUpDispatchers() { Dispatchers.setMain(dispatcher) } + @After fun resetDispatchers() { Dispatchers.resetMain() } private val context = ApplicationProvider.getApplicationContext() private val facts = compliantFacts().copy(cadence = known(Cadence.Constant), videoProfile = known(VideoProfile.H264Baseline)) private val conversion = (StandardInputPlanner().plan(facts.copy(averageBitrate = known(9_000_000L)), @@ -160,10 +171,11 @@ class SdrConversionAdapterTests { var starts = 0 var cancellations = 0 override fun start(input: File, output: File, completed: (String?, String?) -> Unit, failed: (SdrConversionFailure) -> Unit) { + assertEquals(Looper.getMainLooper(), Looper.myLooper()) starts++; this.output = output; completions = completed; failures = failed output.writeText("owned partial bytes") } - override fun cancel() { cancellations++ } + override fun cancel() { assertEquals(Looper.getMainLooper(), Looper.myLooper()); cancellations++ } fun complete(video: String? = "video/avc", audio: String? = "audio/mp4a-latm") { completions!!(video, audio) } } private inner class Run( @@ -171,30 +183,18 @@ class SdrConversionAdapterTests { StandardInputPolicyEvaluator().evaluate(facts, selection())), capability: SdrEncoderCapability? = SdrEncoderCapability("fake", 1), duringValidation: (() -> Unit)? = null, + duringPreflight: (() -> Unit)? = null, inputMetadata: MediaMetadataInspection = metadata(), ) { val sourceFile = File.createTempFile("customer-", ".mp4", context.cacheDir).apply { writeText("original") } val unrelated = File(context.cacheDir, "mux-upload/customer-copy.mp4").apply { parentFile!!.mkdirs(); writeText("customer") } val engine = FakeEngine() val results = CopyOnWriteArrayList() - val adapter = SdrConversionAdapter(context, preflight = { _, _, _, _ -> capability }, - validate = { _, _, _, _, _ -> duringValidation?.invoke(); validation }, createEngine = { _, _, _, _, _ -> engine }) + val adapter = SdrConversionAdapter(context, preflight = { _, _, _, _ -> duringPreflight?.invoke(); capability }, + validate = { _, _, _, _, _ -> duringValidation?.invoke(); validation }, createEngine = { _, _, _, _, _ -> engine }, + ioDispatcher = dispatcher) val attempt = adapter.start(sourceFile, inputMetadata, samples(), conversion, results::add) - fun idle() { - val thread = attempt.looper.thread - if (results.isEmpty() && thread.isAlive) { - try { shadowOf(attempt.looper).idle() } - catch (error: IllegalStateException) { - // PAUSED-mode idle can race a terminal quit before its barrier is posted. - thread.join(5_000) - if (results.isEmpty() || thread.isAlive) throw error - } - } - if (results.isNotEmpty()) { - thread.join(5_000) - assertFalse("Terminal attempt thread still running", thread.isAlive) - } - } + fun idle() { dispatcher.scheduler.runCurrent() } fun assertInputSafe() { assertEquals("original", sourceFile.readText()); assertEquals("customer", unrelated.readText()) } } @Test fun successRetainsVerifiedFileUntilOwnerDeletesItAndIgnoresLateEvents() { @@ -234,6 +234,29 @@ class SdrConversionAdapterTests { assertEquals(listOf(SdrConversionResult.Cancelled), run.results) assertEquals(0, run.engine.starts); run.assertInputSafe() } + @Test fun cancelWithoutMedia3CallbackStopsExportAndDeletesPartial() { + val run = Run(duringValidation = { fail("Cancelled export must not validate") }) + run.idle() + run.attempt.cancel() + run.idle() + assertEquals(listOf(SdrConversionResult.Cancelled), run.results) + assertEquals(1, run.engine.cancellations) + assertFalse(run.engine.output!!.exists()) + run.engine.complete() + run.engine.failures!!(SdrConversionFailure.Export) + run.idle() + assertEquals(1, run.results.size) + run.assertInputSafe() + } + @Test fun cancellationDuringPreflightDoesNotAllocateOrStartExport() { + lateinit var run: Run + run = Run(duringPreflight = { run.attempt.cancel() }) + run.idle() + assertEquals(listOf(SdrConversionResult.Cancelled), run.results) + assertNull(run.engine.output) + assertEquals(0, run.engine.starts) + run.assertInputSafe() + } @Test fun forbiddenFallbackWinsOverLaterCompletion() { val run = Run(); run.idle(); run.engine.failures!!(SdrConversionFailure.ForbiddenFallback); run.engine.complete(); run.idle() assertEquals(listOf(SdrConversionResult.Failed(SdrConversionFailure.ForbiddenFallback)), run.results)