Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 13 additions & 2 deletions src/app/recorder.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -276,9 +276,20 @@ export default function RecorderScreen() {
}, [device]);

// Pinned 1080p output + 30fps so every recorded clip is format-uniform (fast-path merge).
//
// `resolutionBias` FIRST, deliberately. VisionCamera picks the capture format by scoring every
// format against a weighted list of constraints — weight is `count - index`, so earlier entries
// outrank later ones — and it auto-appends one `{ resolutionBias: output }` per output, in the
// order the outputs are given. `<Camera>` puts its own preview output ahead of ours, and the
// preview asks for a format at least as large as the SCREEN. Every modern iPhone screen is taller
// than 1920px, so 1080p could never satisfy the preview and the vote elected 4K: clips shipped at
// 3840x2160 / ~23 Mbps while `useVideoOutput` below asked for 1080p / 5 Mbps. VisionCamera 5.2.0
// rescored that case so 1080p wins on its own, but only by ~13% on the largest screens — naming
// the video output's bias explicitly, at the top of the list, turns a margin into a mandate.
// See `logNegotiatedResolution` in use-recorder.ts for the runtime check that this held.
const constraints = useMemo<Constraint[]>(
() => [{ videoStabilizationMode: stabilization }, { fps: 30 }],
[stabilization],
() => [{ resolutionBias: videoOutput }, { videoStabilizationMode: stabilization }, { fps: 30 }],
[stabilization, videoOutput],
);
const outputs = useMemo(() => [videoOutput], [videoOutput]);

Expand Down
43 changes: 41 additions & 2 deletions src/features/recorder/use-recorder.ts
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,35 @@ import { generateThumbnailFile, getDurationMs } from '@/utils/video';
import CallDetector from '../../../modules/expo-call-detector/src/CallDetectorModule';
import { useCallState } from './use-call-state';

import { UPLOAD_MAX_LONG_EDGE } from '@/utils/upload-contract';

/**
* Report what the camera session ACTUALLY negotiated, and shout if it isn't what we asked for.
*
* `targetResolution` on the video output is a bias in a weighted vote across all outputs, not a
* setting — the preview output's preference outranked it and every clip came out 4K while the
* config said 1080p, silently, for weeks. The failure mode of a lost vote is indistinguishable
* from success unless something looks. This looks.
*
* Cheap (one property read on session start) and non-fatal by design: a device that can only
* offer something larger should still record, it just shouldn't do so unnoticed.
*/
function logNegotiatedResolution(output: {
currentResolution?: { width: number; height: number };
}) {
const size = output.currentResolution;
if (!size) return;
const longEdge = Math.max(size.width, size.height);
if (longEdge > UPLOAD_MAX_LONG_EDGE) {
console.warn(
`[recorder] capture format negotiated to ${size.width}x${size.height}, above the ${UPLOAD_MAX_LONG_EDGE} ` +
`long-edge target — clips will be re-encoded before upload. Check the resolutionBias constraint order.`,
);
return;
}
console.log(`[recorder] capture format: ${size.width}x${size.height}`);
}

// 'cinematic' is an iOS-only AVCaptureVideoStabilizationMode — CameraX has no equivalent, so
// Android only cycles through the modes it can actually honor. The union type keeps 'cinematic'
// on both platforms so persisted iOS prefs and shared UI maps still typecheck.
Expand Down Expand Up @@ -365,7 +394,13 @@ export function useRecorder(initialDraftId?: string) {
if (probe) {
const decision = decideImport(probe);
if (decision.action === 'normalize') {
const normalized = await compress(picked.uri, decision.options).catch(() => null);
const normalized = await compress(picked.uri, decision.options).catch((e: unknown) => {
// Falling back to the original bytes is the right call — a failed normalize should not
// block the import — but it must not be SILENT. This is how a 4K HDR master enters a
// draft looking exactly like a clip that was normalized successfully.
console.warn('[import] normalize failed; importing the original', decision.reasons, e);
return null;
});
if (normalized) {
normalizedPath = normalized.outputPath;
sourceUri = normalized.outputPath;
Expand Down Expand Up @@ -472,7 +507,11 @@ export function useRecorder(initialDraftId?: string) {
callActive,
appActive,
reportMicPriorityError,
onCameraReady: () => setCameraReady(true),
onCameraReady: () => {
// The session has started, so the output is attached and its negotiated format is readable.
logNegotiatedResolution(videoOutput);
setCameraReady(true);
},
toggleRecording,
finalizeRecording,
importClip: () => void importClip(),
Expand Down
44 changes: 42 additions & 2 deletions src/features/upload/upload-manager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import { getDraftToken } from '@/db/secure-token';
import { getDraftTranscriptRow } from '@/db/transcripts';
import { linesToVtt } from '@/features/transcription/vtt';
import { parseTranscriptLines } from '@/features/transcription/whisper';
import { ensureUploadContract, ensureUploadContractCached } from '@/utils/ensure-upload-contract';
import { absolutize, toFileUri } from '@/utils/file-store';
import { effFile } from '@/utils/segment-window';
import { generateThumbnailFile } from '@/utils/video';
Expand Down Expand Up @@ -578,6 +579,26 @@ class BackgroundUploadManager {
private async uploadMerged(session: UploadSession, signal: AbortSignal): Promise<string> {
const { draftId, destination, segments, merged } = session;
if (!merged) throw new Error('Export is not ready yet');

// Bring the video into the upload contract (H.264 / <=1920 long edge / <=5 Mbps / AAC) before
// anything reads its bytes. A compliant file costs one probe and is returned untouched; only a
// breach pays for a re-encode. This is the gate, NOT the export step: Share and Save-to-Photos
// read `state.outputPath` directly and must keep full capture quality — only what leaves the
// device for a browser to play is constrained.
const contract = await ensureUploadContract(merged.path);
if (contract.changed) {
// Persist the conditioned path so an after-kill resume re-uploads the SAME bytes. Without
// this, resume would re-encode from the original and a TUS PATCH could continue a transfer
// with bytes from a different encode. (Re-running the gate on an already-conditioned file is
// a no-op passthrough, so this is a cost saving as well as a correctness one.)
merged.path = contract.path;
await setUploadMerged(draftId, merged);
}
if (contract.failure) {
// Fail open, loudly: the upload proceeds with the original bytes, but it is not silent.
console.warn(`[contract] uploading unconditioned video for ${draftId}: ${contract.failure}`);
}

// merged.path is a bare filesystem path on Android (RNVT) — normalize to a file:// URI or the
// File API rejects it outright ("URI is not absolute").
const file = new File(toFileUri(merged.path));
Expand Down Expand Up @@ -652,7 +673,13 @@ class BackgroundUploadManager {
const result = await this.uploadOne(
draftId,
destination,
{ artifactId: destination.artifactId, filename: `${draftId}.mp4`, kind: 'video', name: draftName, file },
{
artifactId: destination.artifactId,
filename: `${draftId}.mp4`,
kind: 'video',
name: draftName,
file,
},
destination.resourceUrl,
checksum,
signal,
Expand Down Expand Up @@ -739,7 +766,20 @@ class BackgroundUploadManager {

for (const [index, segment] of segments.entries()) {
reportClip(index + 1, index);
const file = new File(absolutize(effFile(segment)));
// Same contract as the merged unit. `uploadUnit` is chosen by the DESTINATION SERVER, not
// by us, so leaving this path ungated would silently disable the contract for any server
// that asks for segments — the protection would look present and not run.
const contract = await ensureUploadContractCached(
absolutize(effFile(segment)),
draftId,
segment.id,
);
if (contract.failure) {
console.warn(
`[contract] uploading unconditioned clip ${segment.id} for ${draftId}: ${contract.failure}`,
);
}
const file = new File(toFileUri(contract.path));
const checksum = await md5Checksum(file);

const videoKey = `${segment.id}:video` as const;
Expand Down
113 changes: 113 additions & 0 deletions src/utils/ensure-upload-contract.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
import { File } from 'expo-file-system';
import { compress, probeVideo } from 'react-native-video-trim';

import { toFileUri, uploadDest } from './file-store';
import { decideUploadContract } from './upload-contract';

/**
* What conditioning did to a file on its way to being uploaded.
*
* `path` is always usable — on any failure it falls back to the input, so a broken probe
* or a failed encode degrades to "upload the original", never to "upload nothing".
*/
export type ContractResult = {
/** The file to upload: the conditioned copy, or the input when nothing was needed. */
path: string;
/** True when `path` differs from the input. */
changed: boolean;
/** Human-readable contract breaches that triggered the re-encode, for logging/UI. */
reasons: string[];
/**
* Set when the file could NOT be brought into the contract and the original is being
* uploaded instead. Never silently empty — the whole point of this gate is that a
* failure is visible. (`importClip` swallows exactly this case today, which is how a
* 4K master can still enter a draft.)
*/
failure?: string;
};

/**
* Bring a file into the upload contract before it is uploaded: probe it, and re-encode
* only if it breaches (see {@link decideUploadContract}).
*
* A compliant file is returned untouched — no copy, no re-encode, no quality generation
* spent. That is the intended steady state once the recorder emits 1080p/5 Mbps: this
* gate costs one probe and nothing else. It exists for the case where the recorder's
* format negotiation loses on some device we have not tested, or an import slips a 4K
* master through — outcomes we cannot prevent, only catch.
*
* Failure policy is deliberately "fail open, loudly": an upload that happens at reduced
* quality is better than an upload that does not happen, but it must be reported rather
* than absorbed.
*/
export async function ensureUploadContract(path: string): Promise<ContractResult> {
const probe = await probeVideo(path).catch((e: unknown) => {
console.warn('[contract] probe failed; uploading the original', e);
return null;
});
if (!probe) {
return { path, changed: false, reasons: [], failure: 'could not probe the file' };
}

const decision = decideUploadContract(probe);
if (decision.action === 'passthrough') {
return { path, changed: false, reasons: [] };
}

const result = await compress(path, { ...decision.options, outputExt: 'mp4' }).catch(
(e: unknown) => {
console.warn('[contract] re-encode failed; uploading the original', decision.reasons, e);
return null;
},
);
if (!result) {
return {
path,
changed: false,
reasons: decision.reasons,
failure: `could not re-encode (${decision.reasons.join(', ')})`,
};
}

return { path: result.outputPath, changed: true, reasons: decision.reasons };
}

/**
* {@link ensureUploadContract} with a stable, reusable output location — the form the SEGMENT
* upload path needs.
*
* Segment uploads resume byte-wise (TUS `HEAD` for the offset, then `PATCH` from there), so a run
* that resumes must send exactly the bytes it began with. Re-running a re-encode would produce a
* second, subtly different encode and splice it into a half-finished transfer. Conditioning into a
* fixed per-clip path means a resumed run finds the file it already made and reuses it — correct
* first, and a saved re-encode second.
*
* The merged path solves the same problem differently: it persists the conditioned path in the
* draft row, which segments have no column for.
*/
export async function ensureUploadContractCached(
sourcePath: string,
draftId: string,
segmentId: string,
): Promise<ContractResult> {
const dest = uploadDest(draftId, segmentId);
if (dest.exists && (dest.size ?? 0) > 0) {
// Already conditioned on an earlier attempt — reuse verbatim.
return { path: dest.uri, changed: true, reasons: [] };
}

const result = await ensureUploadContract(sourcePath);
if (!result.changed) return result;

try {
// compress() writes into the OS-purgeable cache dir; move it somewhere a resume can find it.
await new File(toFileUri(result.path)).move(dest);
return { ...result, path: dest.uri };
} catch (e) {
// The conditioned bytes exist but could not be parked. Upload them from where they are
// rather than falling back to the oversized original; a resume may re-encode, which is
// worse than this but still better than uploading 4K.
console.warn('[contract] could not park the conditioned clip; using the cache copy', e);
return result;
}
}
25 changes: 23 additions & 2 deletions src/utils/file-store.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,14 @@ import { Directory, File, Paths } from 'expo-file-system';
// between launches without invalidating references (§2.2).
// drafts/{draftId}/segments/{segmentId}.mp4 — pristine original
// drafts/{draftId}/segments/{segmentId}.edited.{rev}.mp4 — re-encoded editor output
// drafts/{draftId}/upload/{segmentId}.mp4 — upload-contract copy (see uploadDir)

/**
* Normalize a bare filesystem path to a `file://` URI. `merge()` / `getFrameAt` / the camera hand
* back bare paths, but expo's `File`, expo-video, whisper, sharing, etc. all want a URI. A value
* that already has a scheme is returned unchanged.
*/
export const toFileUri = (path: string): string =>
path.startsWith('/') ? `file://${path}` : path;
export const toFileUri = (path: string): string => (path.startsWith('/') ? `file://${path}` : path);

export function segmentRelPath(draftId: string, segmentId: string): string {
return `drafts/${draftId}/segments/${segmentId}.mp4`;
Expand Down Expand Up @@ -55,6 +55,27 @@ function segmentsDir(draftId: string): Directory {
return dir;
}

/**
* The draft's conditioned-upload dir, creating it (and any missing parents) if needed.
*
* A STABLE location inside the draft's own directory, not a cache temp name. Segment uploads
* resume byte-wise (TUS HEAD + PATCH), so a resumed run must re-send the exact bytes it started
* with — re-encoding on resume would splice a second encode into a half-finished transfer. A
* fixed path means the conditioned file is found and reused instead. Living under
* `drafts/{draftId}/` means `deleteDraftDir` reclaims it with the draft, and it sits beside
* `segments/` rather than inside it so it can never be mistaken for a clip.
*/
function uploadDir(draftId: string): Directory {
const dir = new Directory(Paths.document, 'drafts', draftId, 'upload');
dir.create({ intermediates: true, idempotent: true });
return dir;
}

/** The on-disk conditioned upload copy for a clip, creating the upload dir if needed. */
export function uploadDest(draftId: string, segmentId: string): File {
return new File(uploadDir(draftId), `${segmentId}.mp4`);
}

/** The on-disk pristine segment file for a draft, creating the segments dir if needed. */
function segmentDest(draftId: string, segmentId: string): File {
return new File(segmentsDir(draftId), `${segmentId}.mp4`);
Expand Down
8 changes: 4 additions & 4 deletions src/utils/import-normalization.ts
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ export const NORMALIZE_MAX_BITRATE = 8_000_000;
/** Video codecs the merge pipeline handles natively (hardware decode on both platforms). */
const NATIVE_VIDEO_CODECS = new Set(['h264', 'hevc']);
/** HDR transfer functions: HLG (iPhone camera default) and PQ (HDR10 / Dolby Vision 8.x). */
const HDR_TRANSFERS = new Set(['arib-std-b67', 'smpte2084']);
export const HDR_TRANSFERS = new Set(['arib-std-b67', 'smpte2084']);

export type ImportDecision =
| { action: 'passthrough' }
Expand All @@ -49,17 +49,17 @@ export type ImportDecision =
* suffix (yuv420p10le, p010le, ...) — matching the suffix rather than a bare `includes('10')`
* keeps 8-bit chroma-subsampling names like `yuv410p` from being misclassified.
*/
function is10Bit(pixelFormat: string): boolean {
export function is10Bit(pixelFormat: string): boolean {
return /10(le|be)?$/.test(pixelFormat);
}

/** Effective fps for the decision: average when known (catches VFR), else nominal. */
function effectiveFps(probe: VideoProbeResult): number {
export function effectiveFps(probe: VideoProbeResult): number {
return probe.averageFps > 0 ? probe.averageFps : probe.nominalFps;
}

/** Display (post-rotation) dimensions: a 90/270 rotation swaps coded width/height. */
function displaySize(probe: VideoProbeResult): { width: number; height: number } {
export function displaySize(probe: VideoProbeResult): { width: number; height: number } {
const swapped = probe.rotation % 180 !== 0;
return {
width: swapped ? probe.height : probe.width,
Expand Down
Loading