Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions crates/enc-ffmpeg/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ pub use mux::*;

pub mod decode_limit;
pub mod fragment_order;
pub mod mp4_boxes;
mod relocatable_source;
pub mod remux;
pub use relocatable_source::{RelocatableReader, RelocatableSource};
Expand Down
105 changes: 105 additions & 0 deletions crates/enc-ffmpeg/src/mp4_boxes.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
use std::path::Path;

const MAX_MOOV_BYTES: u64 = 64 * 1024 * 1024;

fn box_header(data: &[u8]) -> Option<([u8; 4], usize, usize)> {
let size = u32::from_be_bytes(data.get(0..4)?.try_into().ok()?) as u64;
let kind: [u8; 4] = data.get(4..8)?.try_into().ok()?;
let (size, header) = match size {
0 => (data.len() as u64, 8),
1 => (u64::from_be_bytes(data.get(8..16)?.try_into().ok()?), 16),
size => (size, 8),
};
let size = usize::try_from(size).ok()?;
(size >= header && size <= data.len()).then_some((kind, header, size))
}

fn child_boxes(mut data: &[u8]) -> impl Iterator<Item = ([u8; 4], &[u8])> {
std::iter::from_fn(move || {
let (kind, header, size) = box_header(data)?;
let payload = &data[header..size];
data = &data[size..];
Some((kind, payload))
})
}

fn child_box<'a>(data: &'a [u8], kind: &[u8; 4]) -> Option<&'a [u8]> {
child_boxes(data)
.find(|(child, _)| child == kind)
.map(|(_, payload)| payload)
}

fn read_be_u32(data: &[u8], at: usize) -> Option<u32> {
Some(u32::from_be_bytes(data.get(at..at + 4)?.try_into().ok()?))
}

fn read_moov(path: &Path) -> Option<Vec<u8>> {
use std::io::{Read, Seek, SeekFrom};

let mut file = std::fs::File::open(path).ok()?;
let file_len = file.metadata().ok()?.len();
let mut offset = 0u64;
let mut moov = None;
while offset + 8 <= file_len {
file.seek(SeekFrom::Start(offset)).ok()?;
let mut header = [0u8; 16];
let available = (file_len - offset).min(16) as usize;
file.read_exact(&mut header[..available]).ok()?;
let kind: [u8; 4] = header[4..8].try_into().ok()?;
let (size, header_len) = match u32::from_be_bytes(header[0..4].try_into().ok()?) {
0 => (file_len - offset, 8),
1 if available == 16 => (u64::from_be_bytes(header[8..16].try_into().ok()?), 16),
1 => return None,
size => (u64::from(size), 8),
};
if size < header_len || offset + size > file_len {
return None;
}
match &kind {
b"moof" => return None,
b"moov" if moov.is_none() && size <= MAX_MOOV_BYTES => {
let mut data = vec![0u8; size as usize];
file.seek(SeekFrom::Start(offset)).ok()?;
file.read_exact(&mut data).ok()?;
moov = Some(data);
}
b"moov" => return None,
_ => {}
}
offset += size;
}
moov
}

// The sample table stores decode times, so it only matches presentation times
// when every composition offset in the track is zero: the track has no `ctts`
// box, or all of its entries are zero. Fragmented files keep offsets per
// fragment instead and are never eligible.
pub fn track_has_no_composition_offsets(path: &Path, track_id: i32) -> bool {
let check = || -> Option<bool> {
let moov_box = read_moov(path)?;
let (_, header, size) = box_header(&moov_box)?;
let moov = &moov_box[header..size];
if child_box(moov, b"mvex").is_some() {
return None;
}
let track = child_boxes(moov)
.filter(|(kind, _)| kind == b"trak")
.map(|(_, trak)| trak)
.find(|trak| {
child_box(trak, b"tkhd").and_then(|tkhd| {
let id_at = if tkhd.first() == Some(&1) { 20 } else { 12 };
read_be_u32(tkhd, id_at)
}) == Some(track_id as u32)
})?;
let stbl = child_box(child_box(child_box(track, b"mdia")?, b"minf")?, b"stbl")?;
let Some(ctts) = child_box(stbl, b"ctts") else {
return Some(true);
};
let entries = read_be_u32(ctts, 4)? as usize;
(0..entries)
.map(|entry| read_be_u32(ctts, 8 + entry * 8 + 4))
.try_fold(true, |all_zero, offset| Some(all_zero && offset? == 0))
};
check().unwrap_or(false)
}
141 changes: 133 additions & 8 deletions crates/enc-ffmpeg/src/remux.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1006,11 +1006,11 @@ pub fn get_video_capture_fps(path: &Path) -> Option<u32> {
result
}

fn get_video_capture_fps_inner(path: &Path) -> Option<u32> {
const MAX_PACKETS: usize = 20_000;
const CAPTURE_FPS_MAX_PACKETS: usize = 20_000;

fn get_video_capture_fps_inner(path: &Path) -> Option<u32> {
let mut ictx = avformat::input(path).ok()?;
let (stream_index, time_base) = {
let (stream_index, time_base, indexed) = {
let stream = ictx.streams().best(ffmpeg::media::Type::Video)?;
let tb = stream.time_base();
if tb.numerator() <= 0 || tb.denominator() <= 0 {
Expand All @@ -1019,21 +1019,60 @@ fn get_video_capture_fps_inner(path: &Path) -> Option<u32> {
(
stream.index(),
f64::from(tb.numerator()) / f64::from(tb.denominator()),
indexed_video_timestamps(path, &stream),
)
};
let timestamps = indexed.unwrap_or_else(|| packet_video_timestamps(&mut ictx, stream_index));
capture_fps_from_timestamps(
timestamps
.into_iter()
.map(|timestamp| timestamp as f64 * time_base)
.collect(),
)
}

fn packet_video_timestamps(ictx: &mut avformat::context::Input, stream_index: usize) -> Vec<i64> {
let mut timestamps = Vec::new();
for (stream, packet) in ictx.packets() {
if stream.index() != stream_index {
continue;
}
if let Some(pts) = packet.pts().or(packet.dts()) {
timestamps.push(pts as f64 * time_base);
if let Some(timestamp) = packet.pts().or(packet.dts()) {
timestamps.push(timestamp);
}
if timestamps.len() >= MAX_PACKETS {
if timestamps.len() >= CAPTURE_FPS_MAX_PACKETS {
break;
}
}
capture_fps_from_timestamps(timestamps)
timestamps
}

fn indexed_video_timestamps(path: &Path, stream: &avformat::stream::Stream) -> Option<Vec<i64>> {
if !crate::mp4_boxes::track_has_no_composition_offsets(path, stream.id()) {
return None;
}
unsafe {
let stream = stream.as_ptr().cast_mut();
if (*(*stream).codecpar).video_delay > 0 {
return None;
}
let count = ffmpeg::ffi::avformat_index_get_entries_count(stream);
if count <= 0 || i64::from(count) != (*stream).nb_frames {
return None;
}
let mut timestamps = Vec::with_capacity((count as usize).min(CAPTURE_FPS_MAX_PACKETS));
for index in 0..count {
let entry = ffmpeg::ffi::avformat_index_get_entry(stream, index);
if entry.is_null() || (*entry).flags() & ffmpeg::ffi::AVINDEX_DISCARD_FRAME != 0 {
return None;
}
timestamps.push((*entry).timestamp);
if timestamps.len() >= CAPTURE_FPS_MAX_PACKETS {
break;
}
}
Some(timestamps)
}
}

/// The shortest gap between frames that recurs in runs, in 5% bins, refined
Expand Down Expand Up @@ -1500,7 +1539,8 @@ mod tests {
concatenate_m4s_segments_with_init_validated,
concatenate_m4s_segments_with_init_validated_with_reader, concatenate_video_fragments,
concatenate_video_fragments_on_timeline, get_video_capture_fps, get_video_fps,
probe_video_pts_ladder, write_concat_list,
indexed_video_timestamps, packet_video_timestamps, probe_video_pts_ladder,
write_concat_list,
};
use std::{
path::{Path, PathBuf},
Expand Down Expand Up @@ -2304,6 +2344,91 @@ mod tests {
);
}

fn indexed_and_scanned_timestamps(path: &Path) -> (Option<Vec<i64>>, Vec<i64>) {
let mut input = ffmpeg::format::input(path).unwrap();
let (stream_index, indexed) = {
let stream = input.streams().best(ffmpeg::media::Type::Video).unwrap();
(stream.index(), indexed_video_timestamps(path, &stream))
};
let mut scanned = packet_video_timestamps(&mut input, stream_index);
scanned.sort_unstable();
(
indexed.map(|mut indexed| {
indexed.sort_unstable();
indexed
}),
scanned,
)
}

#[test]
fn indexed_timestamps_match_a_full_packet_scan() {
let constant: Vec<Duration> = (0..300u64)
.map(|i| Duration::from_nanos(i * 1_000_000_000 / 30))
.collect();
let jittered: Vec<Duration> = (0..300u64)
.map(|i| Duration::from_secs_f64(i as f64 / 60.0 + 0.002 + jitter(i, 0.002)))
.collect();
let paused: Vec<Duration> = (0..120u64)
.map(|i| {
let gap = if i >= 60 { 5_000_000_000u64 } else { 0 };
Duration::from_nanos(i * 1_000_000_000 / 30 + gap)
})
.collect();
for timestamps in [bursty_timestamps(20, 6, 30), constant, jittered, paused] {
let dir = tempfile::tempdir().unwrap();
let path = encode_test_mp4(dir.path(), &timestamps);
let (indexed, scanned) = indexed_and_scanned_timestamps(&path);
assert_eq!(scanned.len(), timestamps.len());
assert_eq!(indexed, Some(scanned));
}
}

#[test]
fn varying_composition_offsets_fall_back_to_reading_packets() {
let fixtures = Path::new(env!("CARGO_MANIFEST_DIR")).join("../video-decode/tests/fixtures");

let varying = fixtures.join("h264-varying-composition-offset.mp4");
let (indexed, scanned) = indexed_and_scanned_timestamps(&varying);
assert_eq!(indexed, None);
let mut input = ffmpeg::format::input(&varying).unwrap();
let (stream_index, time_base) = {
let stream = input.streams().best(ffmpeg::media::Type::Video).unwrap();
(stream.index(), f64::from(stream.time_base()))
};
assert_eq!(packet_video_timestamps(&mut input, stream_index), scanned);
assert_eq!(
get_video_capture_fps(&varying),
capture_fps_from_timestamps(
scanned
.iter()
.map(|timestamp| *timestamp as f64 * time_base)
.collect()
)
);

let plain = fixtures.join("h264-no-reordering.mp4");
let (indexed, scanned) = indexed_and_scanned_timestamps(&plain);
assert_eq!(indexed, Some(scanned));
}

#[test]
fn a_fragmented_track_falls_back_to_reading_packets() {
let dir = tempfile::tempdir().unwrap();
let (init, segments) = encode_test_frames(dir.path(), 120);
let path = dir.path().join("fragmented.mp4");
let mut bytes = std::fs::read(&init).unwrap();
for segment in &segments {
bytes.extend(std::fs::read(segment).unwrap());
}
std::fs::write(&path, bytes).unwrap();

let (indexed, scanned) = indexed_and_scanned_timestamps(&path);
assert_eq!(indexed, None);
assert_eq!(scanned.len(), 120);
assert_eq!(get_video_capture_fps(&path), Some(30));
}

#[test]
fn capture_fps_reads_an_encoded_variable_rate_track() {
let dir = tempfile::tempdir().unwrap();
Expand Down
42 changes: 42 additions & 0 deletions crates/recording/examples/track-heal-benchmark.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
use std::{path::PathBuf, time::Instant};

use cap_recording::track_heal::heal_stretched_tracks;

fn read_bytes() -> Option<u64> {
std::fs::read_to_string("/proc/self/io")
.ok()?
.lines()
.find_map(|line| line.strip_prefix("rchar: "))
.and_then(|value| value.trim().parse().ok())
}

fn main() -> anyhow::Result<()> {
let mut args = std::env::args_os().skip(1);
let path = args.next().map(PathBuf::from).ok_or_else(|| {
anyhow::anyhow!("Pass a Studio project (.cap) and optionally a run count")
})?;
let runs: usize = args
.next()
.and_then(|runs| runs.to_str()?.parse().ok())
.unwrap_or(1);
ffmpeg::init()?;
for run in 0..runs {
let bytes_before = read_bytes();
let started = Instant::now();
let healed = heal_stretched_tracks(&path)?;
let elapsed = started.elapsed();
let bytes_read = bytes_before
.zip(read_bytes())
.map(|(before, after)| after - before);
println!(
"{}",
serde_json::json!({
"run": run,
"elapsedMs": elapsed.as_secs_f64() * 1000.0,
"bytesRead": bytes_read,
"healed": healed,
})
);
}
Ok(())
}
29 changes: 29 additions & 0 deletions crates/recording/src/track_heal.rs
Original file line number Diff line number Diff line change
Expand Up @@ -852,6 +852,35 @@ mod tests {
assert_eq!(stored_display_fps(project.path()), 30);
}

#[test]
fn a_jittered_variable_rate_display_at_its_average_is_healed() {
let timestamps: Vec<Duration> = (0..24u64)
.flat_map(|burst| (0..5).map(move |i| (burst, burst * 20 + i)))
.map(|(burst, slot)| {
let jitter_us = (burst * 7919 + slot * 104_729) % 2_000;
Duration::from_nanos(slot * 1_000_000_000 / 30 + jitter_us * 1_000)
})
.collect();
let project = display_project(&timestamps, |display| get_video_fps(display).unwrap());
let average = stored_display_fps(project.path());
assert!(average < 10, "fixture averages {average} fps");

assert!(heal_average_display_fps(project.path()).unwrap());
assert_eq!(stored_display_fps(project.path()), 30);
assert!(!heal_average_display_fps(project.path()).unwrap());
assert_eq!(stored_display_fps(project.path()), 30);
}

#[test]
fn a_display_fps_that_differs_from_its_track_is_left_alone() {
let timestamps: Vec<Duration> = (0..180u64)
.map(|i| Duration::from_nanos(i * 1_000_000_000 / 60))
.collect();
let project = display_project(&timestamps, |_| 30);
assert!(!heal_average_display_fps(project.path()).unwrap());
assert_eq!(stored_display_fps(project.path()), 30);
}

const USER_REPORT_CAMERA_SECS: f64 = 13.666;

fn user_report_references() -> Vec<(&'static str, f64)> {
Expand Down
Loading
Loading