Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 59 additions & 0 deletions crates/enc-ffmpeg/src/mux/segmented_stream.rs
Original file line number Diff line number Diff line change
Expand Up @@ -334,6 +334,7 @@ pub struct SegmentedVideoEncoderConfig {
/// [`SegmentedVideoEncoder::queue_hw_pixel_buffer`]) before falling back
/// to the software-frame path. Ignored on other platforms.
pub prefer_videotoolbox_hw_input: bool,
pub realtime_videotoolbox_keyframe_interval: Option<Duration>,
}

impl Default for SegmentedVideoEncoderConfig {
Expand All @@ -344,6 +345,7 @@ impl Default for SegmentedVideoEncoderConfig {
bpp: H264EncoderBuilder::QUALITY_BPP,
output_size: None,
prefer_videotoolbox_hw_input: false,
realtime_videotoolbox_keyframe_interval: None,
}
}
}
Expand Down Expand Up @@ -442,6 +444,9 @@ impl SegmentedVideoEncoder {
if let Some(priority) = encoder_priority {
builder = builder.with_encoder_priority_override(priority);
}
if let Some(interval) = config.realtime_videotoolbox_keyframe_interval {
builder = builder.with_realtime_videotoolbox_keyframe_interval(interval);
}

if let Some((width, height)) = config.output_size {
builder = builder.with_output_size(width, height)?;
Expand Down Expand Up @@ -2001,6 +2006,60 @@ mod tests {
assert!(crate::remux::probe_video_can_decode(&remuxed_path).unwrap_or(false));
}

#[cfg(target_os = "macos")]
#[test]
fn realtime_videotoolbox_keyframe_interval_keeps_two_second_segments() {
ffmpeg::init().ok();
let encode = |interval: Option<Duration>| {
let temp = tempfile::tempdir().unwrap();
let base_path = temp.path().to_path_buf();
let mut encoder = SegmentedVideoEncoder::init(
base_path.clone(),
test_video_info(),
SegmentedVideoEncoderConfig {
realtime_videotoolbox_keyframe_interval: interval,
..Default::default()
},
)
.unwrap();
if encoder.encoder.codec_name() != "h264_videotoolbox" {
return None;
}
for i in 0..200u64 {
let mut frame = create_test_frame(320, 240);
frame.data_mut(0)[(i as usize * 7) % 76_800] = (i % 251) as u8;
encoder
.queue_frame(frame, Duration::from_nanos(i * 33_333_333))
.unwrap();
}
encoder.finish().unwrap();
let segments = std::fs::read_dir(&base_path)
.unwrap()
.filter_map(|e| e.ok().map(|e| e.path()))
.filter(|p| p.extension().is_some_and(|ext| ext == "m4s"))
.count();
let track = recorded_track(&base_path);
let mut input = format::input(&track).unwrap();
let index = input
.streams()
.best(ffmpeg::media::Type::Video)
.unwrap()
.index();
let keyframes = input
.packets()
.filter(|(stream, packet)| stream.index() == index && packet.is_key())
.count();
Some((segments, keyframes, temp))
};
let Some((segments_default, keyframes_default, _a)) = encode(None) else {
return;
};
let (segments_1s, keyframes_1s, _b) = encode(Some(Duration::from_secs(1))).unwrap();
assert_eq!(segments_default, segments_1s);
assert!(keyframes_default >= 15, "{keyframes_default}");
assert!((6..=8).contains(&keyframes_1s), "{keyframes_1s}");
}

fn recorded_track(base_path: &Path) -> PathBuf {
let mut segment_paths: Vec<PathBuf> = std::fs::read_dir(base_path)
.unwrap()
Expand Down
113 changes: 113 additions & 0 deletions crates/enc-ffmpeg/src/video/h264.rs
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ pub struct H264EncoderBuilder {
is_export: bool,
crf: Option<u8>,
software_tuning: Option<SoftwareEncoderTuning>,
realtime_videotoolbox_keyframe_interval: Option<Duration>,
}

/// libx264 settings for an export split across several encoders running at
Expand Down Expand Up @@ -114,6 +115,7 @@ impl H264EncoderBuilder {
is_export: false,
crf: None,
software_tuning: None,
realtime_videotoolbox_keyframe_interval: None,
}
}

Expand Down Expand Up @@ -158,6 +160,14 @@ impl H264EncoderBuilder {
self
}

/// Keyframe interval for realtime `h264_videotoolbox` recording. Without
/// it VideoToolbox gets FFmpeg's default GOP of 12 frames. No other
/// encoder, and no export, reads this.
pub fn with_realtime_videotoolbox_keyframe_interval(mut self, interval: Duration) -> Self {
self.realtime_videotoolbox_keyframe_interval = Some(interval);
self
}

pub fn with_crf(mut self, crf: u8) -> Self {
self.crf = Some(crf);
self
Expand Down Expand Up @@ -196,6 +206,7 @@ impl H264EncoderBuilder {
self.encoder_priority_override,
self.is_export,
self.crf,
self.realtime_videotoolbox_keyframe_interval,
);
if candidates.is_empty() {
return Err(H264EncoderError::CodecNotFound);
Expand Down Expand Up @@ -423,6 +434,7 @@ impl H264EncoderBuilder {
Some(&["libx264"]),
self.is_export,
self.crf,
self.realtime_videotoolbox_keyframe_interval,
)
.into_iter()
.next()
Expand Down Expand Up @@ -511,6 +523,7 @@ impl H264EncoderBuilder {
Some(&["h264_videotoolbox"]),
self.is_export,
None,
self.realtime_videotoolbox_keyframe_interval,
)
.into_iter()
.next()
Expand Down Expand Up @@ -615,6 +628,7 @@ impl H264EncoderBuilder {
self.encoder_priority_override,
self.is_export,
self.crf,
self.realtime_videotoolbox_keyframe_interval,
);
if candidates.is_empty() {
return Err(H264EncoderError::CodecNotFound);
Expand Down Expand Up @@ -1661,6 +1675,7 @@ fn get_codec_and_options(
encoder_priority_override: Option<&'static [&'static str]>,
is_export: bool,
crf: Option<u8>,
realtime_videotoolbox_keyframe_interval: Option<Duration>,
) -> Vec<(Codec, Dictionary<'static>)> {
let keyframe_interval_secs = DEFAULT_KEYFRAME_INTERVAL_SECS;
let denominator = config.frame_rate.denominator();
Expand Down Expand Up @@ -1701,6 +1716,10 @@ fn get_codec_and_options(
options.set("realtime", "true");
options.set("prio_speed", "true");
options.set("profile", "main");
if let Some(interval) = realtime_videotoolbox_keyframe_interval {
let frames = (interval.as_secs_f64() * frames_per_sec).round().max(1.0);
options.set("g", &(frames as i32).to_string());
}
}
}
"h264_nvenc" => {
Expand Down Expand Up @@ -2213,6 +2232,95 @@ mod self_test_tests {
.expect("healthy encoder passes the round trip");
}

fn options_by_encoder(
config: &VideoInfo,
is_export: bool,
interval: Option<Duration>,
) -> Vec<(String, Vec<(String, String)>)> {
let names: &'static [&'static str] = &[
"h264_videotoolbox",
"h264_nvenc",
"h264_qsv",
"h264_amf",
"h264_mf",
"h264_vaapi",
"libx264",
];
get_codec_and_options(
config,
H264Preset::Ultrafast,
Some(names),
is_export,
None,
interval,
)
.into_iter()
.map(|(codec, options)| {
(
codec.name().to_string(),
options
.iter()
.map(|(k, v)| (k.to_string(), v.to_string()))
.collect(),
)
})
.collect()
}

#[test]
fn realtime_keyframe_interval_only_changes_realtime_videotoolbox() {
ffmpeg::init().ok();
let config = VideoInfo::from_raw(cap_media_info::RawVideoFormat::Nv12, 1920, 1080, 60);
let interval = Some(Duration::from_secs(1));
for is_export in [false, true] {
let without = options_by_encoder(&config, is_export, None);
let with = options_by_encoder(&config, is_export, interval);
assert_eq!(
without.iter().map(|(n, _)| n).collect::<Vec<_>>(),
with.iter().map(|(n, _)| n).collect::<Vec<_>>()
);
for ((name, base), (_, head)) in without.iter().zip(with.iter()) {
if name == "h264_videotoolbox" && !is_export {
continue;
}
assert_eq!(base, head, "{name} export={is_export}");
}
}
}

#[cfg(target_os = "macos")]
#[test]
fn realtime_videotoolbox_keyframe_interval_sets_gop_in_frames() {
if force_software_encoder() {
return;
}
ffmpeg::init().ok();
let realtime_gop = |fps: u32, interval: Option<Duration>| {
let config = VideoInfo::from_raw(cap_media_info::RawVideoFormat::Nv12, 1920, 1080, fps);
options_by_encoder(&config, false, interval)
.into_iter()
.find(|(name, _)| name == "h264_videotoolbox")
.expect("VideoToolbox encoder is available on macOS")
.1
.into_iter()
.find(|(k, _)| k == "g")
.map(|(_, v)| v)
};
assert_eq!(realtime_gop(60, None), None);
assert_eq!(
realtime_gop(60, Some(Duration::from_secs(1))).as_deref(),
Some("60")
);
assert_eq!(
realtime_gop(30, Some(Duration::from_secs(1))).as_deref(),
Some("30")
);
assert_eq!(
realtime_gop(60, Some(Duration::from_secs(2))).as_deref(),
Some("120")
);
}

#[cfg(target_os = "macos")]
#[test]
fn videotoolbox_export_uses_the_default_keyframe_interval() {
Expand All @@ -2228,6 +2336,7 @@ mod self_test_tests {
Some(&["h264_videotoolbox"]),
is_export,
None,
None,
)
.into_iter()
.find(|(codec, _)| codec.name() == "h264_videotoolbox")
Expand Down Expand Up @@ -2328,6 +2437,7 @@ mod self_test_tests {
Some(&["h264_mf"]),
false,
None,
None,
)
.into_iter()
.next()
Expand Down Expand Up @@ -2423,6 +2533,7 @@ mod self_test_tests {
Some(linux_encoder_priority(true, true)),
true,
Some(23),
None,
);

assert_eq!(codecs.len(), 1);
Expand All @@ -2447,6 +2558,7 @@ mod self_test_tests {
Some(&["h264_nvenc", "libx264"]),
false,
None,
None,
)
.into_iter()
.find(|(candidate, _)| candidate.name() == "h264_nvenc")
Expand Down Expand Up @@ -2475,6 +2587,7 @@ mod self_test_tests {
Some(&["h264_vaapi", "libx264"]),
false,
None,
None,
)
.into_iter()
.find(|(candidate, _)| candidate.name() == "h264_vaapi")
Expand Down
10 changes: 9 additions & 1 deletion crates/recording/src/capture_pipeline.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,10 @@ use crate::{
};

#[cfg(target_os = "macos")]
use crate::output_pipeline::{MacOSFragmentedM4SMuxer, MacOSFragmentedM4SMuxerConfig};
use crate::output_pipeline::{
MacOSFragmentedM4SMuxer, MacOSFragmentedM4SMuxerConfig,
STUDIO_DISPLAY_VIDEOTOOLBOX_KEYFRAME_INTERVAL,
};
#[cfg(windows)]
use crate::output_pipeline::{WindowsFragmentedM4SMuxer, WindowsFragmentedM4SMuxerConfig};
use anyhow::anyhow;
Expand Down Expand Up @@ -178,6 +181,11 @@ impl MakeCapturePipeline for screen_capture::CMSampleBufferCapture {
.with_start_gate(start_gate.clone())
.build::<MacOSFragmentedM4SMuxer>(MacOSFragmentedM4SMuxerConfig {
timed_fragments: true,
// Ultra keeps the 12-frame GOP for now: with longer GOPs
// VideoToolbox reaches Ultra's much higher bitrate target
// on motion and those recordings grow instead of shrink.
realtime_videotoolbox_keyframe_interval: (!ultra)
.then_some(STUDIO_DISPLAY_VIDEOTOOLBOX_KEYFRAME_INTERVAL),
preset,
bpp,
output_size,
Expand Down
1 change: 1 addition & 0 deletions crates/recording/src/output_pipeline/ffmpeg.rs
Original file line number Diff line number Diff line change
Expand Up @@ -690,6 +690,7 @@ impl SegmentedVideoMuxer {
bpp: H264EncoderBuilder::QUALITY_BPP,
output_size: self.output_size,
prefer_videotoolbox_hw_input: false,
realtime_videotoolbox_keyframe_interval: None,
};

let slow_threshold_ms = frame_timing_log_threshold_ms(&self.video_config);
Expand Down
11 changes: 11 additions & 0 deletions crates/recording/src/output_pipeline/macos_fragmented_m4s.rs
Original file line number Diff line number Diff line change
Expand Up @@ -225,6 +225,7 @@ struct EncoderState {

pub struct MacOSFragmentedM4SMuxer {
timed_fragments: bool,
realtime_videotoolbox_keyframe_interval: Option<Duration>,
base_path: PathBuf,
video_config: VideoInfo,
segment_duration: Duration,
Expand All @@ -240,8 +241,14 @@ pub struct MacOSFragmentedM4SMuxer {
health_tx: SharedHealthSender,
}

/// Studio screen recordings set an explicit keyframe interval on VideoToolbox;
/// left unset it falls back to FFmpeg's 12-frame GOP, which puts most of the
/// file's bytes into keyframes.
pub const STUDIO_DISPLAY_VIDEOTOOLBOX_KEYFRAME_INTERVAL: Duration = Duration::from_secs(1);

pub struct MacOSFragmentedM4SMuxerConfig {
pub timed_fragments: bool,
pub realtime_videotoolbox_keyframe_interval: Option<Duration>,
pub segment_duration: Duration,
pub preset: H264Preset,
pub bpp: f32,
Expand All @@ -255,6 +262,7 @@ impl Default for MacOSFragmentedM4SMuxerConfig {
fn default() -> Self {
Self {
timed_fragments: false,
realtime_videotoolbox_keyframe_interval: None,
segment_duration: Duration::from_secs(2),
preset: H264Preset::Ultrafast,
bpp: H264EncoderBuilder::QUALITY_BPP,
Expand Down Expand Up @@ -292,6 +300,7 @@ impl Muxer for MacOSFragmentedM4SMuxer {

Ok(Self {
timed_fragments: config.timed_fragments,
realtime_videotoolbox_keyframe_interval: config.realtime_videotoolbox_keyframe_interval,
base_path: output_path,
video_config,
segment_duration: config.segment_duration,
Expand Down Expand Up @@ -368,6 +377,7 @@ impl MacOSFragmentedM4SMuxer {
bpp: self.bpp,
output_size: self.output_size,
prefer_videotoolbox_hw_input: true,
realtime_videotoolbox_keyframe_interval: self.realtime_videotoolbox_keyframe_interval,
};

let mut encoder = SegmentedVideoEncoder::init_write_behind(
Expand Down Expand Up @@ -771,6 +781,7 @@ impl MacOSFragmentedM4SCameraMuxer {
bpp: self.bpp,
output_size: self.output_size,
prefer_videotoolbox_hw_input: false,
realtime_videotoolbox_keyframe_interval: None,
};

let mut encoder = SegmentedVideoEncoder::init_write_behind(
Expand Down
Loading
Loading