From a078e8bfb1022d845f60f4af2fe25ea6d6c49ad8 Mon Sep 17 00:00:00 2001 From: toorux Date: Tue, 13 Oct 2026 20:54:47 +0800 Subject: [PATCH] Fix casting audio and pacing and show actual stream stats --- src/cast/runtime.rs | 93 ++++++++++++++++++++++++++++++++++++------ ui/locales/en-US.json | 2 + ui/locales/zh-CN.json | 2 + ui/src/cast-viewer.tsx | 34 +++++++++++++-- ui/src/casting.css | 1 + ui/src/casting.tsx | 7 ++-- 6 files changed, 120 insertions(+), 19 deletions(-) diff --git a/src/cast/runtime.rs b/src/cast/runtime.rs index bfc5cdf..f8c265e 100644 --- a/src/cast/runtime.rs +++ b/src/cast/runtime.rs @@ -452,6 +452,11 @@ impl Casting { "0", "-framerate", &s.fps.to_string(), + // Timestamp raw packets before FFmpeg schedules the inputs. + // Use the capture frame count rather than arrival wallclock: + // pipe reads can batch frames and distort scheduler pacing. + "-bsf:v", + &video_packet_timestamps(s.fps), "-f", if s.codec == Codec::H265 {"hevc"} else {"h264"}, "-i", @@ -460,7 +465,6 @@ impl Casting { } else { ffmpeg.args(["-thread_queue_size", "4", "-f", "v4l2", "-i", "/dev/video99"]); } - let audio = s.system_audio || s.microphone; if s.system_audio { add_pulse_input(&mut ffmpeg, "@DEFAULT_MONITOR@"); } @@ -468,22 +472,15 @@ impl Casting { add_pulse_input(&mut ffmpeg, "default"); } ffmpeg.args(["-map", "0:v:0"]); - if s.system_audio && s.microphone { - ffmpeg.args([ - "-filter_complex", - "[1:a][2:a]amix=inputs=2:duration=longest:normalize=0[a]", - "-map", - "[a]", - ]); - } else if audio { - ffmpeg.args(["-map", "1:a:0"]); - } + let audio = map_audio(&mut ffmpeg, s.system_audio, s.microphone); if s.source == Source::Screen { ffmpeg.args(["-c:v", "copy"]); - if s.codec==Codec::H265 { + if s.codec == Codec::H265 { // Iris HEVC headers omit reliable frame timing. Write the requested // timing without decoding/re-encoding, avoiding false 50 fps playback. - ffmpeg.args(["-bsf:v",&format!("hevc_metadata=tick_rate={}:num_ticks_poc_diff_one=1",s.fps)]); + ffmpeg.args(["-bsf:v", &format!( + "hevc_metadata=tick_rate={}:num_ticks_poc_diff_one=1", s.fps + )]); } } else { ffmpeg.args(["-vf", &format!("fps={},scale={}:{}:force_original_aspect_ratio=decrease,pad={}:{}:(ow-iw)/2:(oh-ih)/2,format=yuv420p",s.fps,s.width,s.height,s.width,s.height), @@ -598,6 +595,29 @@ fn add_pulse_input(command: &mut Command, device: &str) { command.args(["-thread_queue_size", "4", "-f", "pulse", "-sample_rate", "48000", "-channels", "2", "-fragment_size", "3840", "-i", device]); } +// Pulse capture timestamps use the audio server's clock, while raw panel video +// starts at zero. Generate audio PTS from the sample count in that same origin. +// libopus low-delay mode subtracts its 120-sample pre-roll from packet PTS, so +// offset the input by 120 samples and keep the first RTSP packet non-negative. +fn map_audio(command: &mut Command, system_audio: bool, microphone: bool) -> bool { + let audio = system_audio || microphone; + if system_audio && microphone { + command.args([ + "-filter_complex", + "[1:a][2:a]amix=inputs=2:duration=longest:normalize=0,asetpts=(N+120)/SR/TB[a]", + "-map", + "[a]", + ]); + } else if audio { + command.args(["-map", "1:a:0", "-af", "asetpts=(N+120)/SR/TB"]); + } + audio +} + +fn video_packet_timestamps(fps: u32) -> String { + format!("setts=time_base=1/{fps}:pts=N:dts=N:duration=1") +} + pub fn bind_to_session(command: &mut Command) { // Linux PDEATHSIG follows the spawning thread, not the session process. // HTTP request threads end immediately after returning a response. Worker @@ -609,6 +629,53 @@ pub fn bind_to_session(command: &mut Command) { mod tests { use super::*; #[test] + fn audio_mapping_generates_monotonic_timestamps() { + let arguments = |system_audio, microphone| { + let mut command = Command::new("ffmpeg"); + let enabled = map_audio(&mut command, system_audio, microphone); + let arguments = command + .get_args() + .map(|argument| argument.to_string_lossy().into_owned()) + .collect::>(); + (enabled, arguments) + }; + assert_eq!(arguments(false, false), (false, vec![])); + for inputs in [(true, false), (false, true)] { + assert_eq!( + arguments(inputs.0, inputs.1), + ( + true, + vec!["-map", "1:a:0", "-af", "asetpts=(N+120)/SR/TB"] + .into_iter() + .map(str::to_owned) + .collect() + ) + ); + } + assert_eq!( + arguments(true, true), + ( + true, + vec![ + "-filter_complex", + "[1:a][2:a]amix=inputs=2:duration=longest:normalize=0,asetpts=(N+120)/SR/TB[a]", + "-map", + "[a]", + ] + .into_iter() + .map(str::to_owned) + .collect() + ) + ); + } + #[test] + fn copied_video_packets_receive_frame_timestamps() { + assert_eq!( + video_packet_timestamps(60), + "setts=time_base=1/60:pts=N:dts=N:duration=1" + ); + } + #[test] fn media_worker_survives_the_http_request_thread() { let mut child = std::thread::spawn(|| { let mut command = Command::new("sleep"); diff --git a/ui/locales/en-US.json b/ui/locales/en-US.json index 2a1e9c9..54e5277 100644 --- a/ui/locales/en-US.json +++ b/ui/locales/en-US.json @@ -3,6 +3,8 @@ "locale": "en-US", "name": "English", "messages": { + "正在读取串流信息…": "Reading stream information…", + "当前浏览器实际接收并解码的串流信息": "Actual stream received and decoded by this browser", "L2 / R2 按键触发行程": "L2 / R2 button activation threshold", "默认 80%。达到此行程才触发按键，模拟轴保持完整行程；保存后立即生效。": "Default: 80%. Buttons activate at this trigger travel. Analog axes retain their full range; saving applies immediately.", "更新应用并保留现有存档和设置。": "Update the app while keeping your saves and settings.", diff --git a/ui/locales/zh-CN.json b/ui/locales/zh-CN.json index ef05387..23e6e8b 100644 --- a/ui/locales/zh-CN.json +++ b/ui/locales/zh-CN.json @@ -3,6 +3,8 @@ "locale": "zh-CN", "name": "简体中文", "messages": { + "正在读取串流信息…": "正在读取串流信息…", + "当前浏览器实际接收并解码的串流信息": "当前浏览器实际接收并解码的串流信息", "L2 / R2 按键触发行程": "L2 / R2 按键触发行程", "默认 80%。达到此行程才触发按键，模拟轴保持完整行程；保存后立即生效。": "默认 80%。达到此行程才触发按键，模拟轴保持完整行程；保存后立即生效。", "安装前需要关闭容器，以安全备份应用数据。关闭会中断其中正在运行的应用，保留存档和设置。": "安装前需要关闭容器，以安全备份应用数据。关闭会中断其中正在运行的应用，保留存档和设置。", diff --git a/ui/src/cast-viewer.tsx b/ui/src/cast-viewer.tsx index 86cd2ea..bacc821 100644 --- a/ui/src/cast-viewer.tsx +++ b/ui/src/cast-viewer.tsx @@ -3,21 +3,49 @@ import {IconArrowLeft,IconPlayerPause,IconPlayerPlay,IconVolume,IconVolumeOff,Ic import {CastRange} from './cast-range'; import {t} from './i18n'; -export function CastVideo({active,onError,controls=true,aspectRatio,videoRef,onState,showStatus=true}:{active:boolean;onError?:(error:string)=>void;controls?:boolean;aspectRatio?:number;videoRef?:React.RefObject;onState?:(state:'connecting'|'playing'|'failed')=>void;showStatus?:boolean}){ +export type CastStreamInfo={width:number|null;height:number|null;codec:string|null;fps:number|null}; +export function CastVideo({active,onError,controls=true,aspectRatio,videoRef,onState,onStreamInfo,showStatus=true}:{active:boolean;onError?:(error:string)=>void;controls?:boolean;aspectRatio?:number;videoRef?:React.RefObject;onState?:(state:'connecting'|'playing'|'failed')=>void;onStreamInfo?:(info:CastStreamInfo|null)=>void;showStatus?:boolean}){ const localVideo=useRef(null),video=videoRef??localVideo,failure=useRef(onError);failure.current=onError; + const streamInfo=useRef(onStreamInfo);streamInfo.current=onStreamInfo; const [state,setState]=useState<'connecting'|'playing'|'failed'>('connecting'); useEffect(()=>{onState?.(state);},[state,onState]); const [ratio,setRatio]=useState(16/9),[retry,setRetry]=useState(0); useEffect(()=>{ if(!active)return; const peer=new RTCPeerConnection(),abort=new AbortController();let resource:string|undefined,closed=false; + let previous:{id:string;frames:number;timestamp:number}|undefined,statsBusy=false; + streamInfo.current?.(null); + async function reportStreamInfo(){ + if(closed||statsBusy||!streamInfo.current)return; + if(peer.connectionState!=='connected'){previous=undefined;streamInfo.current?.(null);return;} + statsBusy=true; + try{ + const stats=await peer.getStats();if(closed)return; + let inbound:RTCInboundRtpStreamStats|undefined; + stats.forEach(report=>{if(report.type==='inbound-rtp'&&(report.kind==='video'||report.mediaType==='video'))inbound=report;}); + if(!inbound){previous=undefined;streamInfo.current?.(null);return;} + const {id,framesDecoded,timestamp,frameWidth,frameHeight,codecId}=inbound; + let fps:number|null=null; + if(typeof framesDecoded==='number'){ + if(previous?.id===id&&framesDecoded>=previous.frames&×tamp>previous.timestamp)fps=(framesDecoded-previous.frames)*1000/(timestamp-previous.timestamp); + previous={id,frames:framesDecoded,timestamp}; + } + let mime:string|undefined; + stats.forEach(report=>{if(report.id===codecId&&report.type==='codec')mime=report.mimeType;}); + const name=typeof mime==='string'?mime.replace(/^video\//i,''):null; + const codec=name?/^(H265|HEVC)$/i.test(name)?'H.265':/^H264$/i.test(name)?'H.264':name.toUpperCase():null; + streamInfo.current?.({width:typeof frameWidth==='number'&&frameWidth>0?frameWidth:video.current?.videoWidth||null,height:typeof frameHeight==='number'&&frameHeight>0?frameHeight:video.current?.videoHeight||null,codec,fps}); + }catch{if(!closed){previous=undefined;streamInfo.current?.(null);}} + finally{statsBusy=false;} + } + const statsTimer=setInterval(()=>void reportStreamInfo(),1000); setState('connecting'); for(const kind of ['video','audio']){ const receiver=peer.addTransceiver(kind,{direction:'recvonly'}).receiver; if('jitterBufferTarget' in receiver){try{(receiver as RTCRtpReceiver&{jitterBufferTarget:number|null}).jitterBufferTarget=0;}catch{}} } peer.ontrack=event=>{if(video.current){const stream=video.current.srcObject as MediaStream|null??new MediaStream();stream.addTrack(event.track);video.current.srcObject=stream;void video.current.play().catch(()=>{});}}; - peer.onconnectionstatechange=()=>{if(peer.connectionState==='failed'){setState('failed');failure.current?.(t('观看连接失败，请重新连接。'));}}; + peer.onconnectionstatechange=()=>{if(peer.connectionState==='failed'){previous=undefined;streamInfo.current?.(null);setState('failed');failure.current?.(t('观看连接失败，请重新连接。'));}}; void(async()=>{ try{ await peer.setLocalDescription(await peer.createOffer()); @@ -34,7 +62,7 @@ export function CastVideo({active,onError,controls=true,aspectRatio,videoRef,onS await peer.setRemoteDescription({type:'answer',sdp:await response.text()}); }catch(error){if(!abort.signal.aborted){setState('failed');failure.current?.(String(error));}} })(); - return()=>{closed=true;abort.abort();peer.close();if(resource)void fetch(resource,{method:'DELETE'}).catch(()=>{});if(video.current)video.current.srcObject=null;}; + return()=>{closed=true;clearInterval(statsTimer);streamInfo.current?.(null);abort.abort();peer.close();if(resource)void fetch(resource,{method:'DELETE'}).catch(()=>{});if(video.current)video.current.srcObject=null;}; },[active,retry]); return
; } diff --git a/ui/src/casting.css b/ui/src/casting.css index 72c5483..caa668d 100644 --- a/ui/src/casting.css +++ b/ui/src/casting.css @@ -12,5 +12,6 @@ .cast-choice-group small{white-space:normal;line-height:1.45;max-width:100%;text-align:center}.cast-framing-section{scroll-margin-top:16px}.cast-framing-stage{max-width:min(640px,calc(100dvh - 330px))}@media(max-width:767px){.cast-choice-group button{min-width:0}.cast-framing-stage{max-width:100%}.cast-watch-icon{min-height:44px}} .cast-framing-stage{max-width:min(640px,calc(100dvh - 400px))}.cast-editor-actions{position:sticky;bottom:-24px;background:transparent;padding:14px 0;z-index:6}@media(max-width:767px){.cast-choice-group button>span{white-space:normal}.cast-framing-stage{max-width:100%}.cast-editor-actions{bottom:-16px}} .cast-protocol-heading{display:flex;align-items:center;justify-content:space-between;gap:16px;margin-bottom:6px}.cast-protocol-heading h2{margin:0}.cast-protocol-heading .ui-switch{flex-shrink:0}.cast-range{--range-fill:0%;appearance:none;-webkit-appearance:none;display:block;min-height:32px!important;height:32px!important;padding:0!important;margin:0;background:transparent!important;cursor:pointer;border:0!important;outline-offset:3px;touch-action:pan-y}.cast-range::-webkit-slider-runnable-track{height:6px;border-radius:999px;background:linear-gradient(to right,var(--blue) var(--range-fill),#484848 var(--range-fill))}.cast-range::-webkit-slider-thumb{appearance:none;-webkit-appearance:none;width:28px;height:16px;margin-top:-5px;border:0;border-radius:999px;background:var(--blue);box-shadow:0 1px 3px #0005}.cast-range::-moz-range-track{height:6px;border-radius:999px;background:#484848}.cast-range::-moz-range-progress{height:6px;border-radius:999px;background:var(--blue)}.cast-range::-moz-range-thumb{width:28px;height:16px;border:0;border-radius:999px;background:var(--blue)}.cast-range:disabled{opacity:.4;cursor:default} +.cast-page-heading{flex-wrap:wrap}.cast-status{font-variant-numeric:tabular-nums;white-space:normal;text-align:right;margin-left:auto} /* Viewer: one restrained floating toolbar; the video always remains complete. */ .cast-watch video{object-fit:contain!important}.cast-watch-top{padding-bottom:40px;gap:12px;background:linear-gradient(rgb(0 0 0/.55),transparent)}.cast-watch-title{font-size:14px;font-weight:500;color:#e9e9e9}.cast-watch-connection{font-size:12px;color:#c6c6c6}.cast-watch-bottom{padding-top:32px;background:transparent;display:flex;justify-content:center}.cast-watch-controls{width:auto;max-width:100%;padding:6px 12px;gap:8px;background:rgb(18 18 18/.88);border:1px solid rgb(255 255 255/.12);border-radius:999px;box-shadow:0 8px 28px #0005;backdrop-filter:blur(16px);-webkit-backdrop-filter:blur(16px)}.cast-watch .cast-watch-icon{width:40px;height:40px;min-height:40px;border:0;border-radius:50%;padding:8px;background:transparent;color:#ededed}.cast-watch .cast-watch-icon:hover{background:#ffffff12}.cast-watch-top .cast-watch-icon{background:transparent}.cast-watch-top .cast-watch-icon:hover{background:#ffffff12}.cast-watch-sound{gap:6px}.cast-watch-sound input{width:80px}.cast-watch-live{margin:0 12px 0 8px;font-size:11px;color:#bcbcbc;letter-spacing:.02em}.cast-watch-tools{gap:4px;padding-left:8px;border-left:1px solid #ffffff1f}.cast-watch .cast-range{--blue:#e7e7e7}.cast-watch .cast-range::-webkit-slider-thumb{width:20px;height:12px;margin-top:-3px}.cast-watch .cast-range::-moz-range-thumb{width:20px;height:12px}.cast-watch-message h1{font-size:22px;font-weight:500}.cast-watch-message p{font-size:13px}.cast-watch-message button{border-radius:999px;min-height:40px;font-size:13px;padding:10px 18px}@media(max-width:767px){.cast-watch-bottom{padding-bottom:calc(16px + env(safe-area-inset-bottom))}.cast-watch-controls{gap:4px;padding:5px 9px}.cast-watch-top{padding-top:calc(12px + env(safe-area-inset-top));padding-bottom:32px}.cast-watch-live{display:inline;margin:0 10px 0 4px}.cast-watch-title{font-size:13px}.cast-watch-tools{padding-left:6px;gap:2px}}@media(prefers-reduced-transparency:reduce){.cast-watch-controls{background:#181818;backdrop-filter:none;-webkit-backdrop-filter:none}} diff --git a/ui/src/casting.tsx b/ui/src/casting.tsx index 8d7231a..5f37bc0 100644 --- a/ui/src/casting.tsx +++ b/ui/src/casting.tsx @@ -5,7 +5,7 @@ import {api} from './api'; import {Switch,SwitchRow} from './switch'; import {CastRange} from './cast-range'; import {t} from './i18n'; -import {CastVideo} from './cast-viewer'; +import {CastVideo,type CastStreamInfo} from './cast-viewer'; import {FramingEditor} from './cast-framing-editor'; import {framePresets,frameRatios,type ReferenceView} from './cast-framing'; import './casting.css'; @@ -20,6 +20,7 @@ function Choices({label,value,options,onChange,disabled return
{options.map((option,i)=>)}
; } export function Casting({config,refresh,network,page='casting'}:{config:CastSettings;refresh:()=>Promise;network:NetworkPanel;page?:CastPage}){ + const [streamInfo,setStreamInfo]=useState(null); const [draft,setDraft]=useState(config),[runtime,setRuntime]=useState(null),[pending,setPending]=useState(''),[error,setError]=useState(''),[saved,setSaved]=useState(false),[epoch,setEpoch]=useState(0),[devices,setDevices]=useState([]),[sending,setSending]=useState(''),[copied,setCopied]=useState(false),[editing,setEditing]=useState(false),[searched,setSearched]=useState(false); const beforeFraming=useRef|null>(null),editorActive=useRef(false),deviceScan=useRef(0),alive=useRef(true); const captureSettings=({airplay,dlna,receiverName,...capture}:CastSettings)=>capture; @@ -42,10 +43,10 @@ export function Casting({config,refresh,network,page='casting'}:{config:CastSett const presets=framePresets(aspect); function chooseAspect(value:number){const edge=Math.max(draft.width,draft.height);const size=value>=1?{width:edge,height:Math.round(edge/value/2)*2}:{width:Math.round(edge*value/2)*2,height:edge};change(size);} return
-

{t(titles[page][0])}

{t(titles[page][1])}

{page==='casting'&&{pending==='start'?t('正在启动…'):runtime?.running?t('正在串流'):t('未开始串流')}}
+

{t(titles[page][0])}

{t(titles[page][1])}

{page==='casting'&&{pending==='start'?t('正在启动…'):runtime?.running?streamInfo?<>{streamInfo.width&&streamInfo.height?`${streamInfo.width} × ${streamInfo.height}`:'—'} · {streamInfo.codec??'—'} · {streamInfo.fps===null?'—':streamInfo.fps.toFixed(1)} fps:t('正在读取串流信息…'):t('未开始串流')}}
{errorMessage&&
{t('操作未完成')}

{errorMessage}

} {page==='casting'&&<> -
{!runtime?.running&&

{t('分享你看到的画面')}

{t('启动后，手机和电脑可以通过网页实时观看。')}

}
+
{!runtime?.running&&

{t('分享你看到的画面')}

{t('启动后，手机和电脑可以通过网页实时观看。')}

}
{t(config.source==='screen'?'屏幕采集':'SteamVR')}{config.width} × {config.height} {config.fps} fps
{runtime?.running&&

{t('在其他设备上观看')}

{t('连接同一网络，打开观看地址。')}

{(!['localhost','127.0.0.1','[::1]'].includes(location.hostname)?new URL('/cast/watch',location.origin).href:runtime.watchAddresses?.[0])??new URL('/cast/watch',location.origin).href}
}