Fix casting audio and pacing and show actual stream stats

This commit is contained in:
toorux committed 2026-10-13 20:54:57 +08:00
1 parent 41137e1d26
commit a078e8bfb1
6 files changed
+120 -19

No files matched your search

+80 -13
View File
@@ -452,6 +452,11 @@ impl Casting {
"0",
"-framerate",
&s.fps.to_string(),
// Timestamp raw packets before FFmpeg schedules the inputs.
// Use the capture frame count rather than arrival wallclock:
// pipe reads can batch frames and distort scheduler pacing.
"-bsf:v",
&video_packet_timestamps(s.fps),
"-f",
if s.codec == Codec::H265 {"hevc"} else {"h264"},
"-i",
@@ -460,7 +465,6 @@ impl Casting {
} else {
ffmpeg.args(["-thread_queue_size", "4", "-f", "v4l2", "-i", "/dev/video99"]);
}
let audio = s.system_audio || s.microphone;
if s.system_audio {
add_pulse_input(&mut ffmpeg, "@DEFAULT_MONITOR@");
}
@@ -468,22 +472,15 @@ impl Casting {
add_pulse_input(&mut ffmpeg, "default");
}
ffmpeg.args(["-map", "0:v:0"]);
if s.system_audio && s.microphone {
ffmpeg.args([
"-filter_complex",
"[1:a][2:a]amix=inputs=2:duration=longest:normalize=0[a]",
"-map",
"[a]",
]);
} else if audio {
ffmpeg.args(["-map", "1:a:0"]);
}
let audio = map_audio(&mut ffmpeg, s.system_audio, s.microphone);
if s.source == Source::Screen {
ffmpeg.args(["-c:v", "copy"]);
if s.codec==Codec::H265 {
if s.codec == Codec::H265 {
// Iris HEVC headers omit reliable frame timing. Write the requested
// timing without decoding/re-encoding, avoiding false 50 fps playback.
ffmpeg.args(["-bsf:v",&format!("hevc_metadata=tick_rate={}:num_ticks_poc_diff_one=1",s.fps)]);
ffmpeg.args(["-bsf:v", &format!(
"hevc_metadata=tick_rate={}:num_ticks_poc_diff_one=1", s.fps
)]);
}
} else {
ffmpeg.args(["-vf", &format!("fps={},scale={}:{}:force_original_aspect_ratio=decrease,pad={}:{}:(ow-iw)/2:(oh-ih)/2,format=yuv420p",s.fps,s.width,s.height,s.width,s.height),
@@ -598,6 +595,29 @@ fn add_pulse_input(command: &mut Command, device: &str) {
command.args(["-thread_queue_size", "4", "-f", "pulse", "-sample_rate", "48000", "-channels", "2", "-fragment_size", "3840", "-i", device]);
}
// Pulse capture timestamps use the audio server's clock, while raw panel video
// starts at zero. Generate audio PTS from the sample count in that same origin.
// libopus low-delay mode subtracts its 120-sample pre-roll from packet PTS, so
// offset the input by 120 samples and keep the first RTSP packet non-negative.
fn map_audio(command: &mut Command, system_audio: bool, microphone: bool) -> bool {
let audio = system_audio || microphone;
if system_audio && microphone {
command.args([
"-filter_complex",
"[1:a][2:a]amix=inputs=2:duration=longest:normalize=0,asetpts=(N+120)/SR/TB[a]",
"-map",
"[a]",
]);
} else if audio {
command.args(["-map", "1:a:0", "-af", "asetpts=(N+120)/SR/TB"]);
}
audio
}
fn video_packet_timestamps(fps: u32) -> String {
format!("setts=time_base=1/{fps}:pts=N:dts=N:duration=1")
}
pub fn bind_to_session(command: &mut Command) {
// Linux PDEATHSIG follows the spawning thread, not the session process.
// HTTP request threads end immediately after returning a response. Worker
@@ -609,6 +629,53 @@ pub fn bind_to_session(command: &mut Command) {
mod tests {
use super::*;
#[test]
fn audio_mapping_generates_monotonic_timestamps() {
let arguments = |system_audio, microphone| {
let mut command = Command::new("ffmpeg");
let enabled = map_audio(&mut command, system_audio, microphone);
let arguments = command
.get_args()
.map(|argument| argument.to_string_lossy().into_owned())
.collect::<Vec<_>>();
(enabled, arguments)
};
assert_eq!(arguments(false, false), (false, vec![]));
for inputs in [(true, false), (false, true)] {
assert_eq!(
arguments(inputs.0, inputs.1),
(
true,
vec!["-map", "1:a:0", "-af", "asetpts=(N+120)/SR/TB"]
.into_iter()
.map(str::to_owned)
.collect()
)
);
}
assert_eq!(
arguments(true, true),
(
true,
vec![
"-filter_complex",
"[1:a][2:a]amix=inputs=2:duration=longest:normalize=0,asetpts=(N+120)/SR/TB[a]",
"-map",
"[a]",
]
.into_iter()
.map(str::to_owned)
.collect()
)
);
}
#[test]
fn copied_video_packets_receive_frame_timestamps() {
assert_eq!(
video_packet_timestamps(60),
"setts=time_base=1/60:pts=N:dts=N:duration=1"
);
}
#[test]
fn media_worker_survives_the_http_request_thread() {
let mut child = std::thread::spawn(|| {
let mut command = Command::new("sleep");
+2
View File
@@ -3,6 +3,8 @@
"locale": "en-US",
"name": "English",
"messages": {
"正在读取串流信息…": "Reading stream information…",
"当前浏览器实际接收并解码的串流信息": "Actual stream received and decoded by this browser",
"L2 / R2 按键触发行程": "L2 / R2 button activation threshold",
"默认 80%。达到此行程才触发按键,模拟轴保持完整行程;保存后立即生效。": "Default: 80%. Buttons activate at this trigger travel. Analog axes retain their full range; saving applies immediately.",
"更新应用并保留现有存档和设置。": "Update the app while keeping your saves and settings.",
+2
View File
@@ -3,6 +3,8 @@
"locale": "zh-CN",
"name": "简体中文",
"messages": {
"正在读取串流信息…": "正在读取串流信息…",
"当前浏览器实际接收并解码的串流信息": "当前浏览器实际接收并解码的串流信息",
"L2 / R2 按键触发行程": "L2 / R2 按键触发行程",
"默认 80%。达到此行程才触发按键,模拟轴保持完整行程;保存后立即生效。": "默认 80%。达到此行程才触发按键,模拟轴保持完整行程;保存后立即生效。",
"安装前需要关闭容器,以安全备份应用数据。关闭会中断其中正在运行的应用,保留存档和设置。": "安装前需要关闭容器,以安全备份应用数据。关闭会中断其中正在运行的应用,保留存档和设置。",
+31 -3
View File
@@ -3,21 +3,49 @@ import {IconArrowLeft,IconPlayerPause,IconPlayerPlay,IconVolume,IconVolumeOff,Ic
import {CastRange} from './cast-range';
import {t} from './i18n';
export function CastVideo({active,onError,controls=true,aspectRatio,videoRef,onState,showStatus=true}:{active:boolean;onError?:(error:string)=>void;controls?:boolean;aspectRatio?:number;videoRef?:React.RefObject<HTMLVideoElement|null>;onState?:(state:'connecting'|'playing'|'failed')=>void;showStatus?:boolean}){
export type CastStreamInfo={width:number|null;height:number|null;codec:string|null;fps:number|null};
export function CastVideo({active,onError,controls=true,aspectRatio,videoRef,onState,onStreamInfo,showStatus=true}:{active:boolean;onError?:(error:string)=>void;controls?:boolean;aspectRatio?:number;videoRef?:React.RefObject<HTMLVideoElement|null>;onState?:(state:'connecting'|'playing'|'failed')=>void;onStreamInfo?:(info:CastStreamInfo|null)=>void;showStatus?:boolean}){
const localVideo=useRef<HTMLVideoElement>(null),video=videoRef??localVideo,failure=useRef(onError);failure.current=onError;
const streamInfo=useRef(onStreamInfo);streamInfo.current=onStreamInfo;
const [state,setState]=useState<'connecting'|'playing'|'failed'>('connecting');
useEffect(()=>{onState?.(state);},[state,onState]);
const [ratio,setRatio]=useState(16/9),[retry,setRetry]=useState(0);
useEffect(()=>{
if(!active)return;
const peer=new RTCPeerConnection(),abort=new AbortController();let resource:string|undefined,closed=false;
let previous:{id:string;frames:number;timestamp:number}|undefined,statsBusy=false;
streamInfo.current?.(null);
async function reportStreamInfo(){
if(closed||statsBusy||!streamInfo.current)return;
if(peer.connectionState!=='connected'){previous=undefined;streamInfo.current?.(null);return;}
statsBusy=true;
try{
const stats=await peer.getStats();if(closed)return;
let inbound:RTCInboundRtpStreamStats|undefined;
stats.forEach(report=>{if(report.type==='inbound-rtp'&&(report.kind==='video'||report.mediaType==='video'))inbound=report;});
if(!inbound){previous=undefined;streamInfo.current?.(null);return;}
const {id,framesDecoded,timestamp,frameWidth,frameHeight,codecId}=inbound;
let fps:number|null=null;
if(typeof framesDecoded==='number'){
if(previous?.id===id&&framesDecoded>=previous.frames&&timestamp>previous.timestamp)fps=(framesDecoded-previous.frames)*1000/(timestamp-previous.timestamp);
previous={id,frames:framesDecoded,timestamp};
}
let mime:string|undefined;
stats.forEach(report=>{if(report.id===codecId&&report.type==='codec')mime=report.mimeType;});
const name=typeof mime==='string'?mime.replace(/^video\//i,''):null;
const codec=name?/^(H265|HEVC)$/i.test(name)?'H.265':/^H264$/i.test(name)?'H.264':name.toUpperCase():null;
streamInfo.current?.({width:typeof frameWidth==='number'&&frameWidth>0?frameWidth:video.current?.videoWidth||null,height:typeof frameHeight==='number'&&frameHeight>0?frameHeight:video.current?.videoHeight||null,codec,fps});
}catch{if(!closed){previous=undefined;streamInfo.current?.(null);}}
finally{statsBusy=false;}
}
const statsTimer=setInterval(()=>void reportStreamInfo(),1000);
setState('connecting');
for(const kind of ['video','audio']){
const receiver=peer.addTransceiver(kind,{direction:'recvonly'}).receiver;
if('jitterBufferTarget' in receiver){try{(receiver as RTCRtpReceiver&{jitterBufferTarget:number|null}).jitterBufferTarget=0;}catch{}}
}
peer.ontrack=event=>{if(video.current){const stream=video.current.srcObject as MediaStream|null??new MediaStream();stream.addTrack(event.track);video.current.srcObject=stream;void video.current.play().catch(()=>{});}};
peer.onconnectionstatechange=()=>{if(peer.connectionState==='failed'){setState('failed');failure.current?.(t('观看连接失败,请重新连接。'));}};
peer.onconnectionstatechange=()=>{if(peer.connectionState==='failed'){previous=undefined;streamInfo.current?.(null);setState('failed');failure.current?.(t('观看连接失败,请重新连接。'));}};
void(async()=>{
try{
await peer.setLocalDescription(await peer.createOffer());
@@ -34,7 +62,7 @@ export function CastVideo({active,onError,controls=true,aspectRatio,videoRef,onS
await peer.setRemoteDescription({type:'answer',sdp:await response.text()});
}catch(error){if(!abort.signal.aborted){setState('failed');failure.current?.(String(error));}}
})();
return()=>{closed=true;abort.abort();peer.close();if(resource)void fetch(resource,{method:'DELETE'}).catch(()=>{});if(video.current)video.current.srcObject=null;};
return()=>{closed=true;clearInterval(statsTimer);streamInfo.current?.(null);abort.abort();peer.close();if(resource)void fetch(resource,{method:'DELETE'}).catch(()=>{});if(video.current)video.current.srcObject=null;};
},[active,retry]);
return <div className="cast-video-stage" style={{aspectRatio:aspectRatio??ratio}}><video ref={video} autoPlay playsInline muted controls={controls} onLoadedMetadata={()=>{if(video.current?.videoHeight)setRatio(video.current.videoWidth/video.current.videoHeight);}} onPlaying={()=>setState('playing')}/>{showStatus&&active&&state==='connecting'&&<span role="status">{t('正在连接画面…')}</span>}{showStatus&&active&&state==='failed'&&<div className="cast-video-failure"><p>{t('观看连接失败,请重新连接。')}</p><button onClick={()=>setRetry(v=>v+1)}>{t('重新连接')}</button></div>}{!active&&<span>{t('开始串流后显示头显画面。')}</span>}</div>;
}
+1
View File
@@ -12,5 +12,6 @@
.cast-choice-group small{white-space:normal;line-height:1.45;max-width:100%;text-align:center}.cast-framing-section{scroll-margin-top:16px}.cast-framing-stage{max-width:min(640px,calc(100dvh - 330px))}@media(max-width:767px){.cast-choice-group button{min-width:0}.cast-framing-stage{max-width:100%}.cast-watch-icon{min-height:44px}}
.cast-framing-stage{max-width:min(640px,calc(100dvh - 400px))}.cast-editor-actions{position:sticky;bottom:-24px;background:transparent;padding:14px 0;z-index:6}@media(max-width:767px){.cast-choice-group button>span{white-space:normal}.cast-framing-stage{max-width:100%}.cast-editor-actions{bottom:-16px}}
.cast-protocol-heading{display:flex;align-items:center;justify-content:space-between;gap:16px;margin-bottom:6px}.cast-protocol-heading h2{margin:0}.cast-protocol-heading .ui-switch{flex-shrink:0}.cast-range{--range-fill:0%;appearance:none;-webkit-appearance:none;display:block;min-height:32px!important;height:32px!important;padding:0!important;margin:0;background:transparent!important;cursor:pointer;border:0!important;outline-offset:3px;touch-action:pan-y}.cast-range::-webkit-slider-runnable-track{height:6px;border-radius:999px;background:linear-gradient(to right,var(--blue) var(--range-fill),#484848 var(--range-fill))}.cast-range::-webkit-slider-thumb{appearance:none;-webkit-appearance:none;width:28px;height:16px;margin-top:-5px;border:0;border-radius:999px;background:var(--blue);box-shadow:0 1px 3px #0005}.cast-range::-moz-range-track{height:6px;border-radius:999px;background:#484848}.cast-range::-moz-range-progress{height:6px;border-radius:999px;background:var(--blue)}.cast-range::-moz-range-thumb{width:28px;height:16px;border:0;border-radius:999px;background:var(--blue)}.cast-range:disabled{opacity:.4;cursor:default}
.cast-page-heading{flex-wrap:wrap}.cast-status{font-variant-numeric:tabular-nums;white-space:normal;text-align:right;margin-left:auto}
/* Viewer: one restrained floating toolbar; the video always remains complete. */
.cast-watch video{object-fit:contain!important}.cast-watch-top{padding-bottom:40px;gap:12px;background:linear-gradient(rgb(0 0 0/.55),transparent)}.cast-watch-title{font-size:14px;font-weight:500;color:#e9e9e9}.cast-watch-connection{font-size:12px;color:#c6c6c6}.cast-watch-bottom{padding-top:32px;background:transparent;display:flex;justify-content:center}.cast-watch-controls{width:auto;max-width:100%;padding:6px 12px;gap:8px;background:rgb(18 18 18/.88);border:1px solid rgb(255 255 255/.12);border-radius:999px;box-shadow:0 8px 28px #0005;backdrop-filter:blur(16px);-webkit-backdrop-filter:blur(16px)}.cast-watch .cast-watch-icon{width:40px;height:40px;min-height:40px;border:0;border-radius:50%;padding:8px;background:transparent;color:#ededed}.cast-watch .cast-watch-icon:hover{background:#ffffff12}.cast-watch-top .cast-watch-icon{background:transparent}.cast-watch-top .cast-watch-icon:hover{background:#ffffff12}.cast-watch-sound{gap:6px}.cast-watch-sound input{width:80px}.cast-watch-live{margin:0 12px 0 8px;font-size:11px;color:#bcbcbc;letter-spacing:.02em}.cast-watch-tools{gap:4px;padding-left:8px;border-left:1px solid #ffffff1f}.cast-watch .cast-range{--blue:#e7e7e7}.cast-watch .cast-range::-webkit-slider-thumb{width:20px;height:12px;margin-top:-3px}.cast-watch .cast-range::-moz-range-thumb{width:20px;height:12px}.cast-watch-message h1{font-size:22px;font-weight:500}.cast-watch-message p{font-size:13px}.cast-watch-message button{border-radius:999px;min-height:40px;font-size:13px;padding:10px 18px}@media(max-width:767px){.cast-watch-bottom{padding-bottom:calc(16px + env(safe-area-inset-bottom))}.cast-watch-controls{gap:4px;padding:5px 9px}.cast-watch-top{padding-top:calc(12px + env(safe-area-inset-top));padding-bottom:32px}.cast-watch-live{display:inline;margin:0 10px 0 4px}.cast-watch-title{font-size:13px}.cast-watch-tools{padding-left:6px;gap:2px}}@media(prefers-reduced-transparency:reduce){.cast-watch-controls{background:#181818;backdrop-filter:none;-webkit-backdrop-filter:none}}
+4 -3
View File
@@ -5,7 +5,7 @@ import {api} from './api';
import {Switch,SwitchRow} from './switch';
import {CastRange} from './cast-range';
import {t} from './i18n';
import {CastVideo} from './cast-viewer';
import {CastVideo,type CastStreamInfo} from './cast-viewer';
import {FramingEditor} from './cast-framing-editor';
import {framePresets,frameRatios,type ReferenceView} from './cast-framing';
import './casting.css';
@@ -20,6 +20,7 @@ function Choices<T extends string|number>({label,value,options,onChange,disabled
return <div className="cast-choice-group" role="radiogroup" aria-label={label}>{options.map((option,i)=><button type="button" role="radio" aria-checked={value===option.value} className={value===option.value?'selected':''} disabled={disabled} key={option.value} onClick={()=>onChange(option.value)} onKeyDown={e=>{if(['ArrowLeft','ArrowRight','ArrowUp','ArrowDown'].includes(e.key)){e.preventDefault();const next=(i+(['ArrowLeft','ArrowUp'].includes(e.key)?-1:1)+options.length)%options.length;onChange(options[next].value);(e.currentTarget.parentElement?.children[next] as HTMLElement)?.focus();}}}><span>{option.label}</span>{option.description&&<small>{option.description}</small>}</button>)}</div>;
}
export function Casting({config,refresh,network,page='casting'}:{config:CastSettings;refresh:()=>Promise<void>;network:NetworkPanel;page?:CastPage}){
const [streamInfo,setStreamInfo]=useState<CastStreamInfo|null>(null);
const [draft,setDraft]=useState(config),[runtime,setRuntime]=useState<Runtime|null>(null),[pending,setPending]=useState(''),[error,setError]=useState(''),[saved,setSaved]=useState(false),[epoch,setEpoch]=useState(0),[devices,setDevices]=useState<Device[]>([]),[sending,setSending]=useState<string>(''),[copied,setCopied]=useState(false),[editing,setEditing]=useState(false),[searched,setSearched]=useState(false);
const beforeFraming=useRef<Pick<CastSettings,'horizontalFov'|'centerX'|'centerY'>|null>(null),editorActive=useRef(false),deviceScan=useRef(0),alive=useRef(true);
const captureSettings=({airplay,dlna,receiverName,...capture}:CastSettings)=>capture;
@@ -42,10 +43,10 @@ export function Casting({config,refresh,network,page='casting'}:{config:CastSett
const presets=framePresets(aspect);
function chooseAspect(value:number){const edge=Math.max(draft.width,draft.height);const size=value>=1?{width:edge,height:Math.round(edge/value/2)*2}:{width:Math.round(edge*value/2)*2,height:edge};change(size);}
return <div className="casting cast-workspace">
<header className="cast-page-heading"><div><h1>{t(titles[page][0])}</h1><p>{t(titles[page][1])}</p></div>{page==='casting'&&<span className={`cast-status ${runtime?.running?'live':''}`} role="status">{pending==='start'?t('正在启动…'):runtime?.running?t('正在串流'):t('未开始串流')}</span>}</header>
<header className="cast-page-heading"><div><h1>{t(titles[page][0])}</h1><p>{t(titles[page][1])}</p></div>{page==='casting'&&<span className={`cast-status ${runtime?.running?'live':''}`} title={t('当前浏览器实际接收并解码的串流信息')} role="status">{pending==='start'?t('正在启动…'):runtime?.running?streamInfo?<>{streamInfo.width&&streamInfo.height?`${streamInfo.width} × ${streamInfo.height}`:'—'} · {streamInfo.codec??'—'} · {streamInfo.fps===null?'—':streamInfo.fps.toFixed(1)} fps</>:t('正在读取串流信息…'):t('未开始串流')}</span>}</header>
{errorMessage&&<div className="cast-error" role="alert"><strong>{t('操作未完成')}</strong><p>{errorMessage}</p></div>}
{page==='casting'&&<>
<div className="cast-monitor"><CastVideo key={epoch} active={!!runtime?.running&&!runtime.preview} onError={setError}/>{!runtime?.running&&<div className="cast-monitor-empty"><IconDeviceVisionPro size={44} stroke={1.5}/><h2>{t('分享你看到的画面')}</h2><p>{t('启动后,手机和电脑可以通过网页实时观看。')}</p></div>}</div>
<div className="cast-monitor"><CastVideo key={epoch} active={!!runtime?.running&&!runtime.preview} onError={setError} onStreamInfo={setStreamInfo}/>{!runtime?.running&&<div className="cast-monitor-empty"><IconDeviceVisionPro size={44} stroke={1.5}/><h2>{t('分享你看到的画面')}</h2><p>{t('启动后,手机和电脑可以通过网页实时观看。')}</p></div>}</div>
<div className="cast-stream-bar"><div><strong>{t(config.source==='screen'?'屏幕采集':'SteamVR')}</strong><span>{config.width} × {config.height}<i/> {config.fps} fps</span></div><button className={runtime?.running?'':'primary'} disabled={busy||!runtime} onClick={()=>void action('start',async()=>{if(runtime?.running)await api('cast.stop');else{if(dirty)await apply();await api('cast.start');}setEpoch(v=>v+1);})}>{runtime?.running?<IconPlayerStop size={21}/>:<IconPlayerPlay size={21}/>}<span>{pending==='start'?t('正在启动…'):runtime?.running?t('停止串流'):t('开始串流')}</span></button></div>
{runtime?.running&&<section className="cast-share"><div><IconWifi size={24}/><div><h2>{t('在其他设备上观看')}</h2><p>{t('连接同一网络,打开观看地址。')}</p></div></div><div className="cast-share-link"><code>{(!['localhost','127.0.0.1','[::1]'].includes(location.hostname)?new URL('/cast/watch',location.origin).href:runtime.watchAddresses?.[0])??new URL('/cast/watch',location.origin).href}</code><button onClick={()=>void action('copy',copy)} aria-label={t('复制观看地址')}>{copied?<IconCheck size={20}/>:<IconCopy size={20}/>}<span>{copied?t('已复制'):t('复制')}</span></button><a className="cast-icon-link" href="/cast/watch" aria-label={t('打开观看网页')}><IconExternalLink size={21}/></a></div></section>}