Correct AirPlay video classification and update accepted player windows

media/native/frame_sink.h

@@ -16,10 +16,10 @@
 struct FrameSink {
  std::string directory,id,protocol; std::atomic<bool> announced{false}; std::atomic<unsigned> width{0},height{0};
  FrameSink(std::string d,std::string i,std::string p):directory(d),id(i),protocol(p){}
- void event(const std::string& type,unsigned w=0,unsigned h=0){
+ void event(const std::string& type,unsigned w=0,unsigned h=0,const std::string& media=""){
   int fd=socket(AF_UNIX,SOCK_DGRAM|SOCK_CLOEXEC,0); if(fd<0)return;
   sockaddr_un a{};a.sun_family=AF_UNIX; auto path=directory+"/events.sock";if(path.size()>=sizeof(a.sun_path)){close(fd);return;}strcpy(a.sun_path,path.c_str());
-  auto msg="{\"event\":\""+type+"\",\"id\":\""+id+"\",\"protocol\":\""+protocol+"\",\"width\":"+std::to_string(w)+",\"height\":"+std::to_string(h)+"}";
+  auto msg="{\"event\":\""+type+"\",\"id\":\""+id+"\",\"protocol\":\""+protocol+"\",\"width\":"+std::to_string(w)+",\"height\":"+std::to_string(h)+",\"mediaType\":\""+media+"\"}";
   sendto(fd,msg.data(),msg.size(),MSG_DONTWAIT,(sockaddr*)&a,sizeof(a));close(fd);
  }
  bool accepted() const {return access((directory+"/"+id+".accept").c_str(),F_OK)==0;}

media/patch-uxplay.py

@@ -62,3 +62,18 @@ if 'framely_mirror_request' not in s:
         FrameSink(directory,id,"AirPlay").event("request",(unsigned)*width_source,(unsigned)*height_source);
 ''',1)
     p.write_text(s)
+
+# Classify media before a decoded buffer arrives, including URL playback.
+s=p.read_text()
+if 'framely_media_kind' not in s:
+    marker='if(directory&&id)FrameSink(directory,id,"AirPlay").event("request");'
+    assert marker in s
+    s=s.replace(marker, 'if(directory&&id)FrameSink(directory,id,"AirPlay").event("request",0,0,*usingScreen?"video":"audio"); // framely_media_kind',1)
+    marker='extern "C" void on_video_play(void *cls, const char* location, const float start_position) {'
+    assert marker in s
+    s=s.replace(marker, marker+'\n    const char* directory=getenv("FRAMELY_CAST_DIR"),*id=getenv("FRAMELY_CAST_ID");\n    if(directory&&id)FrameSink(directory,id,"AirPlay").event("request",0,0,"video");',1)
+    # stdout is a file in production; retain useful errors before a restart.
+    s=s.replace('    va_end(vargs);', '    va_end(vargs);\n    fflush(stdout);',1)
+    p.write_text(s)
+from shutil import copyfile
+copyfile(Path(__file__).parent/'native/frame_sink.h',p.parent/'framely/frame_sink.h')

src/cast/receiver.rs

@@ -216,19 +216,25 @@ impl Receivers {
                 self.sessions.insert(id.clone(), e.clone());
                 self.events
                     .push_back(json!({"kind":"cast.request","session":e}));
-            } else if kind=="request" && e["protocol"]=="AirPlay" && self.sessions[&id]["accepted"]!=true {
-                // SETUP can recur on an open control connection after the
-                // previous consent toast expired. Renew the pending request.
-                self.events.push_back(json!({"kind":"cast.request","session":self.sessions[&id]}));
-            } else if matches!(kind, "video" | "audio") {
+            } else {
                 let session = self.sessions.get_mut(&id).unwrap();
-                if kind == "video" {
+                let mut changed = false;
+                if e["width"].as_u64().unwrap_or(0) > 0 && e["height"].as_u64().unwrap_or(0) > 0 {
+                    changed = session["width"] != e["width"] || session["height"] != e["height"];
                     session["width"] = e["width"].clone();
                     session["height"] = e["height"].clone();
+                    session["mediaType"] = json!("video");
+                } else if e["mediaType"] == "video" && session["mediaType"] != "video" {
+                    session["mediaType"] = json!("video");
+                    changed = true;
                 }
                 if session["accepted"] == true {
-                    self.events
-                        .push_back(json!({"kind":"cast.window","session":session.clone()}));
+                    if changed || matches!(kind, "video" | "audio") {
+                        self.events.push_back(json!({"kind":"cast.window","session":session.clone()}));
+                    }
+                } else if kind == "request" && e["protocol"] == "AirPlay" {
+                    session["createdAt"] = json!(crate::service::now_ms());
+                    self.events.push_back(json!({"kind":"cast.request","session":session.clone()}));
                 }
             }
         } else if matches!(kind, "ended" | "error") {
@@ -522,6 +528,24 @@ mod tests {
         }
     }
     #[test]
+    fn airplay_video_setup_promotes_audio_session_before_decoding() {
+        let root = tempfile::tempdir().unwrap();
+        let mut r = Receivers::new(root.path().to_owned(), root.path().join("bin"));
+        r.airplay_id = "0123".into();
+        r.event(json!({"event":"request","id":"0123","protocol":"AirPlay","mediaType":"audio"}));
+        r.control("0123", "accept", json!({})).unwrap();
+        r.events.clear();
+        r.event(json!({"event":"request","id":"0123","protocol":"AirPlay","mediaType":"video"}));
+        assert_eq!(r.window("0123").unwrap()["mediaType"], "video");
+        assert_eq!(r.events.pop_front().unwrap()["kind"], "cast.window");
+        r.event(json!({"event":"request","id":"0123","protocol":"AirPlay","width":1080,"height":1920}));
+        let update = r.events.pop_front().unwrap();
+        assert_eq!(update["session"]["width"], 1080);
+        assert_eq!(update["session"]["height"], 1920);
+        r.event(json!({"event":"audio","id":"0123","protocol":"AirPlay","mediaType":"audio"}));
+        assert_eq!(r.window("0123").unwrap()["mediaType"], "video");
+    }
+    #[test]
     fn consent_blocks_playback_controls_and_rejects_stale_airplay_sessions() {
         let root = tempfile::tempdir().unwrap();
         let mut r = Receivers::new(root.path().to_owned(), root.path().join("bin"));

src/session.rs

@@ -901,7 +901,8 @@ impl Agent {
                 "cast.window" => {
                     let _=self.core("notification.remove",json!({"plugin":"","id":format!("cast.{id}")}));
                     let w=session["width"].as_u64().unwrap_or(0);let h=session["height"].as_u64().unwrap_or(0);
-                    let width=if w>0 { w.min(1920) } else {640};let height=if w>0&&h>0 {width*h/w}else{240};
+                    let video=session["mediaType"]=="video";
+                    let width=if w>0 { w.min(1920) } else if video {1280} else {640};let height=if w>0&&h>0 {width*h/w}else if video {720}else{240};
                     let frame=self.receivers.lock().unwrap().directory.join(format!("{id}.frame"));
                     push(&self.commands,json!({"kind":"cast.window","id":id,"width":width,"height":height,"frame":frame}));
                 },

ui/src/cast-player.tsx

@@ -3,10 +3,10 @@ import {IconMusic,IconPlayerPause,IconPlayerPlay,IconPlayerStop,IconVolume,IconV
 import {api} from './api';
 import {CastRange} from './cast-range';
 import {t} from './i18n';
-type Session={id:string;protocol:string;width?:number;height?:number;error?:string};
+type Session={id:string;protocol:string;mediaType?:string;width?:number;height?:number;error?:string};
 export function CastPlayer(){
  const id=location.pathname.split('/').pop()!,[session,setSession]=useState<Session|null>(null),[status,setStatus]=useState({paused:false,position:0,duration:0,seekable:false}),[error,setError]=useState(''),[volume,setVolume]=useState(1),[seek,setSeek]=useState<number|null>(null),[busy,setBusy]=useState(false),[visible,setVisible]=useState(true);
- const timer=useRef<ReturnType<typeof setTimeout>|null>(null),root=useRef<HTMLElement>(null),video=!!session?.width&&!!session?.height;
+ const timer=useRef<ReturnType<typeof setTimeout>|null>(null),root=useRef<HTMLElement>(null),video=session?.mediaType==='video'||(!!session?.width&&!!session?.height);
  function reveal(){setVisible(true);if(timer.current)clearTimeout(timer.current);if(video&&!status.paused)timer.current=setTimeout(()=>{if(!root.current?.querySelector(':focus-visible'))setVisible(false);},3000);}
  useEffect(()=>{reveal();return()=>{if(timer.current)clearTimeout(timer.current);};},[video,status.paused]);
  useEffect(()=>{let alive=true;const poll=()=>void Promise.all([api<Session>('cast.window',{id}),api<typeof status>('cast.control',{id,action:'status'})]).then(([session,status])=>{if(alive){setSession(session);setStatus(status);}}).catch(e=>{if(alive)setError(e.message);});poll();const interval=setInterval(poll,1000);return()=>{alive=false;clearInterval(interval);};},[id]);