feat(youtube): detect the video's projection

Pre-selects the projection when the cardboard icon is clicked: 360, VR180
or flat from YouTube's player data, read by a new script in the page's own
world, and 3D from the frame itself, since YouTube doesn't report stereo
layouts. The two eyes' halves of a 3D frame are nearly identical.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

manifest.json

@@ -7,6 +7,11 @@
         {
             "matches": ["*://*.youtube.com/*"],
             "js": ["script.js"]
+        },
+        {
+            "matches": ["*://*.youtube.com/*"],
+            "js": ["youtube-page.js"],
+            "world": "MAIN"
         }
     ],
     "action": {

scripts/build.mjs

@@ -29,7 +29,12 @@ for (const file of STATIC_FILES) {
 }
 
 const context = await esbuild.context({
-    entryPoints: { script: 'src/youtube.js', generic: 'src/generic.js', popup: 'src/popup.js' },
+    entryPoints: {
+        script: 'src/youtube.js',
+        'youtube-page': 'src/youtube-page.js',
+        generic: 'src/generic.js',
+        popup: 'src/popup.js',
+    },
     outdir,
     bundle: true,
     format: 'iife',

src/stereo.js

@@ -0,0 +1,64 @@
+// Tells 3D videos from mono ones by their pixels: a 3D frame holds the two eyes' views next to
+// each other (or above each other), and those are nearly the same picture.
+
+const SIZE = 64; // the frame is scaled down to SIZE x SIZE, which is plenty to compare halves
+
+// Correlation of two equally long brightness arrays, from -1 to 1; null if either is flat
+function correlation(a, b) {
+    const n = a.length;
+    let meanA = 0;
+    let meanB = 0;
+    for (let i = 0; i < n; i++) {
+        meanA += a[i];
+        meanB += b[i];
+    }
+    meanA /= n;
+    meanB /= n;
+    let cov = 0;
+    let varA = 0;
+    let varB = 0;
+    for (let i = 0; i < n; i++) {
+        const da = a[i] - meanA;
+        const db = b[i] - meanB;
+        cov += da * db;
+        varA += da * da;
+        varB += db * db;
+    }
+    // a black or single-coloured frame (fades, intros) says nothing
+    if (varA / n < 25 || varB / n < 25) return null;
+    return cov / Math.sqrt(varA * varB);
+}
+
+// How alike the two halves of the current frame are, split 'LR' (left/right) or 'TB'
+// (top/bottom); null if the frame can't be read or is too plain to tell
+export function halvesCorrelation(video, split) {
+    if (video.readyState < video.HAVE_CURRENT_DATA) return null;
+    const canvas = document.createElement('canvas');
+    canvas.width = SIZE;
+    canvas.height = SIZE;
+    const context = canvas.getContext('2d', { willReadFrequently: true });
+    let pixels;
+    try {
+        context.drawImage(video, 0, 0, SIZE, SIZE);
+        pixels = context.getImageData(0, 0, SIZE, SIZE).data;
+    } catch {
+        return null; // not readable (CORS)
+    }
+    const half = SIZE / 2;
+    const a = new Float32Array(half * SIZE);
+    const b = new Float32Array(half * SIZE);
+    let i = 0;
+    for (let y = 0; y < (split === 'LR' ? SIZE : half); y++) {
+        for (let x = 0; x < (split === 'LR' ? half : SIZE); x++) {
+            const other = split === 'LR' ? (y * SIZE + x + half) * 4 : ((y + half) * SIZE + x) * 4;
+            const own = (y * SIZE + x) * 4;
+            a[i] = pixels[own] * 0.3 + pixels[own + 1] * 0.59 + pixels[own + 2] * 0.11;
+            b[i] = pixels[other] * 0.3 + pixels[other + 1] * 0.59 + pixels[other + 2] * 0.11;
+            i++;
+        }
+    }
+    return correlation(a, b);
+}
+
+// Above this the halves are taken to be the two eyes of one picture
+export const STEREO_THRESHOLD = 0.85;

src/vr-player.js

@@ -244,6 +244,7 @@ export class VRPlayer {
 
     setProjection(name) {
         this.projection = name;
+        if (this.select) this.select.value = name;
         this.disposeMeshes();
         this.meshes = PROJECTIONS[name].build(this.material);
         for (const mesh of this.meshes) this.scene.add(mesh);
@@ -273,6 +274,7 @@ export class VRPlayer {
         select.value = this.projection;
         styleControl(select);
         select.onchange = () => this.setProjection(select.value);
+        this.select = select;
         return select;
     }
 

src/youtube-page.js

@@ -0,0 +1,23 @@
+// Runs in YouTube's own JavaScript world, where the player's API is reachable, and answers
+// youtube.js' requests for the current video's projection data. Content scripts can't see
+// page objects, so the two sides talk through DOM events; the data goes as a JSON string
+// because objects don't cross between the worlds.
+
+document.addEventListener('video2webxr-info-request', () => {
+    let info = null;
+    try {
+        const player = document.getElementById('movie_player');
+        const response = player?.getPlayerResponse?.() || window.ytInitialPlayerResponse;
+        if (response) {
+            const formats = response.streamingData?.adaptiveFormats || [];
+            info = {
+                videoId: response.videoDetails?.videoId,
+                projections: [...new Set(formats.map((f) => f.projectionType).filter(Boolean))],
+                partialSpherical: !!response.playerConfig?.vrConfig?.partialSpherical,
+            };
+        }
+    } catch {
+        // no player yet; youtube.js falls back to its default
+    }
+    document.dispatchEvent(new CustomEvent('video2webxr-info', { detail: JSON.stringify(info) }));
+});

src/youtube.js

@@ -1,4 +1,5 @@
 import cardboardSvg from '../icons/cardboard.svg';
+import { halvesCorrelation, STEREO_THRESHOLD } from './stereo.js';
 import { VRPlayer } from './vr-player.js';
 
 let player = null;
@@ -60,6 +61,77 @@ videoElements.forEach((video) => {
 // Start observing the entire document for changes
 observer.observe(document, { childList: true, subtree: true });
 
+// Asks youtube-page.js, which can reach YouTube's player, for the video's projection data.
+// DOM events are dispatched synchronously, so the answer is there when dispatchEvent returns.
+function playerInfo() {
+    let info = null;
+    const onInfo = (event) => {
+        info = JSON.parse(event.detail);
+    };
+    document.addEventListener('video2webxr-info', onInfo);
+    document.dispatchEvent(new CustomEvent('video2webxr-info-request'));
+    document.removeEventListener('video2webxr-info', onInfo);
+    return info;
+}
+
+// The projection YouTube says the video has. YouTube serves both 360° and VR180 videos as
+// "MESH" on desktop; VR180 ones are marked partialSpherical.
+function detectProjection() {
+    const info = playerInfo();
+    console.log('video2webxr: YouTube player info', info);
+    if (!info) return 'EAC';
+    const is180 = info.partialSpherical;
+    if (info.projections.includes('MESH')) return is180 ? '180' : 'EAC';
+    if (info.projections.includes('EQUIRECTANGULAR')) return is180 ? '180' : '360';
+    return 'FLAT';
+}
+
+// YouTube doesn't say whether a video is 3D, so look at the frame: the two eyes' views sit
+// side by side (or above each other) and look nearly the same
+const STEREO_VARIANTS = {
+    EAC: [['LR', 'EAC_LR']],
+    360: [
+        ['TB', '360_TB'],
+        ['LR', '360_LR'],
+    ],
+    180: [['LR', '180_LR']],
+};
+
+const STEREO_CHECK_MS = 200; // a check scales one frame down to 64x64, so it's cheap
+const STEREO_MAX_CHECKS = 60000 / STEREO_CHECK_MS;
+
+// Samples frames until two agree on the layout, skipping ads (they play in the same <video>)
+// and frames too dark or plain to tell
+function detectStereo(vrPlayer, mono, votes = {}, attempt = 0) {
+    const variants = STEREO_VARIANTS[mono];
+    // stop once the player is gone or the user picked a projection themselves
+    if (!variants || player !== vrPlayer || vrPlayer.projection !== mono) return;
+    const adShowing = document.querySelector('#movie_player')?.classList.contains('ad-showing');
+    if (!adShowing) {
+        let vote = null;
+        for (const [split, stereo] of variants) {
+            const similarity = halvesCorrelation(vrPlayer.video, split);
+            if (similarity === null) continue;
+            console.log(`video2webxr: ${split} halves correlation ${similarity.toFixed(2)}`);
+            if (similarity > STEREO_THRESHOLD) {
+                vote = stereo;
+                break;
+            }
+            vote ??= mono;
+        }
+        if (vote) {
+            votes[vote] = (votes[vote] || 0) + 1;
+            if (votes[vote] === 2) {
+                vrPlayer.setProjection(vote);
+                return;
+            }
+        }
+    }
+    // give up after a minute of the actual video (ads don't count) without a clear answer
+    const checked = adShowing ? attempt : attempt + 1;
+    if (checked < STEREO_MAX_CHECKS) setTimeout(() => detectStereo(vrPlayer, mono, votes, checked), STEREO_CHECK_MS);
+}
+
 function EnableVRVideo(videoElement) {
     console.log(`enabling VR on ${videoElement}`);
 
@@ -73,8 +145,9 @@ function EnableVRVideo(videoElement) {
         spherecontrol.remove();
     }
 
-    // EAC is what YouTube serves 360° videos as on desktop
-    player = new VRPlayer(videoElement, 'EAC');
+    const projection = detectProjection();
+    player = new VRPlayer(videoElement, projection);
+    detectStereo(player, projection);
 
     const container = document.querySelector('.html5-video-container');
     container.appendChild(player.canvas);