Pick formats by decode cost, and fix auto picture-in-picture

Two things, both about playing several videos at once.

Capping the resolution per panel wasn't enough, because pixel count isn't
the only cost. A split panel now also prefers a progressive file over HLS
-- every HLS panel runs its own JavaScript demuxer over every segment, so
four panels means four media pipelines doing work a plain MP4 skips
entirely -- and H.264 over AV1 or VP9, which are often decoded in software
and are a cliff rather than a gradient, and 30fps over 60. The height
ceiling still comes first, so cheapness cannot argue a panel into a bigger
picture than it should have, and every format stays reachable as fallback.
The preloaded step's hls.js instances now park after buffering one
fragment and resume when the reader swipes to them, instead of fetching
and demuxing ahead for a step nobody reached.

Auto picture-in-picture had been implemented since the custom player was
written and had never worked. requestPictureInPicture() from a
visibilitychange handler carries no user activation, browsers refuse those,
and .catch(() => {}) swallowed the refusal -- so it failed silently every
time, in the reels feed and the standalone player alike. The declarative
autoPictureInPicture attribute is the form made for this: the browser is
told in advance which video should follow the reader out. The imperative
call stays as a fallback.

With panels there are several candidates and only one window, so binding
every pane made them race for it. The feed picks one deliberately -- the
panel you can hear, or the first if they are all muted -- re-picks when the
step or a mute switch changes, and releases it on close.

Whether a window actually opens is browser policy, not ours: Safari honours
the attribute, Chrome honours it for installed apps.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01QBDkEXP4htyXTCZUwMLphd

frontend/js/customPlayer.js

@@ -212,8 +212,24 @@ App.customPlayer = App.customPlayer || {};
         }
     };
 
+    // Asking for picture-in-picture the moment a tab is hidden is a request
+    // with no user gesture behind it, and browsers refuse those -- which is why
+    // the imperative call below fails silently. `autoPictureInPicture` is the
+    // declarative form made for exactly this: the browser is told in advance
+    // which video should follow the reader out, and does it itself. Safari
+    // honours it outright; Chrome honours it for installed apps. The call is
+    // kept as a fallback for anywhere the flag is ignored but the request is
+    // allowed.
+    App.customPlayer.setAutoPiP = function(video, on) {
+        if (!video) return;
+        try { video.autoPictureInPicture = !!on; } catch (err) { /* unsupported */ }
+        if (on) video.setAttribute('autopictureinpicture', '');
+        else video.removeAttribute('autopictureinpicture');
+    };
+
     App.customPlayer.bindAutoPiP = function(video) {
         if (!video) return function destroy() {};
+        App.customPlayer.setAutoPiP(video, true);
         const trigger = () => {
             if (document.visibilityState !== 'hidden') return;
             if (!document.pictureInPictureEnabled || video.disablePictureInPicture) return;
@@ -224,6 +240,7 @@ App.customPlayer = App.customPlayer || {};
         document.addEventListener('visibilitychange', trigger);
         window.addEventListener('pagehide', trigger);
         return function destroy() {
+            App.customPlayer.setAutoPiP(video, false);
             document.removeEventListener('visibilitychange', trigger);
             window.removeEventListener('pagehide', trigger);
         };

frontend/js/feed.js

@@ -264,7 +264,9 @@ App.feed = App.feed || {};
             pipBtn.addEventListener('click', onClick);
             cleanups.push(() => pipBtn.removeEventListener('click', onClick));
         }
-        cleanups.push(App.customPlayer.bindAutoPiP(video));
+        // Auto-PiP is not bound per pane: only one picture-in-picture window can
+        // exist, so binding every pane makes them race and the winner arbitrary.
+        // The feed picks one deliberately -- see updateAutoPiPTarget.
 
         const formatBtn = slide.querySelector('.feed-format-btn');
         const formatMenu = slide.querySelector('.feed-format-menu');
@@ -335,6 +337,8 @@ App.feed = App.feed || {};
         if (!video) return;
         if (slide.classList.contains('is-loaded')) {
             if (autoplay) {
+                // Picks up a stream that was parked after preloading.
+                if (video._hlsPlayer) video._hlsPlayer.startLoad();
                 video.muted = slide._muted !== false;
                 const playPromise = video.play();
                 if (playPromise && typeof playPromise.catch === 'function') playPromise.catch(() => {});
@@ -360,7 +364,13 @@ App.feed = App.feed || {};
 
         const resolved = slide._formatOverride
             ? App.videos.resolveSourceForFormat(videoData, slide._formatOverride)
-            : App.videos.resolveStreamSource(videoData, { maxHeight: paneHeightCap(slide) });
+            : App.videos.resolveStreamSource(videoData, {
+                maxHeight: paneHeightCap(slide),
+                // Several panels at once is a decoder problem, not a picture
+                // problem: prefer a progressive file over HLS, H.264 over AV1,
+                // 30fps over 60.
+                cheapest: paneCount() > 1
+            });
         if (!resolved || !resolved.url) {
             // No playable source -- treat exactly like a load failure so the
             // clip is dropped from the queue and the next one takes its place.
@@ -401,6 +411,16 @@ App.feed = App.feed || {};
             video._hlsPlayer = hls;
             hls.loadSource(streamUrl);
             hls.attachMedia(video);
+            if (!autoplay && split) {
+                // Buffered enough to start instantly, then stopped: four
+                // panels' worth of hls.js all fetching and demuxing ahead is
+                // work for a step the reader has not swiped to yet. loadSlide-
+                // Source runs again with autoplay when they do.
+                hls.on(HlsLib.Events.FRAG_BUFFERED, function once() {
+                    hls.off(HlsLib.Events.FRAG_BUFFERED, once);
+                    if (video._hlsPlayer === hls) hls.stopLoad();
+                });
+            }
             hls.on(HlsLib.Events.ERROR, (event, data) => {
                 if (data && data.fatal && video._hlsPlayer === hls) {
                     hls.destroy();
@@ -617,6 +637,8 @@ App.feed = App.feed || {};
             pane._muted = !pane._muted;
             syncMute();
             refreshFeedMuteState();
+            // The panel you can hear is the one that should follow you out.
+            updateAutoPiPTarget();
         });
 
         return pane;
@@ -906,6 +928,7 @@ App.feed = App.feed || {};
             });
         });
 
+        updateAutoPiPTarget();
         prefetchIfNeeded(clamped);
     };
 
@@ -971,6 +994,54 @@ App.feed = App.feed || {};
         });
     };
 
+    // ------------------------------------------------------------------
+    // Auto picture-in-picture
+    //
+    // Leaving the tab while reels is playing should carry the video out with
+    // the reader. With panes there are several candidates and only one window,
+    // so the choice is made here rather than left to whichever pane's handler
+    // fires first: the panel you can hear, or the first one if they are all
+    // muted.
+    // ------------------------------------------------------------------
+    const autoPipVideo = function() {
+        const slide = slidesByIndex.get(state.feedActiveIndex);
+        if (!slide) return null;
+        const panes = panesOf(slide);
+        if (!panes.length) return null;
+        const audible = panes.find((pane) => !pane._muted);
+        const chosen = audible || panes[0];
+        return chosen ? chosen.querySelector('.feed-video') : null;
+    };
+
+    // Marks the chosen video and clears every other, so the browser's own
+    // automatic handling targets the same one this would.
+    const updateAutoPiPTarget = function() {
+        const wanted = state.feedOpen ? autoPipVideo() : null;
+        slidesByIndex.forEach((slide) => {
+            panesOf(slide).forEach((pane) => {
+                const video = pane.querySelector('.feed-video');
+                if (video) App.customPlayer.setAutoPiP(video, video === wanted);
+            });
+        });
+    };
+
+    App.feed.updateAutoPiPTarget = updateAutoPiPTarget;
+
+    // The fallback for browsers that ignore the attribute but would allow the
+    // request. It fails without a user gesture in most of them, which is why
+    // the attribute above is the real mechanism.
+    const onFeedHidden = function() {
+        if (!state.feedOpen) return;
+        if (document.visibilityState !== 'hidden') return;
+        if (!document.pictureInPictureEnabled) return;
+        if (document.pictureInPictureElement) return;
+        const video = autoPipVideo();
+        if (!video || video.paused || video.ended || video.disablePictureInPicture) return;
+        video.requestPictureInPicture().catch(() => {});
+    };
+
+    let autoPipBound = false;
+
     App.feed.isOpen = function() {
         return !!state.feedOpen;
     };
@@ -1041,6 +1112,12 @@ App.feed = App.feed || {};
             scrollBound = true;
         }
 
+        if (!autoPipBound) {
+            document.addEventListener('visibilitychange', onFeedHidden);
+            window.addEventListener('pagehide', onFeedHidden);
+            autoPipBound = true;
+        }
+
         if (!hudActivityBound) {
             container.addEventListener('mousemove', wakeHud, { passive: true });
             container.addEventListener('pointerdown', wakeHud, { passive: true });
@@ -1075,6 +1152,7 @@ App.feed = App.feed || {};
             hudIdleTimer = null;
         }
         document.body.classList.remove('feed-hud-idle');
+        updateAutoPiPTarget();     // feedOpen is false now, so this clears them
         slidesByIndex.forEach((slide) => panesOf(slide).forEach(destroySlidePlayback));
         container.classList.remove('open');
         container.setAttribute('aria-hidden', 'true');

frontend/js/videos.js

@@ -1930,7 +1930,34 @@ App.videos = App.videos || {};
     // list is preferred (start with the last one). When a preferred height is
     // set, formats at or below it come first (best of those first), followed by
     // anything above it ordered closest-to-preferred first as a last resort.
-    App.videos.rankFormats = function(formats, preferredHeight) {
+    // How expensive a format is to decode, beyond its pixel count. Lower is
+    // cheaper, and these are ordered by how much they actually cost:
+    //
+    //   HLS means hls.js demuxes every segment in JavaScript before the browser
+    //   sees it. A progressive file skips that entirely -- and with several
+    //   panels each running their own instance, it is the difference between
+    //   one media pipeline and four.
+    //
+    //   AV1 and VP9 are frequently decoded in software; H.264 has hardware
+    //   support almost everywhere. That is a cliff, not a gradient.
+    //
+    //   60fps is twice the frames of 30fps for the same picture.
+    const decodeCost = function(fmt) {
+        const protocol = String(fmt.protocol || '').toLowerCase();
+        const ext = String(fmt.ext || '').toLowerCase();
+        const vcodec = String(fmt.vcodec || '').toLowerCase();
+        const streaming = protocol.indexOf('m3u8') >= 0 || protocol.indexOf('dash') >= 0
+            || ext === 'm3u8' || ext === 'mpd' ? 1 : 0;
+        const software = (vcodec.indexOf('av01') === 0 || vcodec.indexOf('vp9') === 0
+            || vcodec.indexOf('vp09') === 0) ? 1 : 0;
+        const highFps = App.videos.coerceNumber(fmt.fps) > 35 ? 1 : 0;
+        return streaming * 4 + software * 2 + highFps;
+    };
+
+    // `options.cheapest` ranks by what a machine has to do to play the format,
+    // once the height ceiling has been applied. It is for showing several
+    // videos at once, where the limit is the decoder rather than the picture.
+    App.videos.rankFormats = function(formats, preferredHeight, options) {
         if (!Array.isArray(formats) || formats.length === 0) return [];
         const candidates = formats
             .map((fmt, index) => ({ fmt, index }))
@@ -1944,12 +1971,17 @@ App.videos = App.videos || {};
             return false;
         });
         const pool = videoCandidates.length ? videoCandidates : candidates;
+        const cheapest = !!(options && options.cheapest);
         const score = (fmt) => {
             const height = App.videos.coerceNumber(fmt.height || fmt.quality);
             const width = App.videos.coerceNumber(fmt.width);
             const size = height || width;
             const bitrate = App.videos.coerceNumber(fmt.tbr || fmt.bitrate);
             const fps = App.videos.coerceNumber(fmt.fps);
+            // Cost is negated so that the same descending comparison puts the
+            // cheaper format first, and it outranks bitrate: a slightly softer
+            // picture that plays is worth more than a sharper one that stutters.
+            if (cheapest) return [size, -decodeCost(fmt), bitrate, fps];
             return [size, bitrate, fps];
         };
         // Tie-break on the original index so equal-quality formats start with
@@ -2013,7 +2045,8 @@ App.videos = App.videos || {};
             }
         }
 
-        const sources = App.videos.rankFormats(meta.formats, preferredHeight).map((fmt) => {
+        const ranking = { cheapest: !!(options && options.cheapest) };
+        const sources = App.videos.rankFormats(meta.formats, preferredHeight, ranking).map((fmt) => {
             // An *explicit* Referer (from the extractor) signals the upstream
             // enforces it; deriveReferer is only a best-effort fallback. The
             // browser can't set a cross-origin Referer, so refererRequired tells

tests/unit_formats.js

@@ -123,5 +123,37 @@ ok('the tighter of the two wins (preference)',
 ok('a cap never raises the preference',
    heightOf(resolveStreamSource(video, { maxHeight: 2160 })) === 480);
 
+console.log('\nranking by what it costs to decode');
+// Same picture, four ways of arriving at it.
+const mixed = [
+    { url: 'hls720',  height: 720, vcodec: 'avc1', protocol: 'm3u8_native', fps: 30 },
+    { url: 'av1720',  height: 720, vcodec: 'av01.0.05M.08', protocol: 'https', ext: 'mp4', fps: 30 },
+    { url: 'avc720',  height: 720, vcodec: 'avc1', protocol: 'https', ext: 'mp4', fps: 30, tbr: 900 },
+    { url: 'avc720p60', height: 720, vcodec: 'avc1', protocol: 'https', ext: 'mp4', fps: 60, tbr: 2000 },
+];
+const mixedVideo = { id: 'v2', url: 'https://example.com/w2', meta: { formats: mixed } };
+sandbox.App.storage.getPreferredQuality = () => 'auto';
+
+const best = (opts) => resolveStreamSource(mixedVideo, opts).url;
+ok('without the flag, bitrate still wins', best({}) === 'avc720p60', best({}));
+ok('cheapest avoids HLS demuxing in JS', best({ cheapest: true }) !== 'hls720');
+ok('cheapest avoids software-decoded AV1', best({ cheapest: true }) !== 'av1720');
+ok('cheapest avoids 60fps', best({ cheapest: true }) !== 'avc720p60');
+ok('cheapest picks progressive H.264 at 30fps',
+   best({ cheapest: true }) === 'avc720', best({ cheapest: true }));
+
+// Cheapness must not override the size ceiling, or a split panel would get a
+// bigger picture than it can afford just because it is cheap per pixel.
+const tall = [
+    { url: 'hls480', height: 480, vcodec: 'avc1', protocol: 'm3u8_native' },
+    { url: 'mp4_1080', height: 1080, vcodec: 'avc1', protocol: 'https', ext: 'mp4' },
+];
+const tallVideo = { id: 'v3', url: 'https://example.com/w3', meta: { formats: tall } };
+ok('the height ceiling still comes first',
+   resolveStreamSource(tallVideo, { maxHeight: 480, cheapest: true }).url === 'hls480');
+
+ok('every format stays reachable as fallback',
+   rankFormats(mixed, null, { cheapest: true }).length === mixed.length);
+
 console.log(`\n${failed ? 'FAILED' : 'OK'}: ${failed} check(s) failed`);
 process.exit(failed ? 1 : 0);