From d7086ead270ed27040fbb48bbc873880c88adabd Mon Sep 17 00:00:00 2001 From: Simon Date: Sat, 5 Sep 2026 14:26:15 +0000 Subject: [PATCH] Play favorites from freshly resolved formats, not the page URL Streaming a page URL makes /api/stream re-run yt-dlp on every request -- slow, and a 500 on some sites -- so the player and the reels feed now wait for App.videos.ensureFormats() when an item has no formats (favorites, or a card clicked before its hover-resolve landed) and play a real media URL with the extractor's headers, the same path a hovered card takes. Favorites' download does the same. The page URL survives only as a last resort when resolution yields nothing. Two things in the proxy kept this site broken either way: heavyfetish serves media from paths with a trailing slash (/get_file/.../11097_720p.mp4/), which missed every extension test in stream_video and sent even a resolved media URL down the yt-dlp branch -- a full extraction per request, including every seek. Its CDN (st17.heavyfetish.com) also serves a certificate that expired 2026-02-16, so the upstream fetch failed verification and returned 500. A browser can't play such a host at all, which is much of why this proxy exists, so impersonate_get() now retries once without verification, logs it, and remembers the host so the doomed handshake isn't repeated for every range request. STREAM_TLS_VERIFY_ONLY=1 restores the hard failure. Verified in headless Chrome against the real site: a favorite holding only the page URL now resolves, streams (206, video/mp4, duration 3167.8s, readyState 4, no error) and shows "720p mp4 | 480p mp4" in the quality menu. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01QBDkEXP4htyXTCZUwMLphd --- backend/main.py | 56 ++++++++++++++++++++++++++++++++++------ frontend/js/favorites.js | 4 ++- frontend/js/feed.js | 15 +++++++++++ frontend/js/player.js | 35 ++++++++++++++++--------- 4 files changed, 89 insertions(+), 21 deletions(-) diff --git a/backend/main.py b/backend/main.py index 6de38e3..b5efc36 100644 --- a/backend/main.py +++ b/backend/main.py @@ -33,6 +33,39 @@ def get_impersonate_session(): _thread_local.session = sess return sess + +def _is_tls_verify_error(err): + message = str(err).lower() + return 'certificate' in message or 'curl: (60)' in message or 'ssl: ' in message + + +# Hosts already proven to fail certificate verification. A video is fetched in +# many range requests, so remembering the host keeps us from paying for a +# doomed TLS handshake on every one of them. +_tls_unverified_hosts = set() + + +def impersonate_get(url, **kwargs): + """Upstream GET that survives an origin with a broken certificate. + + Some media hosts serve expired certs (heavyfetish's stNN CDN, for one), which + a browser refuses outright -- part of why this proxy exists. The viewer's + connection to *us* stays verified either way, so rather than failing the + stream we retry once with verification off, and say so in the log. Set + STREAM_TLS_VERIFY_ONLY=1 to keep the hard failure instead.""" + host = urllib.parse.urlparse(url).netloc + if host in _tls_unverified_hosts: + return get_impersonate_session().get(url, verify=False, **kwargs) + try: + return get_impersonate_session().get(url, **kwargs) + except Exception as err: + strict = os.getenv('STREAM_TLS_VERIFY_ONLY', '').strip().lower() in ('1', 'true', 'yes', 'on') + if strict or not _is_tls_verify_error(err): + raise + app.logger.warning("[stream] TLS verification failed for %s (%s); retrying unverified", url, err) + _tls_unverified_hosts.add(host) + return get_impersonate_session().get(url, verify=False, **kwargs) + # Stream params that have dedicated meaning and must never be treated as headers. # `referer` is mapped to a real Referer header by collect_passthrough_headers, but # `live` is purely a playback hint and must not leak upstream as a header. @@ -438,14 +471,21 @@ def stream_video(): dbg(f"method={request.method} url={video_url} live={live_hint}") + def media_path(url): + # Some sites serve media from a path with a trailing slash + # (heavyfetish: /get_file/.../11097_720p.mp4/). Without stripping it, + # every extension test below misses and the URL takes the yt-dlp branch + # instead -- a full extraction per request, including every seek. + return urllib.parse.urlparse(url).path.lower().rstrip('/') + def is_hls(url): - return '.m3u8' in urllib.parse.urlparse(url).path + return '.m3u8' in media_path(url) def is_dash(url): - return urllib.parse.urlparse(url).path.lower().endswith('.mpd') + return media_path(url).endswith('.mpd') def guess_content_type(url): - path = urllib.parse.urlparse(url).path.lower() + path = media_path(url) if path.endswith('.m3u8'): return 'application/vnd.apple.mpegurl' if path.endswith('.mpd'): @@ -467,7 +507,7 @@ def stream_video(): return None def is_direct_media(url): - path = urllib.parse.urlparse(url).path.lower() + path = media_path(url) return any(path.endswith(ext) for ext in ('.mp4', '.m4v', '.m4s', '.ts', '.webm', '.mov')) def looks_like_m3u8_bytes(chunk): @@ -563,7 +603,7 @@ def stream_video(): if 'Range' in request.headers: safe_request_headers['Range'] = request.headers['Range'] - resp = get_impersonate_session().get(target_url, headers=safe_request_headers, stream=True, timeout=30, allow_redirects=True) + resp = impersonate_get(target_url, headers=safe_request_headers, stream=True, timeout=30, allow_redirects=True) # Some channel proxies (e.g. the "animeidhentai" hottub proxy) use # inverted hotlink protection: they 403 any request that carries a # Referer/Origin and only serve referer-less ones. Other CDNs require @@ -572,7 +612,7 @@ def stream_video(): dbg("upstream 403 with referer; retrying without referer/origin") resp.close() referer_less = {k: v for k, v in safe_request_headers.items() if k not in ('Referer', 'Origin')} - resp = get_impersonate_session().get(target_url, headers=referer_less, stream=True, timeout=30, allow_redirects=True) + resp = impersonate_get(target_url, headers=referer_less, stream=True, timeout=30, allow_redirects=True) if debug_enabled: dbg(f"upstream status={resp.status_code} content_type={resp.headers.get('Content-Type')} content_length={resp.headers.get('Content-Length')}") @@ -685,14 +725,14 @@ def stream_video(): for key, value in upstream_headers.items(): if value: headers[key] = value - resp = get_impersonate_session().get(playlist_url, headers=headers, stream=True, timeout=30) + resp = impersonate_get(playlist_url, headers=headers, stream=True, timeout=30) # See proxy_response: retry without referer for inverted hotlink # protection that 403s any refered request. if resp.status_code == 403 and ('Referer' in headers or 'Origin' in headers): dbg("playlist upstream 403 with referer; retrying without referer/origin") resp.close() referer_less = {k: v for k, v in headers.items() if k not in ('Referer', 'Origin')} - resp = get_impersonate_session().get(playlist_url, headers=referer_less, stream=True, timeout=30) + resp = impersonate_get(playlist_url, headers=referer_less, stream=True, timeout=30) base_url = resp.url if resp.status_code >= 400: diff --git a/frontend/js/favorites.js b/frontend/js/favorites.js index f01cf3d..cf32fee 100644 --- a/frontend/js/favorites.js +++ b/frontend/js/favorites.js @@ -187,8 +187,10 @@ App.favorites = App.favorites || {}; if (downloadBtn) { downloadBtn.onclick = (event) => { event.stopPropagation(); - App.videos.downloadVideo(item); App.videos.closeAllMenus(); + // Same as playback: resolve a real media URL first rather + // than pointing the download at the page URL. + App.videos.ensureFormats(item).then(() => App.videos.downloadVideo(item)); }; } const uploaderBtn = card.querySelector('.uploader-link'); diff --git a/frontend/js/feed.js b/frontend/js/feed.js index 8ff25ca..b2d66a4 100644 --- a/frontend/js/feed.js +++ b/frontend/js/feed.js @@ -331,6 +331,21 @@ App.feed = App.feed || {}; } return; } + + // A slide whose formats haven't been resolved yet carries only a page + // URL, and handing that to /api/stream makes the backend re-run yt-dlp + // per request (slow, and a hard failure on some sites). Resolve once, + // then load for real -- `_awaitingFormats` keeps a failed resolve from + // looping, so we still fall back to the page URL as a last resort. + const meta = videoData && (videoData.meta || videoData); + const hasFormats = !!(meta && Array.isArray(meta.formats) && meta.formats.length); + if (!hasFormats && !slide._awaitingFormats && App.videos && typeof App.videos.ensureFormats === 'function') { + slide._awaitingFormats = true; + App.videos.ensureFormats(videoData).then(() => { + if (slide._videoData === videoData) loadSlideSource(slide, videoData, autoplay); + }); + return; + } slide.classList.add('is-loaded'); const resolved = slide._formatOverride diff --git a/frontend/js/player.js b/frontend/js/player.js index ee91165..1f4ed46 100644 --- a/frontend/js/player.js +++ b/frontend/js/player.js @@ -815,19 +815,28 @@ App.player = App.player || {}; }); let destroyFormatMenu = bindFormats(); addCleanup(() => destroyFormatMenu()); + const meta = (source && typeof source === 'object') ? (source.meta || source) : null; + const hasFormats = !!(meta && Array.isArray(meta.formats) && meta.formats.length); // Items that arrive without formats -- a listing card clicked before its // hover-resolve finished, or a favorite (which deliberately stores no - // resolved formats, since their URLs expire) -- would otherwise show an - // empty quality menu. Playback already starts from the page URL via the - // proxy, so resolve in the background and rebuild the menu with the real - // qualities as soon as they land. - if (App.videos && typeof App.videos.ensureFormats === 'function') { - App.videos.ensureFormats(source).then((meta) => { - // A later open() may have taken over in the meantime; its own - // bindFormats() owns the menu then. - if (!meta || cp.source !== source) return; - destroyFormatMenu(); - destroyFormatMenu = bindFormats(); + // resolved formats, since their URLs expire) -- carry only a page URL, + // and they'd also show an empty quality menu. Resolve them first: the + // page-URL fallback makes /api/stream re-run yt-dlp on every single + // request, which is slow and fails outright on some sites, whereas a + // resolved format is a real media URL with the extractor's headers -- + // the same path a hovered card plays through. + let deferredStart = false; + if (!hasFormats && App.videos && typeof App.videos.ensureFormats === 'function') { + deferredStart = true; + App.videos.ensureFormats(source).then((resolved) => { + // A later open() (or a close) may have taken over in the + // meantime; that session owns the player now. + if (cp.source !== source) return; + if (resolved) { + destroyFormatMenu(); + destroyFormatMenu = bindFormats(); + } + playSources(source, { originEl: cp.originEl }); }); } bindGestures(cp.video); @@ -840,7 +849,9 @@ App.player = App.player || {}; document.body.style.overflow = 'hidden'; wakeHud(); - playSources(source, { originEl: cp.originEl }); + // Already-resolved sources start immediately; unresolved ones start from + // the ensureFormats() callback above (the spinner is already up). + if (!deferredStart) playSources(source, { originEl: cp.originEl }); }; App.player.close = function(opts) {