diff --git a/backend/main.py b/backend/main.py index 1c86183..e9f4174 100644 --- a/backend/main.py +++ b/backend/main.py @@ -11,6 +11,7 @@ from yt_dlp.networking.impersonate import ImpersonateTarget from curl_cffi import requests as impersonate_requests import threading import io +import time from urllib.parse import urljoin # Browser to impersonate at the TLS/HTTP layer. Some origins (e.g. the @@ -163,6 +164,135 @@ def videos_proxy(): except Exception as e: return jsonify({"error": str(e)}), 500 +# Cache of yt-dlp metadata resolutions so the on-demand probe endpoint doesn't +# re-extract the same video on every hover/scroll. Signed media URLs expire, so +# entries are intentionally short-lived. +RESOLVE_CACHE_TTL = 300 +_resolve_cache = {} +_resolve_cache_lock = threading.Lock() + +# Per-format fields the frontend needs to rank formats and build stream/probe +# URLs (see resolveStreamSources/rankFormats in videos.js). Everything else in a +# yt-dlp format dict is dropped to keep the payload small. +_RESOLVE_FORMAT_FIELDS = ('url', 'http_headers', 'height', 'width', 'tbr', + 'fps', 'vcodec', 'acodec', 'ext', 'video_ext', 'quality') + +# Some channels surface pages that yt-dlp can't extract because the video is +# embedded in a third-party JS player iframe (e.g. the xtremestream family used +# by tube.perverzija.com). The player page declares its HLS playlist URL as +# `m3u8_loader_url + video_id`, so we scrape the page for the embed iframe, then +# read those two variables out of the player to reconstruct the stream URL. +_EMBED_IFRAME_RE = re.compile(r''']+src=["']([^"']+)''', re.I) +_EMBED_LOADER_RE = re.compile(r'''m3u8_loader_url\s*=\s*[`'"]([^`'"]+)[`'"]''') +_EMBED_VIDEOID_RE = re.compile(r'''video_id\s*=\s*[`'"]([^`'"]+)[`'"]''') + +def resolve_unsupported_embed(page_url): + """Best-effort resolver for iframe-embedded JS players yt-dlp can't handle. + Returns an info-like dict ({'url','formats','http_headers','is_live'}) whose + single format is the embed's HLS playlist, or None if nothing was found.""" + try: + sess = get_impersonate_session() + page = sess.get(page_url, headers={'Referer': page_url}, timeout=15) + embed_url = None + for src in _EMBED_IFRAME_RE.findall(page.text): + candidate = urljoin(page_url, src) + if '/player/' in candidate or 'index.php?data=' in candidate: + embed_url = candidate + break + if not embed_url: + return None + + player = sess.get(embed_url, headers={'Referer': page_url}, timeout=15) + loader = _EMBED_LOADER_RE.search(player.text) + video_id = _EMBED_VIDEOID_RE.search(player.text) + if not (loader and video_id): + return None + stream_url = loader.group(1) + video_id.group(1) + + parsed = urllib.parse.urlparse(embed_url) + referer = f"{parsed.scheme}://{parsed.netloc}/" + headers = {'Referer': referer} + return { + 'url': stream_url, + 'is_live': False, + 'http_headers': headers, + 'formats': [{'url': stream_url, 'ext': 'm3u8', 'protocol': 'm3u8', 'http_headers': headers}], + } + except Exception: + return None + +@app.route('/api/resolve', methods=['POST', 'GET']) +def resolve_video(): + """Resolve a page URL to its playable formats via yt-dlp and return them as + JSON. The frontend calls this on demand (when a card is hovered or scrolled + into view) to learn the real media URLs so it can background-probe them for + direct, proxy-free playability.""" + if request.method == 'POST': + source = request.json or {} + video_url = source.get('url') + else: + source = request.args + video_url = request.args.get('url') + + if not video_url: + return jsonify({"error": "No URL provided"}), 400 + + now = time.time() + with _resolve_cache_lock: + cached = _resolve_cache.get(video_url) + if cached and cached[0] > now: + return jsonify(cached[1]) + + ydl_opts = { + 'quiet': True, + 'no_warnings': True, + 'skip_download': True, + # Match /api/stream so the resolved formats reflect what playback will + # actually fetch from fingerprinting origins. + 'impersonate': ImpersonateTarget.from_str(IMPERSONATE_TARGET), + } + passthrough_headers = collect_passthrough_headers(source) + if passthrough_headers: + ydl_opts['http_headers'] = passthrough_headers + + try: + with yt_dlp.YoutubeDL(ydl_opts) as ydl: + info = ydl.extract_info(video_url, download=False) + except Exception as e: + # Many channels point at sites yt-dlp can't extract ("Unsupported URL"). + # That's not fatal here -- the embed fallback below may still find a + # stream, and otherwise we return empty formats so playback falls back to + # the proxy. + app.logger.info("[resolve] yt-dlp could not extract %s: %s", video_url, e) + info = None + + # Fall back to scraping iframe-embedded JS players yt-dlp doesn't support. + if not (info and (info.get('formats') or info.get('url'))): + embed = resolve_unsupported_embed(video_url) + if embed: + info = embed + + formats = [] + for fmt in ((info.get('formats') if info else None) or []): + if not fmt.get('url'): + continue + formats.append({k: fmt.get(k) for k in _RESOLVE_FORMAT_FIELDS if fmt.get(k) is not None}) + + result = { + 'url': info.get('url') if info else None, + 'http_headers': (info.get('http_headers') if info else None) or {}, + 'isLive': bool(info.get('is_live')) if info else False, + 'formats': formats, + } + + with _resolve_cache_lock: + # Drop expired entries so the cache doesn't grow without bound. + for key in [k for k, v in _resolve_cache.items() if v[0] <= now]: + _resolve_cache.pop(key, None) + _resolve_cache[video_url] = (now + RESOLVE_CACHE_TTL, result) + + return jsonify(result) + @app.route('/api/image', methods=['GET', 'HEAD']) def image_proxy(): image_url = request.args.get('url') @@ -664,7 +794,19 @@ def stream_video(): with yt_dlp.YoutubeDL(ydl_opts) as ydl: # Extract the info - info = ydl.extract_info(video_url, download=False) + try: + info = ydl.extract_info(video_url, download=False) + except Exception as ydl_err: + # yt-dlp can't extract iframe-embedded JS players; scrape the + # embed for its HLS playlist and proxy that directly instead. + embed = resolve_unsupported_embed(video_url) + if not embed: + raise + dbg(f"embed fallback resolved {video_url} -> {embed['url']}") + if request.method == 'HEAD': + return Response("", status=200, content_type='application/vnd.apple.mpegurl') + return proxy_hls_playlist(embed['url'], embed['http_headers'].get('Referer'), + upstream_headers=embed['http_headers']) dbg(f"yt_dlp extractor={info.get('extractor')} protocol={info.get('protocol')}") # Try to get the URL from the info dict (works for progressive downloads) diff --git a/frontend/js/videos.js b/frontend/js/videos.js index e24e85e..3dc7834 100644 --- a/frontend/js/videos.js +++ b/frontend/js/videos.js @@ -290,9 +290,6 @@ App.videos = App.videos || {}; items.forEach(v => { if (state.renderedVideoIds.has(v.id)) return; state.loadedVideos.push(v); - // Probe in the background whether this video's best source plays - // directly, so playback can bypass the proxy when proven. - App.videos.probeVideoSources(v); const card = document.createElement('div'); card.className = 'video-card'; @@ -410,6 +407,11 @@ App.videos = App.videos || {}; App.player.open(v, { originEl: card }); }; grid.appendChild(card); + // Resolve formats + probe direct playability on demand: when the + // card scrolls near the viewport, or the moment it's hovered. + cardVideo.set(card, v); + probeObserver.observe(card); + card.addEventListener('pointerenter', () => App.videos.resolveAndProbe(v), { once: true }); state.renderedVideoIds.add(v.id); }); @@ -650,7 +652,7 @@ App.videos = App.videos || {}; const isLive = !!(videoOrUrl && typeof videoOrUrl === 'object' && (videoOrUrl.isLive || (videoOrUrl.meta && videoOrUrl.meta.isLive))); if (typeof videoOrUrl === 'string') { - return videoOrUrl ? [{ url: videoOrUrl, referer: deriveReferer(videoOrUrl), userAgent: '', isLive }] : []; + return videoOrUrl ? [{ url: videoOrUrl, referer: deriveReferer(videoOrUrl), userAgent: '', isLive, refererRequired: false }] : []; } if (!videoOrUrl || typeof videoOrUrl !== 'object') return []; @@ -664,9 +666,14 @@ App.videos = App.videos || {}; } const sources = App.videos.rankFormats(meta.formats, preferredHeight).map((fmt) => { - const referer = headerValue(fmt.http_headers, 'Referer') || metaReferer || deriveReferer(fmt.url); + // An *explicit* Referer (from the extractor) signals the upstream + // enforces it; deriveReferer is only a best-effort fallback. The + // browser can't set a cross-origin Referer, so refererRequired tells + // callers (the probe) that direct playback can't work. + const explicitReferer = headerValue(fmt.http_headers, 'Referer') || metaReferer; + const referer = explicitReferer || deriveReferer(fmt.url); const userAgent = headerValue(fmt.http_headers, 'User-Agent') || metaUserAgent; - return { url: fmt.url, referer, userAgent, isLive }; + return { url: fmt.url, referer, userAgent, isLive, refererRequired: !!explicitReferer }; }); if (!sources.length) { @@ -676,7 +683,8 @@ App.videos = App.videos || {}; url: fallbackUrl, referer: metaReferer || deriveReferer(fallbackUrl), userAgent: metaUserAgent, - isLive + isLive, + refererRequired: !!metaReferer }); } } @@ -685,37 +693,41 @@ App.videos = App.videos || {}; App.videos.resolveStreamSource = function(videoOrUrl, options) { const sources = App.videos.resolveStreamSources(videoOrUrl, options); - return sources.length ? sources[0] : { url: '', referer: '', userAgent: '', isLive: false }; + return sources.length ? sources[0] : { url: '', referer: '', userAgent: '', isLive: false, refererRequired: false }; }; // Background "direct playability" probe. The backend proxy exists to work // around CORS, hotlink (403) protection, and TLS fingerprinting. When the // browser can fetch a media URL cross-origin and actually read the response // (CORS allowed, not blocked/403), playing it directly works and the proxy - // is pure overhead. We probe each loaded video's best source in the - // background and, only when proven, let the player skip the proxy. + // is pure overhead. CORS is an origin-level policy, so the answer is the + // same for every media URL served by a given host: we probe (and cache) + // once per host and let the player skip the proxy for any URL on a host + // that's been proven. const DIRECT_PROBE_TIMEOUT_MS = 8000; - // Only URLs that the player can hand straight to