advanced probing

backend/main.py

@@ -11,6 +11,7 @@ from yt_dlp.networking.impersonate import ImpersonateTarget
 from curl_cffi import requests as impersonate_requests
 import threading
 import io
+import time
 from urllib.parse import urljoin
 
 # Browser to impersonate at the TLS/HTTP layer. Some origins (e.g. the
@@ -163,6 +164,135 @@ def videos_proxy():
     except Exception as e:
         return jsonify({"error": str(e)}), 500
 
+# Cache of yt-dlp metadata resolutions so the on-demand probe endpoint doesn't
+# re-extract the same video on every hover/scroll. Signed media URLs expire, so
+# entries are intentionally short-lived.
+RESOLVE_CACHE_TTL = 300
+_resolve_cache = {}
+_resolve_cache_lock = threading.Lock()
+
+# Per-format fields the frontend needs to rank formats and build stream/probe
+# URLs (see resolveStreamSources/rankFormats in videos.js). Everything else in a
+# yt-dlp format dict is dropped to keep the payload small.
+_RESOLVE_FORMAT_FIELDS = ('url', 'http_headers', 'height', 'width', 'tbr',
+                          'fps', 'vcodec', 'acodec', 'ext', 'video_ext', 'quality')
+
+# Some channels surface pages that yt-dlp can't extract because the video is
+# embedded in a third-party JS player iframe (e.g. the xtremestream family used
+# by tube.perverzija.com). The player page declares its HLS playlist URL as
+# `m3u8_loader_url + video_id`, so we scrape the page for the embed iframe, then
+# read those two variables out of the player to reconstruct the stream URL.
+_EMBED_IFRAME_RE = re.compile(r'''<iframe[^>]+src=["']([^"']+)''', re.I)
+_EMBED_LOADER_RE = re.compile(r'''m3u8_loader_url\s*=\s*[`'"]([^`'"]+)[`'"]''')
+_EMBED_VIDEOID_RE = re.compile(r'''video_id\s*=\s*[`'"]([^`'"]+)[`'"]''')
+
+def resolve_unsupported_embed(page_url):
+    """Best-effort resolver for iframe-embedded JS players yt-dlp can't handle.
+    Returns an info-like dict ({'url','formats','http_headers','is_live'}) whose
+    single format is the embed's HLS playlist, or None if nothing was found."""
+    try:
+        sess = get_impersonate_session()
+        page = sess.get(page_url, headers={'Referer': page_url}, timeout=15)
+        embed_url = None
+        for src in _EMBED_IFRAME_RE.findall(page.text):
+            candidate = urljoin(page_url, src)
+            if '/player/' in candidate or 'index.php?data=' in candidate:
+                embed_url = candidate
+                break
+        if not embed_url:
+            return None
+
+        player = sess.get(embed_url, headers={'Referer': page_url}, timeout=15)
+        loader = _EMBED_LOADER_RE.search(player.text)
+        video_id = _EMBED_VIDEOID_RE.search(player.text)
+        if not (loader and video_id):
+            return None
+        stream_url = loader.group(1) + video_id.group(1)
+
+        parsed = urllib.parse.urlparse(embed_url)
+        referer = f"{parsed.scheme}://{parsed.netloc}/"
+        headers = {'Referer': referer}
+        return {
+            'url': stream_url,
+            'is_live': False,
+            'http_headers': headers,
+            'formats': [{'url': stream_url, 'ext': 'm3u8', 'protocol': 'm3u8', 'http_headers': headers}],
+        }
+    except Exception:
+        return None
+
+@app.route('/api/resolve', methods=['POST', 'GET'])
+def resolve_video():
+    """Resolve a page URL to its playable formats via yt-dlp and return them as
+    JSON. The frontend calls this on demand (when a card is hovered or scrolled
+    into view) to learn the real media URLs so it can background-probe them for
+    direct, proxy-free playability."""
+    if request.method == 'POST':
+        source = request.json or {}
+        video_url = source.get('url')
+    else:
+        source = request.args
+        video_url = request.args.get('url')
+
+    if not video_url:
+        return jsonify({"error": "No URL provided"}), 400
+
+    now = time.time()
+    with _resolve_cache_lock:
+        cached = _resolve_cache.get(video_url)
+        if cached and cached[0] > now:
+            return jsonify(cached[1])
+
+    ydl_opts = {
+        'quiet': True,
+        'no_warnings': True,
+        'skip_download': True,
+        # Match /api/stream so the resolved formats reflect what playback will
+        # actually fetch from fingerprinting origins.
+        'impersonate': ImpersonateTarget.from_str(IMPERSONATE_TARGET),
+    }
+    passthrough_headers = collect_passthrough_headers(source)
+    if passthrough_headers:
+        ydl_opts['http_headers'] = passthrough_headers
+
+    try:
+        with yt_dlp.YoutubeDL(ydl_opts) as ydl:
+            info = ydl.extract_info(video_url, download=False)
+    except Exception as e:
+        # Many channels point at sites yt-dlp can't extract ("Unsupported URL").
+        # That's not fatal here -- the embed fallback below may still find a
+        # stream, and otherwise we return empty formats so playback falls back to
+        # the proxy.
+        app.logger.info("[resolve] yt-dlp could not extract %s: %s", video_url, e)
+        info = None
+
+    # Fall back to scraping iframe-embedded JS players yt-dlp doesn't support.
+    if not (info and (info.get('formats') or info.get('url'))):
+        embed = resolve_unsupported_embed(video_url)
+        if embed:
+            info = embed
+
+    formats = []
+    for fmt in ((info.get('formats') if info else None) or []):
+        if not fmt.get('url'):
+            continue
+        formats.append({k: fmt.get(k) for k in _RESOLVE_FORMAT_FIELDS if fmt.get(k) is not None})
+
+    result = {
+        'url': info.get('url') if info else None,
+        'http_headers': (info.get('http_headers') if info else None) or {},
+        'isLive': bool(info.get('is_live')) if info else False,
+        'formats': formats,
+    }
+
+    with _resolve_cache_lock:
+        # Drop expired entries so the cache doesn't grow without bound.
+        for key in [k for k, v in _resolve_cache.items() if v[0] <= now]:
+            _resolve_cache.pop(key, None)
+        _resolve_cache[video_url] = (now + RESOLVE_CACHE_TTL, result)
+
+    return jsonify(result)
+
 @app.route('/api/image', methods=['GET', 'HEAD'])
 def image_proxy():
     image_url = request.args.get('url')
@@ -664,7 +794,19 @@ def stream_video():
 
         with yt_dlp.YoutubeDL(ydl_opts) as ydl:
             # Extract the info
-            info = ydl.extract_info(video_url, download=False)
+            try:
+                info = ydl.extract_info(video_url, download=False)
+            except Exception as ydl_err:
+                # yt-dlp can't extract iframe-embedded JS players; scrape the
+                # embed for its HLS playlist and proxy that directly instead.
+                embed = resolve_unsupported_embed(video_url)
+                if not embed:
+                    raise
+                dbg(f"embed fallback resolved {video_url} -> {embed['url']}")
+                if request.method == 'HEAD':
+                    return Response("", status=200, content_type='application/vnd.apple.mpegurl')
+                return proxy_hls_playlist(embed['url'], embed['http_headers'].get('Referer'),
+                                          upstream_headers=embed['http_headers'])
             dbg(f"yt_dlp extractor={info.get('extractor')} protocol={info.get('protocol')}")
             
             # Try to get the URL from the info dict (works for progressive downloads)