From 95700c67a5c4ad2d187f6b07061ab1de11ef924e Mon Sep 17 00:00:00 2001 From: Simon Date: Sat, 10 Oct 2026 08:36:25 +0000 Subject: [PATCH] check.py: apply the channel's status ytdlpCommand to yt-dlp probes; log open decision Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01MUYdCh2nCVsQYgJAT7gwSi --- check.py | 32 ++++++++++++++++++++++++++++++-- docs/provider-fix-todo.md | 10 +++++++++- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/check.py b/check.py index 9bd23b2..1c1d33a 100644 --- a/check.py +++ b/check.py @@ -18,6 +18,8 @@ Checks per channel: - yt-dlp -J on each format.url that is not a direct media file """ +import os +import shlex import sys import json import subprocess @@ -178,6 +180,29 @@ def http_ok(url: str, headers: dict | None = None) -> tuple[bool, int]: return False, -3 +# Per-channel yt-dlp options taken from the channel's declared `ytdlpCommand` +# (e.g. "yt-dlp --impersonate chrome-120"), which real clients apply. Probing +# without them reports failures (TLS-fingerprint 403/502) no client sees. +_CHANNEL_YTDLP_ARGS: dict[str, list[str]] = {} + + +def register_ytdlp_command(channel_id: str, command: Optional[str]) -> None: + if not command: + return + try: + parts = shlex.split(command) + except ValueError: + return + if parts and os.path.basename(parts[0]) in ("yt-dlp", "yt_dlp"): + parts = parts[1:] + if parts: + _CHANNEL_YTDLP_ARGS[channel_id] = parts + + +def ytdlp_args_for(channel_id: str) -> list[str]: + return list(_CHANNEL_YTDLP_ARGS.get(channel_id, [])) + + def ytdlp_extract(url: str, extra_args: list[str] | None = None) -> tuple[Optional[dict], str]: """Run yt-dlp -j and return (parsed_info_or_None, stderr_text). @@ -389,7 +414,7 @@ def check_video(video: dict, channel_id: str, results: Results, run_ytdlp: bool) ytdlp_url = vurl if vurl and "127.0.0.1" not in vurl and "localhost" not in vurl else "" if not formats and ytdlp_url and is_page_url(ytdlp_url): results.info(channel_id, f"{label}: yt-dlp extract {ytdlp_url}") - yt, stderr = ytdlp_extract(ytdlp_url) + yt, stderr = ytdlp_extract(ytdlp_url, extra_args=ytdlp_args_for(channel_id)) if yt is None: if _is_cf_protected(ytdlp_url): results.warn( @@ -457,7 +482,7 @@ def check_video(video: dict, channel_id: str, results: Results, run_ytdlp: bool) for hk, hv in fheaders.items(): extra_args += ["--add-header", f"{hk}:{hv}"] results.info(channel_id, f"{label} format[{j}]: yt-dlp extract {furl}") - yt, stderr = ytdlp_extract(furl, extra_args=extra_args) + yt, stderr = ytdlp_extract(furl, extra_args=ytdlp_args_for(channel_id) + extra_args) if yt is None: resolved = follow_proxy_redirect(furl) if _is_cf_protected(furl) or _is_cf_protected(resolved): @@ -581,6 +606,9 @@ def main(): print("ERROR: no channels in /api/status response") sys.exit(1) + for ch in channels: + register_ytdlp_command(ch["id"], ch.get("ytdlpCommand")) + print(f"status OK — {len(channels)} channels") # --- per-channel checks --- diff --git a/docs/provider-fix-todo.md b/docs/provider-fix-todo.md index db3fbb8..f53bd72 100644 --- a/docs/provider-fix-todo.md +++ b/docs/provider-fix-todo.md @@ -28,7 +28,7 @@ Status: `[ ]` open, `[~]` in progress, `[x]` fixed and verified, `[-]` not repro | [x] | `hypnotube` | page URLs 404 on all sampled items | the 404 was a check.py artifact (server answers HEAD 404, GET 200); check.py now confirms with a ranged GET. Real bug found on the way: the site returns ONE results page per searchgate POST, so any 2nd search on the shared session (and every page 2) came back empty. Each (query, page) now uses its own isolated session. 3 consecutive searches + page 2 verified | | [x] | `fullpornerorg` | `lulustream` proxy 403 / 502 | three separate causes: (1) hoster files deleted ("File is no longer available") but still listed -> now probed and dropped; (2) `myvidplay.com` embeds lost their host (bare ID sent to lulustream, wrong namespace) -> routed to the doodstream proxy with host kept, which now allows myvidplay/playmogo; (3) tnmr.org m3u8 is TLS-fingerprint gated (works with chrome TLS = the channel's `--impersonate chrome-120`; plain yt-dlp 403s are the checker's expected CF warnings). myvidplay playback could not be verified: the embed now returns a Turnstile captcha shell to this client | | [-] | `ripnsfw` | `doodstream` format 502 | lulustream format on the same items resolves (302). The doodstream format cannot be verified here: `*.dood.video` resolves to 127.0.0.1 on this host's resolver, so the CDN is unreachable. Needs the production network | -| [ ] | `freepornvideosxxx` | yt-dlp 502 on page URL | may be transient, verify | +| [-] | `freepornvideosxxx` | yt-dlp 502 on page URL | listing/search/p2 fine (24 items). Page URL is yt-dlp-native (html5 `` 480/720/2160). The 502 is the site's edge answering plain yt-dlp's TLS fingerprint (3 of 4 items 502; 4/4 fine with `--impersonate chrome-120`; curl always 200). The channel declares that command but see "Open decision" below: status never delivers it | | [ ] | `hdporn92` | m3u8 formats time out (curl + yt-dlp) | morencius.com streams | ## 3. Thumbnails @@ -64,6 +64,14 @@ Status: `[ ]` open, `[~]` in progress, `[x]` fixed and verified, `[-]` not repro |---|---|---| | [ ] | `check.py` "url points to hottub server" | contradicts the hottub skill (proxy-needing videos use the proxy as `item.url`). Affects spankbang, hqporner, pornhd3x, pimpbunny, sxyprn, noodlemagazine. Decide: exempt proxy URLs that resolve, rather than hard-fail. | +## Open decision (needs the owner) + +`ytdlpCommand` set on providers (`Channel.ytdlpCommand`, ~90 channels, e.g. `yt-dlp --impersonate chrome-120`) is **never sent to clients**: +`decorate_channel` in `src/providers/mod.rs` builds `ChannelView.ytdlpCommand` from a hardcoded `match` (only `pimpbunny`), and +`decorates_channel_with_group_and_tags` pins `hsex` to `None`. `/api/status` therefore reports `null` for all of them. Either deliver +`channel.ytdlpCommand` (clients then need yt-dlp impersonation support) or drop the dead declarations. Channels that only fail +under plain yt-dlp TLS (freepornvideosxxx, missav, ...) depend on this. `check.py` now applies whatever `/api/status` exposes. + ## Found while fixing (pre-existing, unrelated) - `src/proxy.rs`: the `pornhub-thumb` resource was registered as `/proxy/pornhub-thumb` inside the `/proxy` scope (unreachable); fixed alongside `aylo-thumb`. No provider uses it today.