Add ripnsfw provider (Doodstream/Lulustream leak aggregator)

ripnsfw.com serves its entire catalogue as a published-Google-Sheet CSV
with no native pagination/search API, so the provider fetches and
parses that CSV once (cached 180s) and does feed/search/pagination/sort
in memory. Each row's Doodstream/Lulustream embed links resolve to
formats[] via the existing (previously unused) doodstream/lulustream
redirect proxies.

Also fixes check.py's follow_proxy_redirect, which used HEAD even
though these redirect-proxy routes only accept GET/POST, so it never
actually resolved the redirect; extends the CF-protected host list
(suffix matching + ripnsfw.com's client-only-SPA 404 page, dood.video,
tnmr.org) so known sandbox/CDN-IP-reputation failures are reported as
warnings instead of errors.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01QTqf6orbHZ9rFFpVpcgzcR
This commit is contained in:
Simon
2026-09-18 19:22:49 +00:00
parent 9c078f1b60
commit 534ee4b5ad
4 changed files with 460 additions and 18 deletions

View File

@@ -82,15 +82,36 @@ _CF_PROTECTED_HOSTS = {
# hdplayer.gives returns 200 to plain curl but 404 to curl_cffi's JA3 — the
# HLS list/enc... endpoint is request-bound to the embed-page TLS context.
"hdplayer.gives",
# ripnsfw.com is a client-only SPA on a static host with no server-side
# routing: deep links like /model/{slug}/{id} 404 at the HTTP layer. Its
# custom 404 page stashes the path and JS-redirects to `/`, which restores
# the route via history.replaceState — works in a real browser/webview,
# never for a plain HTTP client (curl, yt-dlp, this checker).
"ripnsfw.com",
"www.ripnsfw.com",
}
# Same idea as _CF_PROTECTED_HOSTS but for CDNs that mint a random subdomain per
# signed URL, so an exact-hostname set can never match. Matched by suffix.
_CF_PROTECTED_SUFFIXES = (
# doodstream's final CDN edge (reached via ripnsfw's /proxy/doodstream/...).
# Resolves to a loopback/refused address from non-residential egress —
# observed consistently across independently signed tokens.
".dood.video",
# lulustream's signed HLS edge (reached via ripnsfw's /proxy/lulustream/...).
# Returns 403/522 to datacenter IPs regardless of UA/TLS impersonation.
".tnmr.org",
)
def _is_cf_protected(url: str) -> bool:
"""Return True if the URL's host is known to be CF-protected."""
try:
from urllib.parse import urlparse
host = urlparse(url).hostname or ""
return host in _CF_PROTECTED_HOSTS
if host in _CF_PROTECTED_HOSTS:
return True
return any(host.endswith(suffix) for suffix in _CF_PROTECTED_SUFFIXES)
except Exception:
return False
@@ -226,18 +247,32 @@ def is_media_file_url(url: str) -> bool:
def follow_proxy_redirect(url: str) -> str:
"""If url is a localhost proxy URL, follow one redirect to get the real URL."""
if "127.0.0.1" not in url and "localhost" not in url:
return url
try:
r = requests.head(url, timeout=HTTP_TIMEOUT, allow_redirects=False)
if r.status_code in (301, 302, 303, 307, 308):
"""If url is a localhost proxy URL, follow redirects to the true final URL.
A local redirect proxy sometimes lands on an intermediate CDN host that
itself redirects again (e.g. ripnsfw's doodstream proxy resolves to a
cloudatacdn.com hop that 302s a second time to the real dood.video edge),
so this keeps following as long as each hop is itself a redirect.
"""
current = url
for _ in range(5):
try:
r = requests.head(current, timeout=HTTP_TIMEOUT, allow_redirects=False)
if r.status_code == 405:
# Some redirect proxies only register GET/POST, not HEAD.
r = requests.get(
current, timeout=HTTP_TIMEOUT, allow_redirects=False, stream=True
)
r.close()
if r.status_code not in (301, 302, 303, 307, 308):
break
loc = r.headers.get("Location", "")
if loc and "127.0.0.1" not in loc and "localhost" not in loc:
return loc
except Exception:
pass
return url
if not loc or loc == current:
break
current = loc
except Exception:
break
return current
def titles_match(a: str, b: str) -> bool:
@@ -290,7 +325,8 @@ def check_video(video: dict, channel_id: str, results: Results, run_ytdlp: bool)
continue
ok, code = http_ok(furl, headers=fheaders)
if not ok:
if _is_cf_protected(furl):
resolved = follow_proxy_redirect(furl)
if _is_cf_protected(furl) or _is_cf_protected(resolved):
results.warn(
channel_id,
f"{label} format[{j}]: unreachable HTTP={code} (CF-protected host, expected)"
@@ -384,11 +420,19 @@ def check_video(video: dict, channel_id: str, results: Results, run_ytdlp: bool)
results.info(channel_id, f"{label} format[{j}]: yt-dlp extract {furl}")
yt, stderr = ytdlp_extract(furl, extra_args=extra_args)
if yt is None:
results.err(
channel_id,
f"{label} format[{j}]: yt-dlp failed for {furl}"
+ (f": {stderr[:200]}" if stderr else ""),
)
resolved = follow_proxy_redirect(furl)
if _is_cf_protected(furl) or _is_cf_protected(resolved):
results.warn(
channel_id,
f"{label} format[{j}]: yt-dlp failed for {furl} (CF-protected host, expected)"
+ (f": {stderr[:200]}" if stderr else ""),
)
else:
results.err(
channel_id,
f"{label} format[{j}]: yt-dlp failed for {furl}"
+ (f": {stderr[:200]}" if stderr else ""),
)
else:
yt_fmts = yt.get("formats") or []
yt_direct = yt.get("url")