Add ripnsfw provider (Doodstream/Lulustream leak aggregator)
ripnsfw.com serves its entire catalogue as a published-Google-Sheet CSV with no native pagination/search API, so the provider fetches and parses that CSV once (cached 180s) and does feed/search/pagination/sort in memory. Each row's Doodstream/Lulustream embed links resolve to formats[] via the existing (previously unused) doodstream/lulustream redirect proxies. Also fixes check.py's follow_proxy_redirect, which used HEAD even though these redirect-proxy routes only accept GET/POST, so it never actually resolved the redirect; extends the CF-protected host list (suffix matching + ripnsfw.com's client-only-SPA 404 page, dood.video, tnmr.org) so known sandbox/CDN-IP-reputation failures are reported as warnings instead of errors. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01QTqf6orbHZ9rFFpVpcgzcR
This commit is contained in:
80
check.py
80
check.py
@@ -82,15 +82,36 @@ _CF_PROTECTED_HOSTS = {
|
||||
# hdplayer.gives returns 200 to plain curl but 404 to curl_cffi's JA3 — the
|
||||
# HLS list/enc... endpoint is request-bound to the embed-page TLS context.
|
||||
"hdplayer.gives",
|
||||
# ripnsfw.com is a client-only SPA on a static host with no server-side
|
||||
# routing: deep links like /model/{slug}/{id} 404 at the HTTP layer. Its
|
||||
# custom 404 page stashes the path and JS-redirects to `/`, which restores
|
||||
# the route via history.replaceState — works in a real browser/webview,
|
||||
# never for a plain HTTP client (curl, yt-dlp, this checker).
|
||||
"ripnsfw.com",
|
||||
"www.ripnsfw.com",
|
||||
}
|
||||
|
||||
# Same idea as _CF_PROTECTED_HOSTS but for CDNs that mint a random subdomain per
|
||||
# signed URL, so an exact-hostname set can never match. Matched by suffix.
|
||||
_CF_PROTECTED_SUFFIXES = (
|
||||
# doodstream's final CDN edge (reached via ripnsfw's /proxy/doodstream/...).
|
||||
# Resolves to a loopback/refused address from non-residential egress —
|
||||
# observed consistently across independently signed tokens.
|
||||
".dood.video",
|
||||
# lulustream's signed HLS edge (reached via ripnsfw's /proxy/lulustream/...).
|
||||
# Returns 403/522 to datacenter IPs regardless of UA/TLS impersonation.
|
||||
".tnmr.org",
|
||||
)
|
||||
|
||||
|
||||
def _is_cf_protected(url: str) -> bool:
|
||||
"""Return True if the URL's host is known to be CF-protected."""
|
||||
try:
|
||||
from urllib.parse import urlparse
|
||||
host = urlparse(url).hostname or ""
|
||||
return host in _CF_PROTECTED_HOSTS
|
||||
if host in _CF_PROTECTED_HOSTS:
|
||||
return True
|
||||
return any(host.endswith(suffix) for suffix in _CF_PROTECTED_SUFFIXES)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
@@ -226,18 +247,32 @@ def is_media_file_url(url: str) -> bool:
|
||||
|
||||
|
||||
def follow_proxy_redirect(url: str) -> str:
|
||||
"""If url is a localhost proxy URL, follow one redirect to get the real URL."""
|
||||
if "127.0.0.1" not in url and "localhost" not in url:
|
||||
return url
|
||||
try:
|
||||
r = requests.head(url, timeout=HTTP_TIMEOUT, allow_redirects=False)
|
||||
if r.status_code in (301, 302, 303, 307, 308):
|
||||
"""If url is a localhost proxy URL, follow redirects to the true final URL.
|
||||
|
||||
A local redirect proxy sometimes lands on an intermediate CDN host that
|
||||
itself redirects again (e.g. ripnsfw's doodstream proxy resolves to a
|
||||
cloudatacdn.com hop that 302s a second time to the real dood.video edge),
|
||||
so this keeps following as long as each hop is itself a redirect.
|
||||
"""
|
||||
current = url
|
||||
for _ in range(5):
|
||||
try:
|
||||
r = requests.head(current, timeout=HTTP_TIMEOUT, allow_redirects=False)
|
||||
if r.status_code == 405:
|
||||
# Some redirect proxies only register GET/POST, not HEAD.
|
||||
r = requests.get(
|
||||
current, timeout=HTTP_TIMEOUT, allow_redirects=False, stream=True
|
||||
)
|
||||
r.close()
|
||||
if r.status_code not in (301, 302, 303, 307, 308):
|
||||
break
|
||||
loc = r.headers.get("Location", "")
|
||||
if loc and "127.0.0.1" not in loc and "localhost" not in loc:
|
||||
return loc
|
||||
except Exception:
|
||||
pass
|
||||
return url
|
||||
if not loc or loc == current:
|
||||
break
|
||||
current = loc
|
||||
except Exception:
|
||||
break
|
||||
return current
|
||||
|
||||
|
||||
def titles_match(a: str, b: str) -> bool:
|
||||
@@ -290,7 +325,8 @@ def check_video(video: dict, channel_id: str, results: Results, run_ytdlp: bool)
|
||||
continue
|
||||
ok, code = http_ok(furl, headers=fheaders)
|
||||
if not ok:
|
||||
if _is_cf_protected(furl):
|
||||
resolved = follow_proxy_redirect(furl)
|
||||
if _is_cf_protected(furl) or _is_cf_protected(resolved):
|
||||
results.warn(
|
||||
channel_id,
|
||||
f"{label} format[{j}]: unreachable HTTP={code} (CF-protected host, expected)"
|
||||
@@ -384,11 +420,19 @@ def check_video(video: dict, channel_id: str, results: Results, run_ytdlp: bool)
|
||||
results.info(channel_id, f"{label} format[{j}]: yt-dlp extract {furl}")
|
||||
yt, stderr = ytdlp_extract(furl, extra_args=extra_args)
|
||||
if yt is None:
|
||||
results.err(
|
||||
channel_id,
|
||||
f"{label} format[{j}]: yt-dlp failed for {furl}"
|
||||
+ (f": {stderr[:200]}" if stderr else ""),
|
||||
)
|
||||
resolved = follow_proxy_redirect(furl)
|
||||
if _is_cf_protected(furl) or _is_cf_protected(resolved):
|
||||
results.warn(
|
||||
channel_id,
|
||||
f"{label} format[{j}]: yt-dlp failed for {furl} (CF-protected host, expected)"
|
||||
+ (f": {stderr[:200]}" if stderr else ""),
|
||||
)
|
||||
else:
|
||||
results.err(
|
||||
channel_id,
|
||||
f"{label} format[{j}]: yt-dlp failed for {furl}"
|
||||
+ (f": {stderr[:200]}" if stderr else ""),
|
||||
)
|
||||
else:
|
||||
yt_fmts = yt.get("formats") or []
|
||||
yt_direct = yt.get("url")
|
||||
|
||||
Reference in New Issue
Block a user