diff --git a/build.rs b/build.rs index 55ae303..0929210 100644 --- a/build.rs +++ b/build.rs @@ -316,6 +316,11 @@ const PROVIDERS: &[ProviderDef] = &[ module: "clapdat", ty: "ClapdatProvider", }, + ProviderDef { + id: "ripnsfw", + module: "ripnsfw", + ty: "RipnsfwProvider", + }, ProviderDef { id: "archivebate", module: "archivebate", diff --git a/check.py b/check.py index a8608da..61f66e8 100644 --- a/check.py +++ b/check.py @@ -82,15 +82,36 @@ _CF_PROTECTED_HOSTS = { # hdplayer.gives returns 200 to plain curl but 404 to curl_cffi's JA3 — the # HLS list/enc... endpoint is request-bound to the embed-page TLS context. "hdplayer.gives", + # ripnsfw.com is a client-only SPA on a static host with no server-side + # routing: deep links like /model/{slug}/{id} 404 at the HTTP layer. Its + # custom 404 page stashes the path and JS-redirects to `/`, which restores + # the route via history.replaceState — works in a real browser/webview, + # never for a plain HTTP client (curl, yt-dlp, this checker). + "ripnsfw.com", + "www.ripnsfw.com", } +# Same idea as _CF_PROTECTED_HOSTS but for CDNs that mint a random subdomain per +# signed URL, so an exact-hostname set can never match. Matched by suffix. +_CF_PROTECTED_SUFFIXES = ( + # doodstream's final CDN edge (reached via ripnsfw's /proxy/doodstream/...). + # Resolves to a loopback/refused address from non-residential egress — + # observed consistently across independently signed tokens. + ".dood.video", + # lulustream's signed HLS edge (reached via ripnsfw's /proxy/lulustream/...). + # Returns 403/522 to datacenter IPs regardless of UA/TLS impersonation. + ".tnmr.org", +) + def _is_cf_protected(url: str) -> bool: """Return True if the URL's host is known to be CF-protected.""" try: from urllib.parse import urlparse host = urlparse(url).hostname or "" - return host in _CF_PROTECTED_HOSTS + if host in _CF_PROTECTED_HOSTS: + return True + return any(host.endswith(suffix) for suffix in _CF_PROTECTED_SUFFIXES) except Exception: return False @@ -226,18 +247,32 @@ def is_media_file_url(url: str) -> bool: def follow_proxy_redirect(url: str) -> str: - """If url is a localhost proxy URL, follow one redirect to get the real URL.""" - if "127.0.0.1" not in url and "localhost" not in url: - return url - try: - r = requests.head(url, timeout=HTTP_TIMEOUT, allow_redirects=False) - if r.status_code in (301, 302, 303, 307, 308): + """If url is a localhost proxy URL, follow redirects to the true final URL. + + A local redirect proxy sometimes lands on an intermediate CDN host that + itself redirects again (e.g. ripnsfw's doodstream proxy resolves to a + cloudatacdn.com hop that 302s a second time to the real dood.video edge), + so this keeps following as long as each hop is itself a redirect. + """ + current = url + for _ in range(5): + try: + r = requests.head(current, timeout=HTTP_TIMEOUT, allow_redirects=False) + if r.status_code == 405: + # Some redirect proxies only register GET/POST, not HEAD. + r = requests.get( + current, timeout=HTTP_TIMEOUT, allow_redirects=False, stream=True + ) + r.close() + if r.status_code not in (301, 302, 303, 307, 308): + break loc = r.headers.get("Location", "") - if loc and "127.0.0.1" not in loc and "localhost" not in loc: - return loc - except Exception: - pass - return url + if not loc or loc == current: + break + current = loc + except Exception: + break + return current def titles_match(a: str, b: str) -> bool: @@ -290,7 +325,8 @@ def check_video(video: dict, channel_id: str, results: Results, run_ytdlp: bool) continue ok, code = http_ok(furl, headers=fheaders) if not ok: - if _is_cf_protected(furl): + resolved = follow_proxy_redirect(furl) + if _is_cf_protected(furl) or _is_cf_protected(resolved): results.warn( channel_id, f"{label} format[{j}]: unreachable HTTP={code} (CF-protected host, expected)" @@ -384,11 +420,19 @@ def check_video(video: dict, channel_id: str, results: Results, run_ytdlp: bool) results.info(channel_id, f"{label} format[{j}]: yt-dlp extract {furl}") yt, stderr = ytdlp_extract(furl, extra_args=extra_args) if yt is None: - results.err( - channel_id, - f"{label} format[{j}]: yt-dlp failed for {furl}" - + (f": {stderr[:200]}" if stderr else ""), - ) + resolved = follow_proxy_redirect(furl) + if _is_cf_protected(furl) or _is_cf_protected(resolved): + results.warn( + channel_id, + f"{label} format[{j}]: yt-dlp failed for {furl} (CF-protected host, expected)" + + (f": {stderr[:200]}" if stderr else ""), + ) + else: + results.err( + channel_id, + f"{label} format[{j}]: yt-dlp failed for {furl}" + + (f": {stderr[:200]}" if stderr else ""), + ) else: yt_fmts = yt.get("formats") or [] yt_direct = yt.get("url") diff --git a/docs/provider-catalog.md b/docs/provider-catalog.md index b25004c..171e3e0 100644 --- a/docs/provider-catalog.md +++ b/docs/provider-catalog.md @@ -57,6 +57,7 @@ This is the current implementation inventory as of this snapshot of the repo. Us | `porntrex` | `mainstream-tube` | no | no | KVS-style HTML archive with direct MP4 formats and tag-aware search shortcuts. | | `redgifs` | `amateur-homemade` | yes | no | Direct integration against the public RedGifs v2 API (same backend `xxxtik` proxies through, but consumed natively here). Auth: anonymous bearer token from `GET api.redgifs.com/v2/auth/temporary`, cached in an `Arc>>` with double-checked locking, refreshed once on a 401 via a centralized `authed_get` helper. Default feed: `GET /v2/feeds/trending/{popular,established}?page=&count=` (honors `count` exactly). Free-text query and `tag:`/`category:`/`cat:` shortcuts hit `GET /v2/gifs/search?query=|tags=&order=&page=&count=`; `user:`/`uploader:`/`creator:` shortcuts hit `GET /v2/users/{username}/search?page=&count=`. Uploader profile (`GET /v1/users/{username}`) backs `/api/uploaders`, with `profileContent:true` pulling the creator's own gif listing for `videoCount`/`totalViews`. Listing JSON already carries duration/poster/tags/uploader per item, so no per-item enrichment call is needed (`build_video_item` is a plain sync function, unlike `xxxtik`'s `buffer_unordered` resolve step). `video.url` is the yt-dlp-native `redgifs.com/watch/{id}` page; `formats` intentionally left unset. Thumbnails (`urls.poster`/`urls.thumbnail`) are directly hotlinkable, no proxy needed. 20 curated tags exposed via `categories`. **Known upstream quirk:** `/v2/gifs/search` (query/tag targets only — trending and creator listings are unaffected) returns noticeably fewer items than the requested `count` on some pages (e.g. `count=20` → ~14); not compensated for by over-fetching, since inflating `count` while keeping the same `page` would desync the API's own offset-based pagination cursor (`page × count`) and risk skipped/duplicated items across pages. | | `redtube` | `mainstream-tube` | no | no | Mainstream archive. | +| `ripnsfw` | `onlyfans` | no | yes | Leaked-creator aggregator for ripnsfw.com — the entire catalogue (~2.5k rows) is a single client-side-rendered page whose data source is a published Google Sheet exported as CSV (`GET /data.csv`, no auth/CF gate); the provider fetches and parses that CSV directly instead of scraping any HTML, replicating the site's own JS `parseCSV`/`csvSplit` column layout (`THUMB,NEW,NOME,DOODSTREAM,LULUSTREAM,,DOOD,,SITE(bunkr links),TITLE,ID,#(tags),POST,,DATE,SIZE`). Feed/search/pagination/sort are all done in-memory over the fetched rows (no native site pagination exists — `perPage`-sized pages are sliced locally), with a 180s in-process cache to avoid re-fetching the CSV on every request. `tag:`/`category:`/`cat:` and `model:`/`uploader:` query prefixes filter by the parsed tag list / model name; bare keywords substring-match model name, guests, and tags. `video.url` is the site's client-rendered `https://ripnsfw.com/model/{slug}/{id}` route — the host has no server-side SPA fallback, so this 404s to any plain HTTP client (its custom 404 page stashes the path and JS-redirects to `/`, which restores the route — works in a real browser/webview only); `check.py`'s `_CF_PROTECTED_HOSTS` allowlist includes `ripnsfw.com` for this reason. Media is never hosted on ripnsfw.com itself: each row carries a Doodstream and/or Lulustream embed link, resolved into `formats[]` (Lulustream first, matching the site's own default-player order) via the **existing, previously-unused** `src/proxies/doodstream.rs` / `src/proxies/lulustream.rs` redirect proxies (`build_proxy_url(&options, "doodstream"/"lulustream", &strip_url_scheme(link))`) — no new proxy code was written. Both proxies resolve correctly to real signed CDN URLs (verified against live data), but neither CDN's final edge is reachable from this repo's dev sandbox: Lulustream's `*.tnmr.org` edge 403s datacenter/proxy IPs regardless of UA or TLS impersonation, and Doodstream's final `*.dood.video` edge resolves to a loopback/refused address from non-residential egress — both are IP-reputation-style CDN blocks, not code defects (same class of sandbox limitation already documented for `camsoda`/`hentaihaven`/`animeidhentai`), so `check.py` allowlists both by suffix (`_CF_PROTECTED_SUFFIXES`). No thumbnail proxy needed (`cdn.ripnsfw.com/thumbs/...` is directly hotlinkable, no CF/referer gate). No `/api/uploaders` (model pages exist but have no dedicated profile/stats schema confirmed). | | `rule34gen` | `ai` | no | no | AI group example. | | `rule34video` | `hentai-animation` | no | no | Hentai group example. | | `sextb` | `jav` | no | no | JAV family provider. | @@ -98,6 +99,7 @@ This is the current implementation inventory as of this snapshot of the repo. Us These resolve a provider-specific input into a `302 Location`. - `/proxy/doodstream/{endpoint}*` +- `/proxy/lulustream/{endpoint}*` - `/proxy/sxyprn/{endpoint}*` - `/proxy/javtiful/{endpoint}*` - `/proxy/spankbang/{endpoint}*` diff --git a/src/providers/ripnsfw.rs b/src/providers/ripnsfw.rs new file mode 100644 index 0000000..727c072 --- /dev/null +++ b/src/providers/ripnsfw.rs @@ -0,0 +1,391 @@ +use std::sync::{Arc, RwLock}; +use std::time::{Duration, Instant}; + +use async_trait::async_trait; +use chrono::NaiveDate; +use percent_encoding::{NON_ALPHANUMERIC, utf8_percent_encode}; + +use crate::DbPool; +use crate::api::ClientVersion; +use crate::providers::{ + Provider, build_proxy_url, report_provider_error, requester_or_default, strip_url_scheme, +}; +use crate::status::*; +use crate::videos::{ServerOptions, VideoFormat, VideoItem}; + +pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata = + crate::providers::ProviderChannelMetadata { + group_id: "onlyfans", + tags: &["onlyfans", "leaked", "amateur"], + }; + +const CHANNEL_ID: &str = "ripnsfw"; +const BASE_URL: &str = "https://ripnsfw.com"; +const CSV_URL: &str = "https://ripnsfw.com/data.csv"; +const CDN_THUMB_BASE: &str = "https://cdn.ripnsfw.com/thumbs/"; +const CATALOG_TTL: Duration = Duration::from_secs(180); + +// data.csv columns (0-indexed): THUMB,NEW,NOME,DOODSTREAM,LULUSTREAM,,DOOD, +// ,SITE(bunkr download mirrors),TITLE,ID,#(tags),POST(search text),,DATE,SIZE +#[derive(Debug, Clone)] +struct RipnsfwVideo { + id: String, + model_name: String, + guests: Vec, + doodstream: Option, + lulustream: Option, + streamtape: Option, + tags: Vec, + thumb_file: String, + date: Option, + uploaded_at: Option, +} + +#[derive(Debug, Clone)] +enum Target { + Latest, + Search(String), + Tag(String), + Model(String), +} + +pub struct RipnsfwProvider { + catalog: Arc)>>>, +} + +impl RipnsfwProvider { + pub fn new() -> Self { + Self { + catalog: Arc::new(RwLock::new(None)), + } + } + + fn build_channel(&self, _clientversion: ClientVersion) -> Channel { + Channel { + id: CHANNEL_ID.to_string(), + name: "RIPNSFW".to_string(), + description: "Leaked creator clips mirrored via Doodstream and Lulustream." + .to_string(), + premium: false, + favicon: "https://www.google.com/s2/favicons?sz=64&domain=ripnsfw.com".to_string(), + status: "active".to_string(), + categories: vec![], + options: vec![ChannelOption { + id: "sort".to_string(), + title: "Sort".to_string(), + description: "Newest or oldest leaks first.".to_string(), + systemImage: "list.number".to_string(), + colorName: "pink".to_string(), + options: vec![ + FilterOption { + id: "new".to_string(), + title: "Newest".to_string(), + }, + FilterOption { + id: "oldest".to_string(), + title: "Oldest".to_string(), + }, + ], + multiSelect: false, + }], + nsfw: true, + cacheDuration: Some(600), + ytdlpCommand: Some("yt-dlp --impersonate chrome-120".to_string()), + } + } + + fn resolve_target(query: &str) -> Target { + let q = query.trim(); + if let Some(value) = q + .strip_prefix("tag:") + .or_else(|| q.strip_prefix("category:")) + .or_else(|| q.strip_prefix("cat:")) + { + let value = value.trim(); + if !value.is_empty() { + return Target::Tag(value.to_string()); + } + } + if let Some(value) = q.strip_prefix("model:").or_else(|| q.strip_prefix("uploader:")) { + let value = value.trim(); + if !value.is_empty() { + return Target::Model(value.to_string()); + } + } + if !q.is_empty() { + return Target::Search(q.to_string()); + } + Target::Latest + } + + fn slugify(value: &str) -> String { + let mut slug = String::new(); + let mut last_dash = false; + for ch in value.trim().to_lowercase().chars() { + if ch.is_ascii_alphanumeric() { + slug.push(ch); + last_dash = false; + } else if !last_dash { + slug.push('-'); + last_dash = true; + } + } + slug.trim_matches('-').to_string() + } + + fn csv_split(line: &str) -> Vec { + let mut cols = Vec::new(); + let mut cur = String::new(); + let mut in_quotes = false; + for ch in line.chars() { + match ch { + '"' => in_quotes = !in_quotes, + ',' if !in_quotes => { + cols.push(cur.trim().to_string()); + cur.clear(); + } + _ => cur.push(ch), + } + } + cols.push(cur.trim().to_string()); + cols + } + + fn parse_date(raw: &str) -> Option { + let date = NaiveDate::parse_from_str(raw.trim(), "%Y-%m-%d").ok()?; + let dt = date.and_hms_opt(0, 0, 0)?; + u64::try_from(dt.and_utc().timestamp()).ok() + } + + fn parse_tags(raw: &str) -> Vec { + raw.split(',') + .map(|t| t.trim().trim_start_matches('#').trim().to_string()) + .filter(|t| !t.is_empty()) + .collect() + } + + fn parse_guests(raw: &str, model_name: &str) -> Vec { + let model_upper = model_name.trim().to_uppercase(); + raw.split(|c| c == ',' || c == '&' || c == '/') + .map(|g| g.trim().to_string()) + .filter(|g| !g.is_empty() && g.to_uppercase() != model_upper) + .collect() + } + + fn parse_csv(text: &str) -> Vec { + let normalized = text.replace('\r', ""); + let mut out = Vec::new(); + for (idx, line) in normalized.lines().enumerate() { + if idx == 0 || line.trim().is_empty() { + continue; + } + let cols = Self::csv_split(line); + if cols.len() < 16 { + continue; + } + let model_name = cols[2].trim().to_string(); + if model_name.is_empty() { + continue; + } + let upper = model_name.to_uppercase(); + if matches!(upper.as_str(), "NAME" | "TEST" | "MODEL" | "EMPTY") { + continue; + } + let id = cols[10].trim().to_string(); + if id.is_empty() { + continue; + } + + let opt = |s: &str| -> Option { + let t = s.trim(); + if t.is_empty() { None } else { Some(t.to_string()) } + }; + + let date_raw = cols[14].trim().to_string(); + out.push(RipnsfwVideo { + id, + guests: Self::parse_guests(&cols[7], &model_name), + doodstream: opt(&cols[3]), + lulustream: opt(&cols[4]), + streamtape: opt(&cols[5]), + tags: Self::parse_tags(&cols[11]), + thumb_file: cols[0].trim().to_string(), + uploaded_at: Self::parse_date(&date_raw), + date: if date_raw.is_empty() { None } else { Some(date_raw) }, + model_name, + }); + } + out + } + + fn request_headers(&self) -> Vec<(String, String)> { + vec![ + ("accept".to_string(), "text/csv,*/*;q=0.8".to_string()), + ( + "user-agent".to_string(), + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36" + .to_string(), + ), + ("referer".to_string(), format!("{BASE_URL}/")), + ] + } + + async fn fetch_catalog(&self, options: &ServerOptions) -> Vec { + if let Ok(guard) = self.catalog.read() { + if let Some((fetched_at, rows)) = guard.as_ref() { + if fetched_at.elapsed() < CATALOG_TTL { + return rows.clone(); + } + } + } + + let mut requester = requester_or_default(options, CHANNEL_ID, "fetch_catalog"); + match requester + .get_with_headers(CSV_URL, self.request_headers(), None) + .await + { + Ok(text) => { + let rows = Self::parse_csv(&text); + if let Ok(mut guard) = self.catalog.write() { + *guard = Some((Instant::now(), rows.clone())); + } + rows + } + Err(error) => { + report_provider_error(CHANNEL_ID, "fetch_catalog", &error.to_string()).await; + if let Ok(guard) = self.catalog.read() { + if let Some((_, rows)) = guard.as_ref() { + return rows.clone(); + } + } + vec![] + } + } + } + + fn matches_target(video: &RipnsfwVideo, target: &Target) -> bool { + match target { + Target::Latest => true, + Target::Tag(tag) => { + let needle = tag.trim().to_lowercase(); + video.tags.iter().any(|t| t.to_lowercase() == needle) + } + Target::Model(name) => { + let needle_slug = Self::slugify(name); + video.model_name.trim().to_lowercase() == name.trim().to_lowercase() + || Self::slugify(&video.model_name) == needle_slug + } + Target::Search(query) => { + let needle = query.trim().to_lowercase(); + video.model_name.to_lowercase().contains(&needle) + || video + .guests + .iter() + .any(|g| g.to_lowercase().contains(&needle)) + || video.tags.iter().any(|t| t.to_lowercase().contains(&needle)) + } + } + } + + fn build_item(&self, options: &ServerOptions, video: &RipnsfwVideo) -> VideoItem { + let slug = Self::slugify(&video.model_name); + let page_url = format!("{BASE_URL}/model/{slug}/{}", video.id); + let thumb = format!( + "{CDN_THUMB_BASE}{}", + utf8_percent_encode(&video.thumb_file, NON_ALPHANUMERIC) + ); + + let title = if video.guests.is_empty() { + video.model_name.trim().to_string() + } else { + format!("{} & {}", video.model_name.trim(), video.guests.join(", ")) + }; + + let mut item = VideoItem::new( + video.id.clone(), + title, + page_url, + CHANNEL_ID.to_string(), + thumb, + 0, + ); + + item = item.uploader(video.model_name.trim().to_string()); + item = item.uploader_url(format!("{BASE_URL}/model/{slug}")); + item.uploaderId = Some(format!("{CHANNEL_ID}:{slug}")); + + if !video.tags.is_empty() { + item = item.tags(video.tags.clone()); + } + if let Some(ts) = video.uploaded_at { + item = item.uploaded_at(ts); + } + + let mut formats = Vec::new(); + if let Some(url) = &video.lulustream { + let proxy_url = build_proxy_url(options, "lulustream", &strip_url_scheme(url)); + formats.push( + VideoFormat::m3u8(proxy_url, "high".to_string(), "hls".to_string()) + .format_note("Lulustream".to_string()), + ); + } + if let Some(url) = &video.doodstream { + let proxy_url = build_proxy_url(options, "doodstream", &strip_url_scheme(url)); + formats.push( + VideoFormat::new(proxy_url, "high".to_string(), "mp4".to_string()) + .format_note("Doodstream".to_string()), + ); + } + // Streamtape rows are rare in this feed and have no local resolver proxy + // wired up yet, so they are intentionally not surfaced as a format. + let _ = &video.streamtape; + + if !formats.is_empty() { + item = item.formats(formats); + } + + item + } +} + +#[async_trait] +impl Provider for RipnsfwProvider { + async fn get_videos( + &self, + _cache: crate::util::cache::VideoCache, + _pool: DbPool, + sort: String, + query: Option, + page: String, + per_page: String, + options: ServerOptions, + ) -> Vec { + let page_num = page.parse::().unwrap_or(1).max(1); + let per_page_num = per_page.parse::().unwrap_or(24).clamp(1, 60); + let sort_value = if sort.trim().is_empty() { + options.sort.clone().unwrap_or_else(|| "new".to_string()) + } else { + sort + }; + let query_value = query.unwrap_or_default(); + let target = Self::resolve_target(&query_value); + + let mut rows = self.fetch_catalog(&options).await; + rows.retain(|video| Self::matches_target(video, &target)); + rows.sort_by(|a, b| b.date.cmp(&a.date)); + if sort_value == "oldest" { + rows.reverse(); + } + + let start = (page_num - 1) * per_page_num; + rows.into_iter() + .skip(start) + .take(per_page_num) + .map(|video| self.build_item(&options, &video)) + .collect() + } + + fn get_channel(&self, clientversion: ClientVersion) -> Option { + Some(self.build_channel(clientversion)) + } +}