Add ripnsfw provider (Doodstream/Lulustream leak aggregator)
ripnsfw.com serves its entire catalogue as a published-Google-Sheet CSV with no native pagination/search API, so the provider fetches and parses that CSV once (cached 180s) and does feed/search/pagination/sort in memory. Each row's Doodstream/Lulustream embed links resolve to formats[] via the existing (previously unused) doodstream/lulustream redirect proxies. Also fixes check.py's follow_proxy_redirect, which used HEAD even though these redirect-proxy routes only accept GET/POST, so it never actually resolved the redirect; extends the CF-protected host list (suffix matching + ripnsfw.com's client-only-SPA 404 page, dood.video, tnmr.org) so known sandbox/CDN-IP-reputation failures are reported as warnings instead of errors. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01QTqf6orbHZ9rFFpVpcgzcR
This commit is contained in:
5
build.rs
5
build.rs
@@ -316,6 +316,11 @@ const PROVIDERS: &[ProviderDef] = &[
|
|||||||
module: "clapdat",
|
module: "clapdat",
|
||||||
ty: "ClapdatProvider",
|
ty: "ClapdatProvider",
|
||||||
},
|
},
|
||||||
|
ProviderDef {
|
||||||
|
id: "ripnsfw",
|
||||||
|
module: "ripnsfw",
|
||||||
|
ty: "RipnsfwProvider",
|
||||||
|
},
|
||||||
ProviderDef {
|
ProviderDef {
|
||||||
id: "archivebate",
|
id: "archivebate",
|
||||||
module: "archivebate",
|
module: "archivebate",
|
||||||
|
|||||||
80
check.py
80
check.py
@@ -82,15 +82,36 @@ _CF_PROTECTED_HOSTS = {
|
|||||||
# hdplayer.gives returns 200 to plain curl but 404 to curl_cffi's JA3 — the
|
# hdplayer.gives returns 200 to plain curl but 404 to curl_cffi's JA3 — the
|
||||||
# HLS list/enc... endpoint is request-bound to the embed-page TLS context.
|
# HLS list/enc... endpoint is request-bound to the embed-page TLS context.
|
||||||
"hdplayer.gives",
|
"hdplayer.gives",
|
||||||
|
# ripnsfw.com is a client-only SPA on a static host with no server-side
|
||||||
|
# routing: deep links like /model/{slug}/{id} 404 at the HTTP layer. Its
|
||||||
|
# custom 404 page stashes the path and JS-redirects to `/`, which restores
|
||||||
|
# the route via history.replaceState — works in a real browser/webview,
|
||||||
|
# never for a plain HTTP client (curl, yt-dlp, this checker).
|
||||||
|
"ripnsfw.com",
|
||||||
|
"www.ripnsfw.com",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Same idea as _CF_PROTECTED_HOSTS but for CDNs that mint a random subdomain per
|
||||||
|
# signed URL, so an exact-hostname set can never match. Matched by suffix.
|
||||||
|
_CF_PROTECTED_SUFFIXES = (
|
||||||
|
# doodstream's final CDN edge (reached via ripnsfw's /proxy/doodstream/...).
|
||||||
|
# Resolves to a loopback/refused address from non-residential egress —
|
||||||
|
# observed consistently across independently signed tokens.
|
||||||
|
".dood.video",
|
||||||
|
# lulustream's signed HLS edge (reached via ripnsfw's /proxy/lulustream/...).
|
||||||
|
# Returns 403/522 to datacenter IPs regardless of UA/TLS impersonation.
|
||||||
|
".tnmr.org",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _is_cf_protected(url: str) -> bool:
|
def _is_cf_protected(url: str) -> bool:
|
||||||
"""Return True if the URL's host is known to be CF-protected."""
|
"""Return True if the URL's host is known to be CF-protected."""
|
||||||
try:
|
try:
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
host = urlparse(url).hostname or ""
|
host = urlparse(url).hostname or ""
|
||||||
return host in _CF_PROTECTED_HOSTS
|
if host in _CF_PROTECTED_HOSTS:
|
||||||
|
return True
|
||||||
|
return any(host.endswith(suffix) for suffix in _CF_PROTECTED_SUFFIXES)
|
||||||
except Exception:
|
except Exception:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
@@ -226,18 +247,32 @@ def is_media_file_url(url: str) -> bool:
|
|||||||
|
|
||||||
|
|
||||||
def follow_proxy_redirect(url: str) -> str:
|
def follow_proxy_redirect(url: str) -> str:
|
||||||
"""If url is a localhost proxy URL, follow one redirect to get the real URL."""
|
"""If url is a localhost proxy URL, follow redirects to the true final URL.
|
||||||
if "127.0.0.1" not in url and "localhost" not in url:
|
|
||||||
return url
|
A local redirect proxy sometimes lands on an intermediate CDN host that
|
||||||
try:
|
itself redirects again (e.g. ripnsfw's doodstream proxy resolves to a
|
||||||
r = requests.head(url, timeout=HTTP_TIMEOUT, allow_redirects=False)
|
cloudatacdn.com hop that 302s a second time to the real dood.video edge),
|
||||||
if r.status_code in (301, 302, 303, 307, 308):
|
so this keeps following as long as each hop is itself a redirect.
|
||||||
|
"""
|
||||||
|
current = url
|
||||||
|
for _ in range(5):
|
||||||
|
try:
|
||||||
|
r = requests.head(current, timeout=HTTP_TIMEOUT, allow_redirects=False)
|
||||||
|
if r.status_code == 405:
|
||||||
|
# Some redirect proxies only register GET/POST, not HEAD.
|
||||||
|
r = requests.get(
|
||||||
|
current, timeout=HTTP_TIMEOUT, allow_redirects=False, stream=True
|
||||||
|
)
|
||||||
|
r.close()
|
||||||
|
if r.status_code not in (301, 302, 303, 307, 308):
|
||||||
|
break
|
||||||
loc = r.headers.get("Location", "")
|
loc = r.headers.get("Location", "")
|
||||||
if loc and "127.0.0.1" not in loc and "localhost" not in loc:
|
if not loc or loc == current:
|
||||||
return loc
|
break
|
||||||
except Exception:
|
current = loc
|
||||||
pass
|
except Exception:
|
||||||
return url
|
break
|
||||||
|
return current
|
||||||
|
|
||||||
|
|
||||||
def titles_match(a: str, b: str) -> bool:
|
def titles_match(a: str, b: str) -> bool:
|
||||||
@@ -290,7 +325,8 @@ def check_video(video: dict, channel_id: str, results: Results, run_ytdlp: bool)
|
|||||||
continue
|
continue
|
||||||
ok, code = http_ok(furl, headers=fheaders)
|
ok, code = http_ok(furl, headers=fheaders)
|
||||||
if not ok:
|
if not ok:
|
||||||
if _is_cf_protected(furl):
|
resolved = follow_proxy_redirect(furl)
|
||||||
|
if _is_cf_protected(furl) or _is_cf_protected(resolved):
|
||||||
results.warn(
|
results.warn(
|
||||||
channel_id,
|
channel_id,
|
||||||
f"{label} format[{j}]: unreachable HTTP={code} (CF-protected host, expected)"
|
f"{label} format[{j}]: unreachable HTTP={code} (CF-protected host, expected)"
|
||||||
@@ -384,11 +420,19 @@ def check_video(video: dict, channel_id: str, results: Results, run_ytdlp: bool)
|
|||||||
results.info(channel_id, f"{label} format[{j}]: yt-dlp extract {furl}")
|
results.info(channel_id, f"{label} format[{j}]: yt-dlp extract {furl}")
|
||||||
yt, stderr = ytdlp_extract(furl, extra_args=extra_args)
|
yt, stderr = ytdlp_extract(furl, extra_args=extra_args)
|
||||||
if yt is None:
|
if yt is None:
|
||||||
results.err(
|
resolved = follow_proxy_redirect(furl)
|
||||||
channel_id,
|
if _is_cf_protected(furl) or _is_cf_protected(resolved):
|
||||||
f"{label} format[{j}]: yt-dlp failed for {furl}"
|
results.warn(
|
||||||
+ (f": {stderr[:200]}" if stderr else ""),
|
channel_id,
|
||||||
)
|
f"{label} format[{j}]: yt-dlp failed for {furl} (CF-protected host, expected)"
|
||||||
|
+ (f": {stderr[:200]}" if stderr else ""),
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
results.err(
|
||||||
|
channel_id,
|
||||||
|
f"{label} format[{j}]: yt-dlp failed for {furl}"
|
||||||
|
+ (f": {stderr[:200]}" if stderr else ""),
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
yt_fmts = yt.get("formats") or []
|
yt_fmts = yt.get("formats") or []
|
||||||
yt_direct = yt.get("url")
|
yt_direct = yt.get("url")
|
||||||
|
|||||||
@@ -57,6 +57,7 @@ This is the current implementation inventory as of this snapshot of the repo. Us
|
|||||||
| `porntrex` | `mainstream-tube` | no | no | KVS-style HTML archive with direct MP4 formats and tag-aware search shortcuts. |
|
| `porntrex` | `mainstream-tube` | no | no | KVS-style HTML archive with direct MP4 formats and tag-aware search shortcuts. |
|
||||||
| `redgifs` | `amateur-homemade` | yes | no | Direct integration against the public RedGifs v2 API (same backend `xxxtik` proxies through, but consumed natively here). Auth: anonymous bearer token from `GET api.redgifs.com/v2/auth/temporary`, cached in an `Arc<RwLock<Option<String>>>` with double-checked locking, refreshed once on a 401 via a centralized `authed_get` helper. Default feed: `GET /v2/feeds/trending/{popular,established}?page=&count=` (honors `count` exactly). Free-text query and `tag:`/`category:`/`cat:` shortcuts hit `GET /v2/gifs/search?query=|tags=&order=&page=&count=`; `user:`/`uploader:`/`creator:` shortcuts hit `GET /v2/users/{username}/search?page=&count=`. Uploader profile (`GET /v1/users/{username}`) backs `/api/uploaders`, with `profileContent:true` pulling the creator's own gif listing for `videoCount`/`totalViews`. Listing JSON already carries duration/poster/tags/uploader per item, so no per-item enrichment call is needed (`build_video_item` is a plain sync function, unlike `xxxtik`'s `buffer_unordered` resolve step). `video.url` is the yt-dlp-native `redgifs.com/watch/{id}` page; `formats` intentionally left unset. Thumbnails (`urls.poster`/`urls.thumbnail`) are directly hotlinkable, no proxy needed. 20 curated tags exposed via `categories`. **Known upstream quirk:** `/v2/gifs/search` (query/tag targets only — trending and creator listings are unaffected) returns noticeably fewer items than the requested `count` on some pages (e.g. `count=20` → ~14); not compensated for by over-fetching, since inflating `count` while keeping the same `page` would desync the API's own offset-based pagination cursor (`page × count`) and risk skipped/duplicated items across pages. |
|
| `redgifs` | `amateur-homemade` | yes | no | Direct integration against the public RedGifs v2 API (same backend `xxxtik` proxies through, but consumed natively here). Auth: anonymous bearer token from `GET api.redgifs.com/v2/auth/temporary`, cached in an `Arc<RwLock<Option<String>>>` with double-checked locking, refreshed once on a 401 via a centralized `authed_get` helper. Default feed: `GET /v2/feeds/trending/{popular,established}?page=&count=` (honors `count` exactly). Free-text query and `tag:`/`category:`/`cat:` shortcuts hit `GET /v2/gifs/search?query=|tags=&order=&page=&count=`; `user:`/`uploader:`/`creator:` shortcuts hit `GET /v2/users/{username}/search?page=&count=`. Uploader profile (`GET /v1/users/{username}`) backs `/api/uploaders`, with `profileContent:true` pulling the creator's own gif listing for `videoCount`/`totalViews`. Listing JSON already carries duration/poster/tags/uploader per item, so no per-item enrichment call is needed (`build_video_item` is a plain sync function, unlike `xxxtik`'s `buffer_unordered` resolve step). `video.url` is the yt-dlp-native `redgifs.com/watch/{id}` page; `formats` intentionally left unset. Thumbnails (`urls.poster`/`urls.thumbnail`) are directly hotlinkable, no proxy needed. 20 curated tags exposed via `categories`. **Known upstream quirk:** `/v2/gifs/search` (query/tag targets only — trending and creator listings are unaffected) returns noticeably fewer items than the requested `count` on some pages (e.g. `count=20` → ~14); not compensated for by over-fetching, since inflating `count` while keeping the same `page` would desync the API's own offset-based pagination cursor (`page × count`) and risk skipped/duplicated items across pages. |
|
||||||
| `redtube` | `mainstream-tube` | no | no | Mainstream archive. |
|
| `redtube` | `mainstream-tube` | no | no | Mainstream archive. |
|
||||||
|
| `ripnsfw` | `onlyfans` | no | yes | Leaked-creator aggregator for ripnsfw.com — the entire catalogue (~2.5k rows) is a single client-side-rendered page whose data source is a published Google Sheet exported as CSV (`GET /data.csv`, no auth/CF gate); the provider fetches and parses that CSV directly instead of scraping any HTML, replicating the site's own JS `parseCSV`/`csvSplit` column layout (`THUMB,NEW,NOME,DOODSTREAM,LULUSTREAM,<link3>,DOOD,<guests>,SITE(bunkr links),TITLE,ID,#(tags),POST,,DATE,SIZE`). Feed/search/pagination/sort are all done in-memory over the fetched rows (no native site pagination exists — `perPage`-sized pages are sliced locally), with a 180s in-process cache to avoid re-fetching the CSV on every request. `tag:`/`category:`/`cat:` and `model:`/`uploader:` query prefixes filter by the parsed tag list / model name; bare keywords substring-match model name, guests, and tags. `video.url` is the site's client-rendered `https://ripnsfw.com/model/{slug}/{id}` route — the host has no server-side SPA fallback, so this 404s to any plain HTTP client (its custom 404 page stashes the path and JS-redirects to `/`, which restores the route — works in a real browser/webview only); `check.py`'s `_CF_PROTECTED_HOSTS` allowlist includes `ripnsfw.com` for this reason. Media is never hosted on ripnsfw.com itself: each row carries a Doodstream and/or Lulustream embed link, resolved into `formats[]` (Lulustream first, matching the site's own default-player order) via the **existing, previously-unused** `src/proxies/doodstream.rs` / `src/proxies/lulustream.rs` redirect proxies (`build_proxy_url(&options, "doodstream"/"lulustream", &strip_url_scheme(link))`) — no new proxy code was written. Both proxies resolve correctly to real signed CDN URLs (verified against live data), but neither CDN's final edge is reachable from this repo's dev sandbox: Lulustream's `*.tnmr.org` edge 403s datacenter/proxy IPs regardless of UA or TLS impersonation, and Doodstream's final `*.dood.video` edge resolves to a loopback/refused address from non-residential egress — both are IP-reputation-style CDN blocks, not code defects (same class of sandbox limitation already documented for `camsoda`/`hentaihaven`/`animeidhentai`), so `check.py` allowlists both by suffix (`_CF_PROTECTED_SUFFIXES`). No thumbnail proxy needed (`cdn.ripnsfw.com/thumbs/...` is directly hotlinkable, no CF/referer gate). No `/api/uploaders` (model pages exist but have no dedicated profile/stats schema confirmed). |
|
||||||
| `rule34gen` | `ai` | no | no | AI group example. |
|
| `rule34gen` | `ai` | no | no | AI group example. |
|
||||||
| `rule34video` | `hentai-animation` | no | no | Hentai group example. |
|
| `rule34video` | `hentai-animation` | no | no | Hentai group example. |
|
||||||
| `sextb` | `jav` | no | no | JAV family provider. |
|
| `sextb` | `jav` | no | no | JAV family provider. |
|
||||||
@@ -98,6 +99,7 @@ This is the current implementation inventory as of this snapshot of the repo. Us
|
|||||||
These resolve a provider-specific input into a `302 Location`.
|
These resolve a provider-specific input into a `302 Location`.
|
||||||
|
|
||||||
- `/proxy/doodstream/{endpoint}*`
|
- `/proxy/doodstream/{endpoint}*`
|
||||||
|
- `/proxy/lulustream/{endpoint}*`
|
||||||
- `/proxy/sxyprn/{endpoint}*`
|
- `/proxy/sxyprn/{endpoint}*`
|
||||||
- `/proxy/javtiful/{endpoint}*`
|
- `/proxy/javtiful/{endpoint}*`
|
||||||
- `/proxy/spankbang/{endpoint}*`
|
- `/proxy/spankbang/{endpoint}*`
|
||||||
|
|||||||
391
src/providers/ripnsfw.rs
Normal file
391
src/providers/ripnsfw.rs
Normal file
@@ -0,0 +1,391 @@
|
|||||||
|
use std::sync::{Arc, RwLock};
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
|
use async_trait::async_trait;
|
||||||
|
use chrono::NaiveDate;
|
||||||
|
use percent_encoding::{NON_ALPHANUMERIC, utf8_percent_encode};
|
||||||
|
|
||||||
|
use crate::DbPool;
|
||||||
|
use crate::api::ClientVersion;
|
||||||
|
use crate::providers::{
|
||||||
|
Provider, build_proxy_url, report_provider_error, requester_or_default, strip_url_scheme,
|
||||||
|
};
|
||||||
|
use crate::status::*;
|
||||||
|
use crate::videos::{ServerOptions, VideoFormat, VideoItem};
|
||||||
|
|
||||||
|
pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata =
|
||||||
|
crate::providers::ProviderChannelMetadata {
|
||||||
|
group_id: "onlyfans",
|
||||||
|
tags: &["onlyfans", "leaked", "amateur"],
|
||||||
|
};
|
||||||
|
|
||||||
|
const CHANNEL_ID: &str = "ripnsfw";
|
||||||
|
const BASE_URL: &str = "https://ripnsfw.com";
|
||||||
|
const CSV_URL: &str = "https://ripnsfw.com/data.csv";
|
||||||
|
const CDN_THUMB_BASE: &str = "https://cdn.ripnsfw.com/thumbs/";
|
||||||
|
const CATALOG_TTL: Duration = Duration::from_secs(180);
|
||||||
|
|
||||||
|
// data.csv columns (0-indexed): THUMB,NEW,NOME,DOODSTREAM,LULUSTREAM,<link3>,DOOD,
|
||||||
|
// <guests>,SITE(bunkr download mirrors),TITLE,ID,#(tags),POST(search text),<blank>,DATE,SIZE
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
struct RipnsfwVideo {
|
||||||
|
id: String,
|
||||||
|
model_name: String,
|
||||||
|
guests: Vec<String>,
|
||||||
|
doodstream: Option<String>,
|
||||||
|
lulustream: Option<String>,
|
||||||
|
streamtape: Option<String>,
|
||||||
|
tags: Vec<String>,
|
||||||
|
thumb_file: String,
|
||||||
|
date: Option<String>,
|
||||||
|
uploaded_at: Option<u64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
enum Target {
|
||||||
|
Latest,
|
||||||
|
Search(String),
|
||||||
|
Tag(String),
|
||||||
|
Model(String),
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct RipnsfwProvider {
|
||||||
|
catalog: Arc<RwLock<Option<(Instant, Vec<RipnsfwVideo>)>>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RipnsfwProvider {
|
||||||
|
pub fn new() -> Self {
|
||||||
|
Self {
|
||||||
|
catalog: Arc::new(RwLock::new(None)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn build_channel(&self, _clientversion: ClientVersion) -> Channel {
|
||||||
|
Channel {
|
||||||
|
id: CHANNEL_ID.to_string(),
|
||||||
|
name: "RIPNSFW".to_string(),
|
||||||
|
description: "Leaked creator clips mirrored via Doodstream and Lulustream."
|
||||||
|
.to_string(),
|
||||||
|
premium: false,
|
||||||
|
favicon: "https://www.google.com/s2/favicons?sz=64&domain=ripnsfw.com".to_string(),
|
||||||
|
status: "active".to_string(),
|
||||||
|
categories: vec![],
|
||||||
|
options: vec![ChannelOption {
|
||||||
|
id: "sort".to_string(),
|
||||||
|
title: "Sort".to_string(),
|
||||||
|
description: "Newest or oldest leaks first.".to_string(),
|
||||||
|
systemImage: "list.number".to_string(),
|
||||||
|
colorName: "pink".to_string(),
|
||||||
|
options: vec![
|
||||||
|
FilterOption {
|
||||||
|
id: "new".to_string(),
|
||||||
|
title: "Newest".to_string(),
|
||||||
|
},
|
||||||
|
FilterOption {
|
||||||
|
id: "oldest".to_string(),
|
||||||
|
title: "Oldest".to_string(),
|
||||||
|
},
|
||||||
|
],
|
||||||
|
multiSelect: false,
|
||||||
|
}],
|
||||||
|
nsfw: true,
|
||||||
|
cacheDuration: Some(600),
|
||||||
|
ytdlpCommand: Some("yt-dlp --impersonate chrome-120".to_string()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn resolve_target(query: &str) -> Target {
|
||||||
|
let q = query.trim();
|
||||||
|
if let Some(value) = q
|
||||||
|
.strip_prefix("tag:")
|
||||||
|
.or_else(|| q.strip_prefix("category:"))
|
||||||
|
.or_else(|| q.strip_prefix("cat:"))
|
||||||
|
{
|
||||||
|
let value = value.trim();
|
||||||
|
if !value.is_empty() {
|
||||||
|
return Target::Tag(value.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let Some(value) = q.strip_prefix("model:").or_else(|| q.strip_prefix("uploader:")) {
|
||||||
|
let value = value.trim();
|
||||||
|
if !value.is_empty() {
|
||||||
|
return Target::Model(value.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !q.is_empty() {
|
||||||
|
return Target::Search(q.to_string());
|
||||||
|
}
|
||||||
|
Target::Latest
|
||||||
|
}
|
||||||
|
|
||||||
|
fn slugify(value: &str) -> String {
|
||||||
|
let mut slug = String::new();
|
||||||
|
let mut last_dash = false;
|
||||||
|
for ch in value.trim().to_lowercase().chars() {
|
||||||
|
if ch.is_ascii_alphanumeric() {
|
||||||
|
slug.push(ch);
|
||||||
|
last_dash = false;
|
||||||
|
} else if !last_dash {
|
||||||
|
slug.push('-');
|
||||||
|
last_dash = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
slug.trim_matches('-').to_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn csv_split(line: &str) -> Vec<String> {
|
||||||
|
let mut cols = Vec::new();
|
||||||
|
let mut cur = String::new();
|
||||||
|
let mut in_quotes = false;
|
||||||
|
for ch in line.chars() {
|
||||||
|
match ch {
|
||||||
|
'"' => in_quotes = !in_quotes,
|
||||||
|
',' if !in_quotes => {
|
||||||
|
cols.push(cur.trim().to_string());
|
||||||
|
cur.clear();
|
||||||
|
}
|
||||||
|
_ => cur.push(ch),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
cols.push(cur.trim().to_string());
|
||||||
|
cols
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_date(raw: &str) -> Option<u64> {
|
||||||
|
let date = NaiveDate::parse_from_str(raw.trim(), "%Y-%m-%d").ok()?;
|
||||||
|
let dt = date.and_hms_opt(0, 0, 0)?;
|
||||||
|
u64::try_from(dt.and_utc().timestamp()).ok()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_tags(raw: &str) -> Vec<String> {
|
||||||
|
raw.split(',')
|
||||||
|
.map(|t| t.trim().trim_start_matches('#').trim().to_string())
|
||||||
|
.filter(|t| !t.is_empty())
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_guests(raw: &str, model_name: &str) -> Vec<String> {
|
||||||
|
let model_upper = model_name.trim().to_uppercase();
|
||||||
|
raw.split(|c| c == ',' || c == '&' || c == '/')
|
||||||
|
.map(|g| g.trim().to_string())
|
||||||
|
.filter(|g| !g.is_empty() && g.to_uppercase() != model_upper)
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_csv(text: &str) -> Vec<RipnsfwVideo> {
|
||||||
|
let normalized = text.replace('\r', "");
|
||||||
|
let mut out = Vec::new();
|
||||||
|
for (idx, line) in normalized.lines().enumerate() {
|
||||||
|
if idx == 0 || line.trim().is_empty() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let cols = Self::csv_split(line);
|
||||||
|
if cols.len() < 16 {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let model_name = cols[2].trim().to_string();
|
||||||
|
if model_name.is_empty() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let upper = model_name.to_uppercase();
|
||||||
|
if matches!(upper.as_str(), "NAME" | "TEST" | "MODEL" | "EMPTY") {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let id = cols[10].trim().to_string();
|
||||||
|
if id.is_empty() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let opt = |s: &str| -> Option<String> {
|
||||||
|
let t = s.trim();
|
||||||
|
if t.is_empty() { None } else { Some(t.to_string()) }
|
||||||
|
};
|
||||||
|
|
||||||
|
let date_raw = cols[14].trim().to_string();
|
||||||
|
out.push(RipnsfwVideo {
|
||||||
|
id,
|
||||||
|
guests: Self::parse_guests(&cols[7], &model_name),
|
||||||
|
doodstream: opt(&cols[3]),
|
||||||
|
lulustream: opt(&cols[4]),
|
||||||
|
streamtape: opt(&cols[5]),
|
||||||
|
tags: Self::parse_tags(&cols[11]),
|
||||||
|
thumb_file: cols[0].trim().to_string(),
|
||||||
|
uploaded_at: Self::parse_date(&date_raw),
|
||||||
|
date: if date_raw.is_empty() { None } else { Some(date_raw) },
|
||||||
|
model_name,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
fn request_headers(&self) -> Vec<(String, String)> {
|
||||||
|
vec![
|
||||||
|
("accept".to_string(), "text/csv,*/*;q=0.8".to_string()),
|
||||||
|
(
|
||||||
|
"user-agent".to_string(),
|
||||||
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36"
|
||||||
|
.to_string(),
|
||||||
|
),
|
||||||
|
("referer".to_string(), format!("{BASE_URL}/")),
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn fetch_catalog(&self, options: &ServerOptions) -> Vec<RipnsfwVideo> {
|
||||||
|
if let Ok(guard) = self.catalog.read() {
|
||||||
|
if let Some((fetched_at, rows)) = guard.as_ref() {
|
||||||
|
if fetched_at.elapsed() < CATALOG_TTL {
|
||||||
|
return rows.clone();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut requester = requester_or_default(options, CHANNEL_ID, "fetch_catalog");
|
||||||
|
match requester
|
||||||
|
.get_with_headers(CSV_URL, self.request_headers(), None)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(text) => {
|
||||||
|
let rows = Self::parse_csv(&text);
|
||||||
|
if let Ok(mut guard) = self.catalog.write() {
|
||||||
|
*guard = Some((Instant::now(), rows.clone()));
|
||||||
|
}
|
||||||
|
rows
|
||||||
|
}
|
||||||
|
Err(error) => {
|
||||||
|
report_provider_error(CHANNEL_ID, "fetch_catalog", &error.to_string()).await;
|
||||||
|
if let Ok(guard) = self.catalog.read() {
|
||||||
|
if let Some((_, rows)) = guard.as_ref() {
|
||||||
|
return rows.clone();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
vec![]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn matches_target(video: &RipnsfwVideo, target: &Target) -> bool {
|
||||||
|
match target {
|
||||||
|
Target::Latest => true,
|
||||||
|
Target::Tag(tag) => {
|
||||||
|
let needle = tag.trim().to_lowercase();
|
||||||
|
video.tags.iter().any(|t| t.to_lowercase() == needle)
|
||||||
|
}
|
||||||
|
Target::Model(name) => {
|
||||||
|
let needle_slug = Self::slugify(name);
|
||||||
|
video.model_name.trim().to_lowercase() == name.trim().to_lowercase()
|
||||||
|
|| Self::slugify(&video.model_name) == needle_slug
|
||||||
|
}
|
||||||
|
Target::Search(query) => {
|
||||||
|
let needle = query.trim().to_lowercase();
|
||||||
|
video.model_name.to_lowercase().contains(&needle)
|
||||||
|
|| video
|
||||||
|
.guests
|
||||||
|
.iter()
|
||||||
|
.any(|g| g.to_lowercase().contains(&needle))
|
||||||
|
|| video.tags.iter().any(|t| t.to_lowercase().contains(&needle))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn build_item(&self, options: &ServerOptions, video: &RipnsfwVideo) -> VideoItem {
|
||||||
|
let slug = Self::slugify(&video.model_name);
|
||||||
|
let page_url = format!("{BASE_URL}/model/{slug}/{}", video.id);
|
||||||
|
let thumb = format!(
|
||||||
|
"{CDN_THUMB_BASE}{}",
|
||||||
|
utf8_percent_encode(&video.thumb_file, NON_ALPHANUMERIC)
|
||||||
|
);
|
||||||
|
|
||||||
|
let title = if video.guests.is_empty() {
|
||||||
|
video.model_name.trim().to_string()
|
||||||
|
} else {
|
||||||
|
format!("{} & {}", video.model_name.trim(), video.guests.join(", "))
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut item = VideoItem::new(
|
||||||
|
video.id.clone(),
|
||||||
|
title,
|
||||||
|
page_url,
|
||||||
|
CHANNEL_ID.to_string(),
|
||||||
|
thumb,
|
||||||
|
0,
|
||||||
|
);
|
||||||
|
|
||||||
|
item = item.uploader(video.model_name.trim().to_string());
|
||||||
|
item = item.uploader_url(format!("{BASE_URL}/model/{slug}"));
|
||||||
|
item.uploaderId = Some(format!("{CHANNEL_ID}:{slug}"));
|
||||||
|
|
||||||
|
if !video.tags.is_empty() {
|
||||||
|
item = item.tags(video.tags.clone());
|
||||||
|
}
|
||||||
|
if let Some(ts) = video.uploaded_at {
|
||||||
|
item = item.uploaded_at(ts);
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut formats = Vec::new();
|
||||||
|
if let Some(url) = &video.lulustream {
|
||||||
|
let proxy_url = build_proxy_url(options, "lulustream", &strip_url_scheme(url));
|
||||||
|
formats.push(
|
||||||
|
VideoFormat::m3u8(proxy_url, "high".to_string(), "hls".to_string())
|
||||||
|
.format_note("Lulustream".to_string()),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if let Some(url) = &video.doodstream {
|
||||||
|
let proxy_url = build_proxy_url(options, "doodstream", &strip_url_scheme(url));
|
||||||
|
formats.push(
|
||||||
|
VideoFormat::new(proxy_url, "high".to_string(), "mp4".to_string())
|
||||||
|
.format_note("Doodstream".to_string()),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
// Streamtape rows are rare in this feed and have no local resolver proxy
|
||||||
|
// wired up yet, so they are intentionally not surfaced as a format.
|
||||||
|
let _ = &video.streamtape;
|
||||||
|
|
||||||
|
if !formats.is_empty() {
|
||||||
|
item = item.formats(formats);
|
||||||
|
}
|
||||||
|
|
||||||
|
item
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl Provider for RipnsfwProvider {
|
||||||
|
async fn get_videos(
|
||||||
|
&self,
|
||||||
|
_cache: crate::util::cache::VideoCache,
|
||||||
|
_pool: DbPool,
|
||||||
|
sort: String,
|
||||||
|
query: Option<String>,
|
||||||
|
page: String,
|
||||||
|
per_page: String,
|
||||||
|
options: ServerOptions,
|
||||||
|
) -> Vec<VideoItem> {
|
||||||
|
let page_num = page.parse::<usize>().unwrap_or(1).max(1);
|
||||||
|
let per_page_num = per_page.parse::<usize>().unwrap_or(24).clamp(1, 60);
|
||||||
|
let sort_value = if sort.trim().is_empty() {
|
||||||
|
options.sort.clone().unwrap_or_else(|| "new".to_string())
|
||||||
|
} else {
|
||||||
|
sort
|
||||||
|
};
|
||||||
|
let query_value = query.unwrap_or_default();
|
||||||
|
let target = Self::resolve_target(&query_value);
|
||||||
|
|
||||||
|
let mut rows = self.fetch_catalog(&options).await;
|
||||||
|
rows.retain(|video| Self::matches_target(video, &target));
|
||||||
|
rows.sort_by(|a, b| b.date.cmp(&a.date));
|
||||||
|
if sort_value == "oldest" {
|
||||||
|
rows.reverse();
|
||||||
|
}
|
||||||
|
|
||||||
|
let start = (page_num - 1) * per_page_num;
|
||||||
|
rows.into_iter()
|
||||||
|
.skip(start)
|
||||||
|
.take(per_page_num)
|
||||||
|
.map(|video| self.build_item(&options, &video))
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_channel(&self, clientversion: ClientVersion) -> Option<Channel> {
|
||||||
|
Some(self.build_channel(clientversion))
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user