simon/hottub
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix six dead providers (shooshtime, hdporn92, camsoda, sextb, supjav, hentaimama)

Browse Source 
- shooshtime: update parse_list_videos selectors for the site's redesigned
  markup (div.card / a.title / a.image img / div.duration / div.list div.box
  / div.name a.text)
- hdporn92: mirror the site's own links.hls4||hls3||hls2 fallback and scan
  every P.A.C.K.E.R. block instead of assuming the first one is the player
  block; prefer a genuine .m3u8 candidate over a decoy .txt one
- camsoda: replace naive first-`{`/last-`}` JSON extraction with a proper
  balanced-brace scan so trailing wrapper markup can't corrupt the parse
  ("key must be a string")
- sextb: fix mirror_url() double-prefixing the scheme when building Jina
  mirror URLs (produced a malformed https://r.jina.ai/http://https://... URL)
- supjav: switch curl_cffi impersonation from chrome to safari18_0 - Cloudflare
  blocks the chrome/firefox profiles outright but passes Safari
- hentaimama: resolve every card on a listing page (not just the first
  perPage) and truncate after filtering, so a bulk "Coming soon" placeholder
  batch at the front of the feed no longer empties the whole page; report a
  provider error if literally every candidate fails

Refreshes graphify's graph and docs/provider-catalog.md with the new
technical details for all six providers.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01QTqf6orbHZ9rFFpVpcgzcR
This commit is contained in:
Simon
2026-09-22 13:56:36 +00:00
parent c244461d0b
commit c40b48b0db
8 changed files with 356 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
src/providers/camsoda.rs
View File

@@ -169,11 +169,43 @@ impl CamsodaProvider {
}
/// Extract the JSON object from a body that may be wrapped in HTML by the
/// Jina mirror (`<pre>{...}</pre>`) or returned raw.
/// Jina mirror (`<pre>{...}</pre>`) or returned raw. Scans for the
/// balanced top-level object starting at the first `{` instead of just
/// pairing it with the last `}` in the body — the naive approach breaks
/// when the wrapper has trailing content of its own (e.g. a tracking
/// `<script>` block after `</pre>` containing an unquoted-key JS object
/// literal), which pulls that garbage into the "JSON" and makes
/// `serde_json` fail with errors like "key must be a string".
fn extract_json(body: &str) -> Option<&str> {
let start = body.find('{')?;
let end = body.rfind('}')?;
(end > start).then(|| &body[start..=end])
let mut depth = 0i32;
let mut in_string = false;
let mut escaped = false;
for (offset, ch) in body[start..].char_indices() {
if in_string {
if escaped {
escaped = false;
} else if ch == '\\' {
escaped = true;
} else if ch == '"' {
in_string = false;
}
continue;
}
match ch {
'"' => in_string = true,
'{' => depth += 1,
'}' => {
depth -= 1;
if depth == 0 {
let end = start + offset + ch.len_utf8();
return Some(&body[start..end]);
}
}
_ => {}
}
}
None
}
/// Parse the `browse/react` JSON (top-level `userList`) into live
@@ -411,6 +443,20 @@ mod tests {
assert!(tags.contains(&"VR".to_string()));
}
#[test]
fn ignores_trailing_garbage_after_the_json_object() {
// A Jina/FlareSolverr wrapper can append its own markup after the
// real JSON (e.g. a tracking <script> block with an unquoted-key JS
// object literal). The naive first-`{`/last-`}` span used to pull
// that in and made serde_json fail with "key must be a string".
let body = r#"<html><body><pre>{"perPageCount":1,"totalCount":1,"userList":[
{"id":1,"username":"theowonder","displayName":"Theo Wonder","connectionCount":"69","status":"online","subjectText":"hi","thumbUrl":"https://media.livemediahost.com/thumbs/199/theowonder.jpg","vr":false}
]}</pre></body><script>ga({event:'view'})</script></html>"#;
let items = CamsodaProvider::parse_items(body).unwrap();
assert_eq!(items.len(), 1);
assert_eq!(items[0].id, "theowonder");
}
#[test]
fn picks_target() {
assert_eq!(

173
src/providers/hdporn92.rs
View File

@@ -265,53 +265,99 @@ impl Hdporn92Provider {
}
}
/// Decodes the P.A.C.K.E.R. `eval(function(p,a,c,k,e,d){...}(...))` block in
/// the morencius embed page and returns the deobfuscated JavaScript source.
fn decode_packered(source: &str) -> Option<String> {
/// Decodes every P.A.C.K.E.R. `eval(function(p,a,c,k,e,d){...}(...))` block
/// found in the morencius embed page and returns the deobfuscated JS
/// source for each. The embed page carries more than one such block (an
/// ad/VAST setup block plus the actual jwplayer setup block), so callers
/// must scan all of them for the data they need rather than assume the
/// first one is the relevant one.
fn decode_packered_blocks(source: &str) -> Vec<String> {
// Match: eval(function(p,a,c,k,e,d){...}('P',A,C,'K'.split('|')
// We capture P, A, C, and the dictionary K.
let re = Regex::new(
let Ok(re) = Regex::new(
r#"eval\(function\(p,a,c,k,e,d\)\{.*?\}\('([\s\S]*?)',(\d+),(\d+),'([\s\S]*?)'\.split\('\|'\)\)\)"#,
)
.ok()?;
let caps = re.captures(source)?;
let p = caps.get(1)?.as_str();
let a: u32 = caps.get(2)?.as_str().parse().ok()?;
let c: usize = caps.get(3)?.as_str().parse().ok()?;
let k: Vec<&str> = caps.get(4)?.as_str().split('|').collect();
) else {
return Vec::new();
};
// P.A.C.K.E.R. deobfuscation: iterate c-1..=0; replace \b<token>\b with
// dictionary[i] (the original JS uses a `\w+` lookalike via the `e`
// parameter, but in practice every token is a base-A digit string).
let mut result = p.to_string();
for i in (0..c).rev() {
let value = k.get(i).copied().unwrap_or("");
if value.is_empty() {
continue;
}
let token = to_base(i, a);
// Build a word-boundary regex for the token. Base-A tokens are
// always made of [0-9a-z], so \b is a real word boundary.
let token_re = Regex::new(&format!(r"\b{}\b", regex::escape(&token))).ok()?;
result = token_re.replace_all(&result, value).into_owned();
}
Some(result)
re.captures_iter(source)
.filter_map(|caps| {
let p = caps.get(1)?.as_str();
let a: u32 = caps.get(2)?.as_str().parse().ok()?;
let c: usize = caps.get(3)?.as_str().parse().ok()?;
let k: Vec<&str> = caps.get(4)?.as_str().split('|').collect();
// P.A.C.K.E.R. deobfuscation: iterate c-1..=0; replace
// \b<token>\b with dictionary[i] (the original JS uses a
// `\w+` lookalike via the `e` parameter, but in practice
// every token is a base-A digit string).
let mut result = p.to_string();
for i in (0..c).rev() {
let value = k.get(i).copied().unwrap_or("");
if value.is_empty() {
continue;
}
let token = to_base(i, a);
// Build a word-boundary regex for the token. Base-A
// tokens are always made of [0-9a-z], so \b is a real
// word boundary.
let token_re =
Regex::new(&format!(r"\b{}\b", regex::escape(&token))).ok()?;
result = token_re.replace_all(&result, value).into_owned();
}
Some(result)
})
.collect()
}
/// Pulls the morencius `/stream/.../master.m3u8` URL out of the decoded JS.
fn extract_hls4_url(decoded_js: &str) -> Option<String> {
// The decoded JS has: "hls4":"/stream/<token>/<secret>/<expiry>/<file_id>/master.m3u8"
// The value is a path (no scheme), so we prefix it.
let re = Regex::new(r#""hls4"\s*:\s*"([^"]+)""#).ok()?;
let caps = re.captures(decoded_js)?;
let path = caps.get(1)?.as_str();
if path.starts_with("http") {
Some(path.to_string())
} else if path.starts_with('/') {
Some(format!("https://morencius.com{path}"))
} else {
Some(format!("https://morencius.com/{path}"))
}
/// Decodes the first P.A.C.K.E.R. block found in `source`. Kept for
/// callers/tests that only care about a single block; production code
/// that needs to find the block with a specific key (e.g. `hls4`) should
/// use [`Self::decode_packered_blocks`] and scan all of them instead.
fn decode_packered(source: &str) -> Option<String> {
Self::decode_packered_blocks(source).into_iter().next()
}
/// Pulls the morencius HLS URL out of the decoded JS. The player itself
/// picks `links.hls4 || links.hls3 || links.hls2` (not every tier is
/// always populated, and hls4 is frequently absent), so mirror that
/// priority order. At least one tier has been observed serving a `.txt`
/// decoy instead of a real `.m3u8` playlist, so within that priority
/// order prefer a candidate that actually looks like a stream URL.
fn extract_hls_url(decoded_js: &str) -> Option<String> {
let re = Regex::new(r#""(hls[234])"\s*:\s*"([^"]+)""#).ok()?;
let candidates: Vec<(String, String)> = re
.captures_iter(decoded_js)
.filter_map(|caps| {
let tier = caps.get(1)?.as_str().to_string();
let raw = caps.get(2)?.as_str().replace("\\/", "/");
Some((tier, raw))
})
.collect();
let normalize = |path: &str| {
if path.starts_with("http") {
path.to_string()
} else if path.starts_with('/') {
format!("https://morencius.com{path}")
} else {
format!("https://morencius.com/{path}")
}
};
const TIERS: [&str; 3] = ["hls4", "hls3", "hls2"];
TIERS
.iter()
.filter_map(|tier| candidates.iter().find(|(t, _)| t == tier))
.map(|(_, raw)| normalize(raw))
.find(|url| url.to_lowercase().ends_with(".m3u8"))
.or_else(|| {
TIERS
.iter()
.filter_map(|tier| candidates.iter().find(|(t, _)| t == tier))
.map(|(_, raw)| normalize(raw))
.next()
})
}
/// Pulls the integer duration (in seconds, may be decimal) from the decoded
@@ -336,11 +382,21 @@ impl Hdporn92Provider {
// Step 2: fetch the morencius embed page (referer = the hdporn92 detail).
let player_html = Self::fetch_html(options, &embed_url, &stub.url).await?;
let decoded = Self::decode_packered(&player_html)
.ok_or_else(|| Error::from(format!("no eval block in {embed_url}")))?;
let stream_path = Self::extract_hls4_url(&decoded)
.ok_or_else(|| Error::from(format!("no hls4 url in decoded JS of {embed_url}")))?;
let duration = Self::extract_duration(&decoded).unwrap_or(0);
let decoded_blocks = Self::decode_packered_blocks(&player_html);
if decoded_blocks.is_empty() {
return Err(Error::from(format!("no eval block in {embed_url}")));
}
// A page may carry multiple P.A.C.K.E.R. blocks (e.g. an ad/VAST
// setup block); only one decodes to JS containing the player's
// hls2/3/4 stream keys, so scan all of them rather than assuming
// the first.
let decoded = decoded_blocks
.iter()
.find(|block| Self::extract_hls_url(block).is_some())
.ok_or_else(|| Error::from(format!("no hls url in decoded JS of {embed_url}")))?;
let stream_path = Self::extract_hls_url(decoded)
.ok_or_else(|| Error::from(format!("no hls url in decoded JS of {embed_url}")))?;
let duration = Self::extract_duration(decoded).unwrap_or(0);
// Build the format. morencius serves the m3u8 only when Referer is the
// embed URL — without it, the response is 403.
@@ -568,7 +624,7 @@ mod tests {
.expect("deobfuscator should return Some");
// Sanity: the decoded text contains the hls4 key.
assert!(decoded.contains("\"hls4\""), "decoded: {decoded}");
let url = Hdporn92Provider::extract_hls4_url(&decoded)
let url = Hdporn92Provider::extract_hls_url(&decoded)
.expect("hls4 should be present after decode");
// The path was /stream/abc/secret/123/456/master.m3u8.
assert_eq!(url, "https://morencius.com/stream/abc/secret/123/456/master.m3u8");
@@ -603,18 +659,19 @@ mod tests {
"src/providers/testdata/hdporn92_morencius.html",
)
.expect("test fixture should exist");
let decoded = Hdporn92Provider::decode_packered(&html)
.expect("real morencius HTML should be deobfuscable");
// The decoded JS contains the hls4 value.
let url = Hdporn92Provider::extract_hls4_url(&decoded)
.expect("hls4 should be present after decode");
assert!(
url.starts_with("https://morencius.com/stream/"),
"got {url}"
);
assert!(url.ends_with("/master.m3u8"), "got {url}");
// The page carries several P.A.C.K.E.R. blocks (ad setup, player
// setup, ...); only the player block decodes to JS containing hls4.
let blocks = Hdporn92Provider::decode_packered_blocks(&html);
assert!(!blocks.is_empty(), "real morencius HTML should be deobfuscable");
let decoded = blocks
.iter()
.find(|block| Hdporn92Provider::extract_hls_url(block).is_some())
.expect("one decoded block should contain an hls2/3/4 key");
let url = Hdporn92Provider::extract_hls_url(decoded)
.expect("an hls url should be present after decode");
assert!(url.starts_with("https://"), "got {url}");
// The duration is also present.
let duration = Hdporn92Provider::extract_duration(&decoded);
let duration = Hdporn92Provider::extract_duration(decoded);
assert!(duration.unwrap_or(0) > 0, "duration should be > 0");
}
}

24
src/providers/hentaimama.rs
View File

@@ -570,20 +570,30 @@ impl HentaimamaProvider {
let mut requester = requester_or_default(&options, CHANNEL_ID, "get_videos");
let html = Self::fetch_html(&mut requester, &list_url, BASE_URL).await?;
let candidate_count;
let items = match target {
Target::Latest => {
let cards = Self::parse_episode_cards(&html)?;
stream::iter(cards.into_iter().take(per_page.max(1)).map(|entry| {
candidate_count = cards.len();
// Resolve every card on the page rather than just the first
// `per_page` - the site sometimes bulk-lists a whole series'
// episodes as "Coming soon" placeholders (get_player_contents
// returns `[]`, no error), which would otherwise silently
// wipe out the entire feed if those happen to sort first.
let mut items = stream::iter(cards.into_iter().map(|entry| {
let mut req = requester.clone();
async move { Self::build_item_from_episode_card(&mut req, &entry).await }
}))
.buffer_unordered(6)
.filter_map(|item| async move { item })
.collect::<Vec<_>>()
.await
.await;
items.truncate(per_page.max(1));
items
}
_ => {
let urls = Self::parse_series_card_urls(&html)?;
candidate_count = urls.len();
stream::iter(urls.into_iter().take(per_page.max(1)).map(|url| {
let mut req = requester.clone();
async move { Self::resolve_series_to_items(&mut req, &url).await }
@@ -595,6 +605,16 @@ impl HentaimamaProvider {
}
};
if items.is_empty() && candidate_count > 0 {
report_provider_error_background(
CHANNEL_ID,
"get.all_candidates_failed",
&format!(
"{candidate_count} candidate card(s) parsed from {list_url} but none resolved to a playable item"
),
);
}
if !items.is_empty() {
cache.insert(list_url, items.clone());
}

19
src/providers/sextb.rs
View File

@@ -243,7 +243,15 @@ impl SextbProvider {
}
fn mirror_url(&self, url: &str) -> String {
format!("{MIRROR_PREFIX}{url}")
// `url` is already an absolute `https://sextb.net/...` URL, and
// MIRROR_PREFIX already supplies a scheme — concatenating them
// directly produces a malformed `.../http://https://...` target
// that Jina rejects, so strip the scheme first.
let stripped = url
.strip_prefix("https://")
.or_else(|| url.strip_prefix("http://"))
.unwrap_or(url);
format!("{MIRROR_PREFIX}{stripped}")
}
fn slugify(value: &str) -> String {
@@ -1258,6 +1266,15 @@ mod tests {
);
}
#[test]
fn builds_jina_mirror_url_without_double_scheme() {
let url = provider().mirror_url("https://sextb.net/censored?genre=all");
assert_eq!(
url,
"https://r.jina.ai/http://sextb.net/censored?genre=all"
);
}
#[test]
fn builds_search_page_two_url() {
let url = provider().build_url_for_target(

14
src/providers/shooshtime.rs
View File

@@ -811,13 +811,13 @@ impl ShooshtimeProvider {
fn parse_list_videos(&self, html: &str) -> Result<Vec<VideoItem>> {
let document = Html::parse_document(html);
let card_selector = Self::selector("div.item.thumb")?;
let video_link_selector = Self::selector("a[href*=\"/videos/\"]")?;
let image_selector = Self::selector("img")?;
let duration_selector = Self::selector("span.duration")?;
let title_selector = Self::selector("span.description, span.description.secondary")?;
let info_selector = Self::selector("ul.info li.val")?;
let uploader_selector = Self::selector("ul.info a[href*=\"/members/\"]")?;
let card_selector = Self::selector("div.card")?;
let video_link_selector = Self::selector("a.title")?;
let image_selector = Self::selector("a.image img")?;
let duration_selector = Self::selector("div.duration")?;
let title_selector = Self::selector("a.title")?;
let info_selector = Self::selector("div.list div.box")?;
let uploader_selector = Self::selector("div.name a.text")?;
let mut items = Vec::new();

2
src/providers/supjav.rs
View File

@@ -587,7 +587,7 @@ if referer:
response = requests.get(
url,
impersonate="chrome",
impersonate="safari18_0",
timeout=30,
allow_redirects=True,
headers=headers,

138
src/providers/testdata/hdporn92_morencius.html vendored Normal file
View File

File diff suppressed because one or more lines are too long