simon/hottub
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix melonstube provider: Cloudflare bot management now blocks it

Browse Source 
melonstube.com started fronting itself with Cloudflare bot management
(a real Turnstile challenge) since this provider was written, so the
shared Requester's Firefox151 emulation got blocked on every path and
the provider returned 0 items.

A/B-probed several browser TLS fingerprints live: Chrome120/124 pass
the plain listing pages but still 403 on /category/* and the
/searching/by-form search POST, while Safari18 passes everywhere.
Listing/search fetches now try a dedicated Safari18-emulated client
first and fall back to the shared Requester if Cloudflare's rules
change again (same pattern as lulustream.rs/doodstream.rs).

Also fixes get_videos ignoring perPage entirely (always returned the
full ~120-card page); it now truncates to the requested size.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01QTqf6orbHZ9rFFpVpcgzcR
This commit is contained in:
Simon
2026-09-21 14:08:28 +00:00
parent 534ee4b5ad
commit c7ae89fc37
2 changed files with 95 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

113
src/providers/melonstube.rs
View File

@@ -12,7 +12,11 @@ use chrono::{Duration as ChronoDuration, Utc};
use error_chain::error_chain;
use regex::Regex;
use scraper::{Html, Selector};
use std::sync::Arc;
use url::form_urlencoded::Serializer;
use wreq::cookie::Jar;
use wreq::redirect::Policy;
use wreq_util::Emulation;
pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata =
crate::providers::ProviderChannelMetadata {
@@ -218,6 +222,65 @@ impl MelonstubeProvider {
Some((destination, id))
}
// melonstube.com started fronting itself with Cloudflare bot management at
// some point after this provider was written: the shared Requester's
// Firefox151 emulation now gets a straight Turnstile "challenge" page on
// every path. Verified live with curl_cffi across many fingerprints —
// Chrome120/124 pass the plain /new /popular /rating listing pages but
// still 403 on /category/* and /searching/by-form; Safari18 is the only
// profile that passes all of them (listing, category/search-result pages,
// and the /searching/by-form POST) — so listing/search fetches try a
// dedicated Safari18 client first and fall back to the shared Requester
// (with its Jina/FlareSolverr fallback chain) if Cloudflare ever changes
// its rules again.
fn build_bypass_client() -> Option<wreq::Client> {
let jar = Arc::new(Jar::default());
wreq::Client::builder()
.tls_cert_verification(false)
.emulation(Emulation::Safari18)
.cookie_provider(jar)
.redirect(Policy::default())
.build()
.ok()
}
async fn fetch_html_bypass(url: &str) -> Option<String> {
let client = Self::build_bypass_client()?;
let response = client
.get(url)
.header(
"accept",
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
)
.header("accept-language", "en-US,en;q=0.5")
.header("referer", BASE_URL)
.send()
.await
.ok()?;
if !response.status().is_success() {
return None;
}
response.text().await.ok()
}
async fn resolve_search_base_url_bypass(query: &str) -> Option<String> {
let search_url = format!("{BASE_URL}/searching/by-form");
let body = Self::build_search_request_body(query);
let client = Self::build_bypass_client()?;
let response = client
.post(&search_url)
.header("content-type", "application/x-www-form-urlencoded")
.header("referer", BASE_URL)
.body(body)
.send()
.await
.ok()?;
if !response.status().is_success() {
return None;
}
Some(response.uri().to_string().trim_end_matches('/').to_string())
}
fn is_junk_host(host: &str) -> bool {
JUNK_HOSTS.contains(&host) || host.starts_with("join.")
}
@@ -388,6 +451,10 @@ impl MelonstubeProvider {
}
async fn resolve_search_base_url(query: &str, options: &ServerOptions) -> Result<String> {
if let Some(base) = Self::resolve_search_base_url_bypass(query).await {
return Ok(base);
}
let search_url = format!("{BASE_URL}/searching/by-form");
let body = Self::build_search_request_body(query);
@@ -440,23 +507,28 @@ impl MelonstubeProvider {
}
}
let mut requester = requester_or_default(&options, CHANNEL_ID, "fetch_and_parse");
let html = requester
.get_with_headers(
&url,
vec![
("user-agent".to_string(), BROWSER_UA.to_string()),
(
"accept".to_string(),
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8".to_string(),
),
("accept-language".to_string(), "en-US,en;q=0.5".to_string()),
],
Some(wreq::Version::HTTP_11),
)
.await
.map_err(|e| Error::from(format!("request failed url={url}: {e}")))?;
let html = match Self::fetch_html_bypass(&url).await {
Some(html) if !html.is_empty() => html,
_ => {
let mut requester = requester_or_default(&options, CHANNEL_ID, "fetch_and_parse");
requester
.get_with_headers(
&url,
vec![
("user-agent".to_string(), BROWSER_UA.to_string()),
(
"accept".to_string(),
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
.to_string(),
),
("accept-language".to_string(), "en-US,en;q=0.5".to_string()),
],
Some(wreq::Version::HTTP_11),
)
.await
.map_err(|e| Error::from(format!("request failed url={url}: {e}")))?
}
};
if html.is_empty() {
return Ok(vec![]);
@@ -485,13 +557,16 @@ impl Provider for MelonstubeProvider {
options: ServerOptions,
) -> Vec<VideoItem> {
let _ = pool;
let _ = per_page;
let per_page = per_page.parse::<usize>().unwrap_or(20).clamp(1, 120);
let page = page.parse::<u32>().unwrap_or(1);
let query_str = query.unwrap_or_default();
let target = Self::resolve_target(&query_str, &sort, page);
match self.fetch_and_parse(cache, target, options).await {
Ok(items) => items,
Ok(mut items) => {
items.truncate(per_page);
items
}
Err(e) => {
report_provider_error(CHANNEL_ID, "get_videos", &e.to_string()).await;
vec![]