From 50ea0e73b7b111fe7daccb1d7b4700a1a2cc154c Mon Sep 17 00:00:00 2001 From: Simon Date: Sun, 22 Mar 2026 12:27:46 +0000 Subject: [PATCH] pimpbunny fix --- build.rs | 9 + src/api.rs | 9 +- src/db.rs | 24 ++- src/main.rs | 5 +- src/providers/all.rs | 1 - src/providers/hanime.rs | 26 +-- src/providers/mod.rs | 25 +++ src/providers/pimpbunny.rs | 374 +++++++++++++++++++++++++++++++------ src/status.rs | 15 +- src/util/flow_debug.rs | 4 +- src/util/mod.rs | 1 + src/util/requester.rs | 141 +++++++++++--- src/videos.rs | 152 +++++++++++++-- 13 files changed, 646 insertions(+), 140 deletions(-) diff --git a/build.rs b/build.rs index 7f10a2b..6fa6e80 100644 --- a/build.rs +++ b/build.rs @@ -231,6 +231,14 @@ fn main() { println!("cargo:rerun-if-env-changed=HOT_TUB_PROVIDER"); println!("cargo:rerun-if-env-changed=HOTTUB_PROVIDER"); println!("cargo:rustc-check-cfg=cfg(hottub_single_provider)"); + let provider_cfg_values = PROVIDERS + .iter() + .map(|provider| format!("\"{}\"", provider.id)) + .collect::>() + .join(", "); + println!( + "cargo:rustc-check-cfg=cfg(hottub_provider, values({provider_cfg_values}))" + ); let selected = env::var("HOT_TUB_PROVIDER") .or_else(|_| env::var("HOTTUB_PROVIDER")) @@ -247,6 +255,7 @@ fn main() { panic!("Unknown provider `{selected_id}` from HOT_TUB_PROVIDER/HOTTUB_PROVIDER") }); println!("cargo:rustc-cfg=hottub_single_provider"); + println!("cargo:rustc-cfg=hottub_provider=\"{selected_id}\""); vec![provider] } None => PROVIDERS.iter().collect(), diff --git a/src/api.rs b/src/api.rs index 7ce10b6..660d88b 100644 --- a/src/api.rs +++ b/src/api.rs @@ -146,6 +146,7 @@ pub fn config(cfg: &mut web::ServiceConfig) { } async fn status(req: HttpRequest) -> Result { + #[cfg(feature = "debug")] let trace_id = crate::util::flow_debug::next_trace_id("status"); let clientversion: ClientVersion = match req.headers().get("User-Agent") { Some(v) => match v.to_str() { @@ -175,6 +176,7 @@ async fn status(req: HttpRequest) -> Result { .to_string(); let public_url_base = format!("{}://{}", req.connection_info().scheme(), host); let mut status = Status::new(); + #[cfg(feature = "debug")] let mut channel_count = 0usize; for (provider_name, provider) in ALL_PROVIDERS.iter() { @@ -191,7 +193,10 @@ async fn status(req: HttpRequest) -> Result { if channel.favicon.starts_with('/') { channel.favicon = format!("{}{}", public_url_base, channel.favicon); } - channel_count += 1; + #[cfg(feature = "debug")] + { + channel_count += 1; + } crate::flow_debug!( "trace={} status added channel id={} provider={}", trace_id, @@ -407,6 +412,7 @@ async fn videos_post( } if let Some(literal_query) = literal_query.as_deref() { + #[cfg(feature = "debug")] let before = video_items.len(); video_items.retain(|video| video_matches_literal_query(video, literal_query)); crate::flow_debug!( @@ -434,6 +440,7 @@ async fn videos_post( let per_page_clone = perPage.to_string(); let options_clone = options.clone(); let channel_clone = channel.clone(); + #[cfg(feature = "debug")] let prefetch_trace_id = trace_id.clone(); task::spawn_local(async move { crate::flow_debug!( diff --git a/src/db.rs b/src/db.rs index c833356..596f2a6 100644 --- a/src/db.rs +++ b/src/db.rs @@ -1,10 +1,17 @@ -use crate::models::DBVideo; use diesel::prelude::*; +#[cfg(any( + not(hottub_single_provider), + hottub_provider = "hanime", + hottub_provider = "hentaihaven", + hottub_provider = "missav", + hottub_provider = "perverzija", +))] pub fn get_video( conn: &mut SqliteConnection, video_id: String, ) -> Result, diesel::result::Error> { + use crate::models::DBVideo; use crate::schema::videos::dsl::*; let result = videos .filter(id.eq(video_id)) @@ -16,11 +23,19 @@ pub fn get_video( } } +#[cfg(any( + not(hottub_single_provider), + hottub_provider = "hanime", + hottub_provider = "hentaihaven", + hottub_provider = "missav", + hottub_provider = "perverzija", +))] pub fn insert_video( conn: &mut SqliteConnection, new_id: &str, new_url: &str, ) -> Result { + use crate::models::DBVideo; use crate::schema::videos::dsl::*; diesel::insert_into(videos) .values(DBVideo { @@ -30,6 +45,13 @@ pub fn insert_video( .execute(conn) } +#[cfg(any( + not(hottub_single_provider), + hottub_provider = "hanime", + hottub_provider = "hentaihaven", + hottub_provider = "missav", + hottub_provider = "perverzija", +))] pub fn delete_video( conn: &mut SqliteConnection, video_id: String, diff --git a/src/main.rs b/src/main.rs index afcdaa9..b809944 100644 --- a/src/main.rs +++ b/src/main.rs @@ -57,10 +57,11 @@ async fn main() -> std::io::Result<()> { ); let mut requester = util::requester::Requester::new(); - requester.set_proxy(env::var("PROXY").unwrap_or("0".to_string()) != "0".to_string()); + let proxy_enabled = env::var("PROXY").unwrap_or("0".to_string()) != "0".to_string(); + requester.set_proxy(proxy_enabled); crate::flow_debug!( "requester initialized proxy_enabled={}", - requester.proxy_enabled() + proxy_enabled ); let cache: util::cache::VideoCache = crate::util::cache::VideoCache::new() diff --git a/src/providers/all.rs b/src/providers/all.rs index 74cf4d4..544530c 100644 --- a/src/providers/all.rs +++ b/src/providers/all.rs @@ -28,7 +28,6 @@ error_chain! { } #[derive(Debug, Clone)] -#[allow(dead_code)] pub struct AllProvider {} impl AllProvider { diff --git a/src/providers/hanime.rs b/src/providers/hanime.rs index 1ec68a6..fe41791 100644 --- a/src/providers/hanime.rs +++ b/src/providers/hanime.rs @@ -37,7 +37,6 @@ struct HanimeSearchRequest { page: u8, } -#[allow(dead_code)] impl HanimeSearchRequest { pub fn new() -> Self { HanimeSearchRequest { @@ -51,26 +50,10 @@ impl HanimeSearchRequest { page: 0, } } - pub fn tags(mut self, tags: Vec) -> Self { - self.tags = tags; - self - } pub fn search_text(mut self, search_text: String) -> Self { self.search_text = search_text; self } - pub fn tags_mode(mut self, tags_mode: String) -> Self { - self.tags_mode = tags_mode; - self - } - pub fn brands(mut self, brands: Vec) -> Self { - self.brands = brands; - self - } - pub fn blacklist(mut self, blacklist: Vec) -> Self { - self.blacklist = blacklist; - self - } pub fn order_by(mut self, order_by: String) -> Self { self.order_by = order_by; self @@ -120,16 +103,11 @@ struct HanimeSearchResult { } #[derive(Debug, Clone)] -#[allow(dead_code)] -pub struct HanimeProvider { - url: String, -} +pub struct HanimeProvider; impl HanimeProvider { pub fn new() -> Self { - HanimeProvider { - url: "https://hanime.tv/".to_string(), - } + HanimeProvider } fn build_channel(&self, _clientversion: ClientVersion) -> Channel { diff --git a/src/providers/mod.rs b/src/providers/mod.rs index 31da4d8..4498baa 100644 --- a/src/providers/mod.rs +++ b/src/providers/mod.rs @@ -230,6 +230,10 @@ fn channel_group_order(group_id: &str) -> usize { pub fn decorate_channel(channel: Channel) -> ChannelView { let metadata = channel_metadata_for(&channel.id); + let ytdlp_command = match channel.id.as_str() { + "pimpbunny" => Some("yt-dlp --compat-options allow-unsafe-ext".to_string()), + _ => None, + }; ChannelView { id: channel.id, name: channel.name, @@ -250,6 +254,7 @@ pub fn decorate_channel(channel: Channel) -> ChannelView { .map(|tag| (*tag).to_string()) .collect() }), + ytdlpCommand: ytdlp_command, cacheDuration: channel.cacheDuration, } } @@ -413,6 +418,7 @@ mod tests { let channel = decorate_channel(base_channel("hsex")); assert_eq!(channel.groupKey.as_deref(), Some("chinese")); assert_eq!(channel.sortOrder, None); + assert_eq!(channel.ytdlpCommand, None); assert_eq!( channel.tags.as_deref(), Some( @@ -438,6 +444,15 @@ mod tests { assert_eq!(groups[2].id, "jav"); } + #[test] + fn decorates_pimpbunny_with_ytdlp_command() { + let channel = decorate_channel(base_channel("pimpbunny")); + assert_eq!( + channel.ytdlpCommand.as_deref(), + Some("yt-dlp --compat-options allow-unsafe-ext") + ); + } + #[test] fn reflects_updated_group_moves() { assert_eq!( @@ -461,6 +476,7 @@ mod tests { base_channel("missav"), base_channel("hsex"), base_channel("all"), + base_channel("pimpbunny"), ]; let json = serde_json::to_value(build_status_response(status)).expect("valid status json"); @@ -487,5 +503,14 @@ mod tests { .find(|group| group["id"] == "chinese") .expect("chinese group present"); assert_eq!(chinese_group["systemImage"], "globe"); + + let pimpbunny_channel = channels + .iter() + .find(|channel| channel["id"] == "pimpbunny") + .expect("pimpbunny channel present"); + assert_eq!( + pimpbunny_channel["ytdlpCommand"], + "yt-dlp --compat-options allow-unsafe-ext" + ); } } diff --git a/src/providers/pimpbunny.rs b/src/providers/pimpbunny.rs index 88c3703..e43190f 100644 --- a/src/providers/pimpbunny.rs +++ b/src/providers/pimpbunny.rs @@ -49,8 +49,7 @@ impl PimpbunnyProvider { const FIREFOX_USER_AGENT: &'static str = "Mozilla/5.0 (X11; Linux x86_64; rv:147.0) Gecko/20100101 Firefox/147.0"; const HTML_ACCEPT: &'static str = - "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8"; - + "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"; pub fn new() -> Self { let provider = Self { url: "https://pimpbunny.com".to_string(), @@ -235,39 +234,218 @@ impl PimpbunnyProvider { format!("{}/", self.url.trim_end_matches('/')) } - fn root_headers(&self) -> Vec<(String, String)> { - Self::html_headers_with_referer(&self.root_referer()) + fn sort_by(sort: &str) -> &'static str { + match sort { + "best rated" => "rating", + "most viewed" => "video_viewed", + _ => "post_date", + } } - fn html_headers_with_referer(referer: &str) -> Vec<(String, String)> { - vec![ - ("Referer".to_string(), referer.to_string()), + fn build_search_path_query(query: &str, separator: &str) -> String { + query.split_whitespace().collect::>().join(separator) + } + + fn append_archive_query(url: String, sort: &str) -> String { + let separator = if url.contains('?') { '&' } else { '?' }; + format!("{url}{separator}sort_by={}", Self::sort_by(sort)) + } + + fn page_family_referer(&self, request_url: &str) -> String { + let Some(url) = Url::parse(request_url).ok() else { + return self.root_referer(); + }; + + let path = url.path(); + let referer_path = if path.starts_with("/videos/") { + "/videos/".to_string() + } else if path.starts_with("/search/") { + let parts: Vec<_> = path.trim_matches('/').split('/').collect(); + if parts.len() >= 2 { + format!("/search/{}/", parts[1]) + } else { + "/search/".to_string() + } + } else if path.starts_with("/categories/") { + let parts: Vec<_> = path.trim_matches('/').split('/').collect(); + if parts.len() >= 2 { + format!("/categories/{}/", parts[1]) + } else { + "/categories/".to_string() + } + } else if path.starts_with("/onlyfans-models/") { + let parts: Vec<_> = path.trim_matches('/').split('/').collect(); + if parts.len() >= 2 { + format!("/onlyfans-models/{}/", parts[1]) + } else { + "/onlyfans-models/".to_string() + } + } else { + "/".to_string() + }; + + format!("{}{}", self.url.trim_end_matches('/'), referer_path) + } + + fn build_browse_url(&self, page: u8, sort: &str) -> String { + let base = if page <= 1 { + format!("{}/videos/", self.url) + } else { + format!("{}/videos/{page}/", self.url) + }; + Self::append_archive_query(base, sort) + } + + fn build_search_url(&self, query: &str, page: u8, sort: &str) -> String { + let path_query = Self::build_search_path_query(query, "-"); + let base = if page <= 1 { + format!("{}/search/{path_query}/", self.url) + } else { + format!("{}/search/{path_query}/{page}/", self.url) + }; + Self::append_archive_query(base, sort) + } + + fn build_common_archive_url(&self, archive_path: &str, page: u8, sort: &str) -> String { + let canonical = format!( + "{}/{}", + self.url.trim_end_matches('/'), + archive_path.trim_start_matches('/') + ); + let base = if page <= 1 { + canonical + } else { + format!("{}/{}", canonical.trim_end_matches('/'), page) + }; + let base = if base.ends_with('/') { + base + } else { + format!("{base}/") + }; + Self::append_archive_query(base, sort) + } + + fn navigation_headers( + referer: Option<&str>, + sec_fetch_site: &'static str, + ) -> Vec<(String, String)> { + let mut headers = vec![ ( "User-Agent".to_string(), Self::FIREFOX_USER_AGENT.to_string(), ), ("Accept".to_string(), Self::HTML_ACCEPT.to_string()), ("Accept-Language".to_string(), "en-US,en;q=0.9".to_string()), - ] + ("Cache-Control".to_string(), "no-cache".to_string()), + ("Pragma".to_string(), "no-cache".to_string()), + ("Priority".to_string(), "u=0, i".to_string()), + ("Connection".to_string(), "keep-alive".to_string()), + ("TE".to_string(), "trailers".to_string()), + ("Sec-Fetch-Dest".to_string(), "document".to_string()), + ("Sec-Fetch-Mode".to_string(), "navigate".to_string()), + ("Sec-Fetch-Site".to_string(), sec_fetch_site.to_string()), + ("Sec-Fetch-User".to_string(), "?1".to_string()), + ("Upgrade-Insecure-Requests".to_string(), "1".to_string()), + ]; + if let Some(referer) = referer { + headers.push(("Referer".to_string(), referer.to_string())); + } + headers } fn headers_with_cookies( &self, requester: &Requester, request_url: &str, - referer: &str, + referer: Option<&str>, + sec_fetch_site: &'static str, ) -> Vec<(String, String)> { - let mut headers = Self::html_headers_with_referer(referer); + let mut headers = Self::navigation_headers(referer, sec_fetch_site); if let Some(cookie) = requester.cookie_header_for_url(request_url) { headers.push(("Cookie".to_string(), cookie)); } headers } + fn is_cloudflare_challenge(html: &str) -> bool { + html.contains("cf-turnstile-response") + || html.contains("Performing security verification") + || html.contains("__cf_chl_rt_tk") + || html.contains("cUPMDTk:\"") + || html.contains("Just a moment...") + } + + fn extract_challenge_path(html: &str) -> Option { + html.split("cUPMDTk:\"") + .nth(1) + .and_then(|s| s.split('"').next()) + .map(str::to_string) + .or_else(|| { + html.split("__cf_chl_rt_tk=") + .nth(1) + .and_then(|s| s.split('"').next()) + .map(|token| format!("/?__cf_chl_rt_tk={token}")) + }) + } + + fn absolute_site_url(&self, path_or_url: &str) -> String { + if path_or_url.starts_with("http://") || path_or_url.starts_with("https://") { + path_or_url.to_string() + } else { + format!( + "{}/{}", + self.url.trim_end_matches('/'), + path_or_url.trim_start_matches('/') + ) + } + } + + async fn fetch_html( + &self, + requester: &mut Requester, + request_url: &str, + referer: Option<&str>, + sec_fetch_site: &'static str, + ) -> Result { + let headers = self.headers_with_cookies(requester, request_url, referer, sec_fetch_site); + let response = requester + .get_raw_with_headers(request_url, headers.clone()) + .await + .map_err(Error::from)?; + let status = response.status(); + let body = response.text().await.map_err(Error::from)?; + + if status.is_success() || status.as_u16() == 404 { + return Ok(body); + } + + if status.as_u16() == 403 && Self::is_cloudflare_challenge(&body) { + if let Some(challenge_path) = Self::extract_challenge_path(&body) { + let challenge_url = self.absolute_site_url(&challenge_path); + let challenge_headers = self.headers_with_cookies( + requester, + &challenge_url, + Some(request_url), + "same-origin", + ); + let _ = requester + .get_raw_with_headers(&challenge_url, challenge_headers) + .await; + } + } + + let retry_headers = + self.headers_with_cookies(requester, request_url, referer, sec_fetch_site); + requester + .get_with_headers(request_url, retry_headers, Some(Version::HTTP_11)) + .await + .map_err(|e| Error::from(format!("{e}"))) + } + async fn warm_root_session(&self, requester: &mut Requester) { let root_url = self.root_referer(); - let _ = requester - .get_with_headers(&root_url, self.root_headers(), Some(Version::HTTP_11)) + let _ = self + .fetch_html(requester, &root_url, None, "none") .await; } @@ -276,7 +454,7 @@ impl PimpbunnyProvider { let _ = requester .get_with_headers( &root_url, - Self::html_headers_with_referer(&root_url), + Self::navigation_headers(None, "none"), Some(Version::HTTP_11), ) .await; @@ -288,7 +466,7 @@ impl PimpbunnyProvider { let request_url = format!("{base}/onlyfans-models/?models_per_page=20"); let headers = { let root_url = format!("{}/", base.trim_end_matches('/')); - let mut headers = Self::html_headers_with_referer(&root_url); + let mut headers = Self::navigation_headers(Some(&root_url), "same-origin"); if let Some(cookie) = requester.cookie_header_for_url(&request_url) { headers.push(("Cookie".to_string(), cookie)); } @@ -343,7 +521,7 @@ impl PimpbunnyProvider { let request_url = format!("{base}/categories/?items_per_page=120"); let headers = { let root_url = format!("{}/", base.trim_end_matches('/')); - let mut headers = Self::html_headers_with_referer(&root_url); + let mut headers = Self::navigation_headers(Some(&root_url), "same-origin"); if let Some(cookie) = requester.cookie_header_for_url(&request_url) { headers.push(("Cookie".to_string(), cookie)); } @@ -393,15 +571,7 @@ impl PimpbunnyProvider { sort: &str, options: ServerOptions, ) -> Result> { - let sort_string = match sort { - "best rated" => "&sort_by=rating", - "most viewed" => "&sort_by=video_viewed", - _ => "&sort_by=post_date", - }; - let video_url = format!( - "{}/videos/{}/?videos_per_page=32{}", - self.url, page, sort_string - ); + let video_url = self.build_browse_url(page, sort); let old_items = match cache.get(&video_url) { Some((time, items)) => { if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { @@ -417,9 +587,14 @@ impl PimpbunnyProvider { let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); self.warm_root_session(&mut requester).await; - let headers = self.headers_with_cookies(&requester, &video_url, &self.root_referer()); - let text = match requester - .get_with_headers(&video_url, headers, Some(Version::HTTP_11)) + let referer = self.page_family_referer(&video_url); + let text = match self + .fetch_html( + &mut requester, + &video_url, + Some(&referer), + "same-origin", + ) .await { Ok(text) => text, @@ -451,27 +626,17 @@ impl PimpbunnyProvider { options: ServerOptions, ) -> Result> { let search_string = query.trim().to_string(); - - let mut video_url = format!( - "{}/search/{}/?mode=async&function=get_block&block_id=list_videos_videos_list_search_result&videos_per_page=32&from_videos={}", - self.url, - search_string.replace(" ", "-"), - page - ); - - let sort_string = match options.sort.as_deref().unwrap_or("") { - "best rated" => "&sort_by=rating", - "most viewed" => "&sort_by=video_viewed", - _ => "&sort_by=post_date", - }; + let sort = options.sort.as_deref().unwrap_or(""); + let mut video_url = self.build_search_url(&search_string, page, sort); if let Ok(stars) = self.stars.read() { if let Some(star) = stars .iter() .find(|s| s.title.to_ascii_lowercase() == search_string.to_ascii_lowercase()) { - video_url = format!( - "{}/onlyfans-models/{}/{}/?videos_per_page=20{}", - self.url, star.id, page, sort_string + video_url = self.build_common_archive_url( + &format!("/onlyfans-models/{}/", star.id), + page, + sort, ); } } else { @@ -486,10 +651,8 @@ impl PimpbunnyProvider { .iter() .find(|c| c.title.to_ascii_lowercase() == search_string.to_ascii_lowercase()) { - video_url = format!( - "{}/categories/{}/{}/?videos_per_page=20{}", - self.url, cat.id, page, sort_string - ); + video_url = + self.build_common_archive_url(&format!("/categories/{}/", cat.id), page, sort); } } else { crate::providers::report_provider_error_background( @@ -516,10 +679,14 @@ impl PimpbunnyProvider { let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); self.warm_root_session(&mut requester).await; - println!("Fetching URL: {}", video_url); - let headers = self.headers_with_cookies(&requester, &video_url, &self.root_referer()); - let text = match requester - .get_with_headers(&video_url, headers, Some(Version::HTTP_2)) + let referer = self.page_family_referer(&video_url); + let text = match self + .fetch_html( + &mut requester, + &video_url, + Some(&referer), + "same-origin", + ) .await { Ok(text) => text, @@ -710,9 +877,17 @@ mod tests { use crate::videos::ServerOptions; use std::sync::{Arc, RwLock}; + fn test_provider() -> PimpbunnyProvider { + PimpbunnyProvider { + url: "https://pimpbunny.com".to_string(), + stars: Arc::new(RwLock::new(vec![])), + categories: Arc::new(RwLock::new(vec![])), + } + } + #[test] fn rewrites_allowed_thumbs_to_proxy_urls() { - let provider = PimpbunnyProvider::new(); + let provider = test_provider(); let options = ServerOptions { featured: None, category: None, @@ -742,7 +917,7 @@ mod tests { #[test] fn rewrites_video_pages_to_redirect_proxy() { - let provider = PimpbunnyProvider::new(); + let provider = test_provider(); let options = ServerOptions { featured: None, category: None, @@ -772,11 +947,7 @@ mod tests { #[test] fn parses_listing_without_detail_requests() { - let provider = PimpbunnyProvider { - url: "https://pimpbunny.com".to_string(), - stars: Arc::new(RwLock::new(vec![])), - categories: Arc::new(RwLock::new(vec![])), - }; + let provider = test_provider(); let options = ServerOptions { featured: None, category: None, @@ -820,4 +991,91 @@ mod tests { assert_eq!(items[0].views, Some(1200)); assert_eq!(items[0].formats.as_ref().map(|f| f.len()), Some(1)); } + + #[test] + fn extracts_cloudflare_challenge_path() { + let html = r#" + + "#; + + assert!(PimpbunnyProvider::is_cloudflare_challenge(html)); + assert_eq!( + PimpbunnyProvider::extract_challenge_path(html).as_deref(), + Some( + "/?mode=async&function=get_block&block_id=videos_videos_list&videos_per_page=8&sort_by=post_date&from=1&__cf_chl_tk=test-token" + ) + ); + } + + #[test] + fn builds_async_browse_url_instead_of_numbered_videos_path() { + let provider = test_provider(); + assert_eq!( + provider.build_browse_url(1, "most recent"), + "https://pimpbunny.com/videos/?sort_by=post_date" + ); + assert_eq!( + provider.build_browse_url(2, "most recent"), + "https://pimpbunny.com/videos/2/?sort_by=post_date" + ); + } + + #[test] + fn builds_search_url_with_query_and_pagination() { + let provider = test_provider(); + assert_eq!( + provider.build_search_url("adriana chechik", 1, "most viewed"), + "https://pimpbunny.com/search/adriana-chechik/?sort_by=video_viewed" + ); + assert_eq!( + provider.build_search_url("adriana chechik", 3, "most viewed"), + "https://pimpbunny.com/search/adriana-chechik/3/?sort_by=video_viewed" + ); + } + + #[test] + fn builds_common_archive_url_with_async_block() { + let provider = test_provider(); + assert_eq!( + provider.build_common_archive_url("/categories/amateur/", 1, "best rated"), + "https://pimpbunny.com/categories/amateur/?sort_by=rating" + ); + assert_eq!( + provider.build_common_archive_url("/categories/amateur/", 4, "best rated"), + "https://pimpbunny.com/categories/amateur/4/?sort_by=rating" + ); + } + + #[test] + fn derives_page_family_referer() { + let provider = test_provider(); + assert_eq!( + provider.page_family_referer("https://pimpbunny.com/videos/2/?sort_by=post_date"), + "https://pimpbunny.com/videos/" + ); + assert_eq!( + provider.page_family_referer( + "https://pimpbunny.com/categories/blowjob/2/?sort_by=post_date" + ), + "https://pimpbunny.com/categories/blowjob/" + ); + assert_eq!( + provider.page_family_referer( + "https://pimpbunny.com/search/adriana-chechik/3/?sort_by=video_viewed" + ), + "https://pimpbunny.com/search/adriana-chechik/" + ); + assert_eq!( + provider.page_family_referer( + "https://pimpbunny.com/onlyfans-models/momoitenshi/3/?sort_by=post_date" + ), + "https://pimpbunny.com/onlyfans-models/momoitenshi/" + ); + } } diff --git a/src/status.rs b/src/status.rs index a7d0abc..acc1a03 100644 --- a/src/status.rs +++ b/src/status.rs @@ -118,22 +118,9 @@ impl Status { .to_string(), } } - #[allow(dead_code)] - pub fn add_notice(&mut self, notice: Notice) { - self.notices.push(notice); - } - #[allow(dead_code)] pub fn add_channel(&mut self, channel: Channel) { self.channels.push(channel); } - #[allow(dead_code)] - pub fn add_option(&mut self, option: Options) { - self.options.push(option); - } - #[allow(dead_code)] - pub fn add_category(&mut self, category: String) { - self.categories.push(category); - } } #[derive(serde::Serialize)] @@ -154,6 +141,8 @@ pub struct ChannelView { #[serde(skip_serializing_if = "Option::is_none")] pub tags: Option>, #[serde(skip_serializing_if = "Option::is_none")] + pub ytdlpCommand: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub cacheDuration: Option, } diff --git a/src/util/flow_debug.rs b/src/util/flow_debug.rs index 03c4fb3..7a67b34 100644 --- a/src/util/flow_debug.rs +++ b/src/util/flow_debug.rs @@ -1,4 +1,5 @@ use std::sync::atomic::{AtomicU64, Ordering}; +#[cfg(feature = "debug")] use std::time::{SystemTime, UNIX_EPOCH}; static NEXT_TRACE_ID: AtomicU64 = AtomicU64::new(1); @@ -17,9 +18,6 @@ pub fn emit(module: &str, line: u32, message: String) { eprintln!("[debug][{millis}][{module}:{line}] {message}"); } -#[cfg(not(feature = "debug"))] -pub fn emit(_module: &str, _line: u32, _message: String) {} - pub fn preview(value: &str, limit: usize) -> String { if value.len() <= limit { return value.to_string(); diff --git a/src/util/mod.rs b/src/util/mod.rs index 03ad843..b70c9db 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -27,6 +27,7 @@ pub fn parse_abbreviated_number(s: &str) -> Option { .map(|n| (n * multiplier) as u32) } +#[cfg(not(hottub_single_provider))] pub fn interleave(lists: &[Vec]) -> Vec { let mut result = Vec::new(); diff --git a/src/util/requester.rs b/src/util/requester.rs index d986b56..39aadb8 100644 --- a/src/util/requester.rs +++ b/src/util/requester.rs @@ -1,14 +1,14 @@ use serde::Serialize; use std::env; use std::fmt; -use std::sync::Arc; +use std::sync::{Arc, OnceLock}; use wreq::Client; use wreq::Proxy; use wreq::Response; use wreq::Uri; use wreq::Version; use wreq::cookie::{CookieStore, Cookies, Jar}; -use wreq::header::{HeaderMap, HeaderValue, USER_AGENT}; +use wreq::header::{HeaderMap, HeaderValue, SET_COOKIE, USER_AGENT}; use wreq::multipart::Form; use wreq::redirect::Policy; use wreq_util::Emulation; @@ -45,6 +45,79 @@ impl fmt::Debug for Requester { } impl Requester { + fn shared_cookie_jar() -> Arc { + static SHARED_COOKIE_JAR: OnceLock> = OnceLock::new(); + SHARED_COOKIE_JAR + .get_or_init(|| Arc::new(Jar::default())) + .clone() + } + + fn origin_url_for_cookie_scope(url: &str) -> Option { + let parsed = url::Url::parse(url).ok()?; + let host = parsed.host_str()?; + let scheme = parsed.scheme(); + url::Url::parse(&format!("{scheme}://{host}/")).ok() + } + + fn store_response_cookies(&self, url: &str, response: &Response) { + let Some(origin) = Self::origin_url_for_cookie_scope(url) else { + return; + }; + + for value in response.headers().get_all(SET_COOKIE).iter() { + if let Ok(cookie) = value.to_str() { + self.cookie_jar.add_cookie_str(cookie, &origin.to_string()); + } + } + } + + fn store_flaresolverr_cookies( + &mut self, + request_url: &str, + cookies: &[crate::util::flaresolverr::FlaresolverrCookie], + ) { + let fallback_origin = Self::origin_url_for_cookie_scope(request_url); + + for cookie in cookies { + let origin = if !cookie.domain.is_empty() { + let scheme = fallback_origin + .as_ref() + .map(|url| url.scheme()) + .unwrap_or("https"); + let host = cookie.domain.trim_start_matches('.'); + url::Url::parse(&format!("{scheme}://{host}/")) + .ok() + .or_else(|| fallback_origin.clone()) + } else { + fallback_origin.clone() + }; + + let Some(origin) = origin else { + continue; + }; + + let mut cookie_string = + format!("{}={}; Path={}", cookie.name, cookie.value, cookie.path); + if !cookie.domain.is_empty() { + cookie_string.push_str(&format!("; Domain={}", cookie.domain)); + } + if cookie.secure { + cookie_string.push_str("; Secure"); + } + if cookie.httpOnly { + cookie_string.push_str("; HttpOnly"); + } + if let Some(same_site) = cookie.sameSite.as_deref() { + if !same_site.is_empty() { + cookie_string.push_str(&format!("; SameSite={same_site}")); + } + } + + self.cookie_jar + .add_cookie_str(&cookie_string, &origin.to_string()); + } + } + fn debug_cookie_preview_from_owned_headers( &self, url: &str, @@ -62,6 +135,7 @@ impl Requester { .unwrap_or_else(|| "none".to_string()) } + #[cfg(any(not(hottub_single_provider), hottub_provider = "hypnotube"))] fn debug_cookie_preview_from_borrowed_headers( &self, url: &str, @@ -98,7 +172,7 @@ impl Requester { } pub fn new() -> Self { - let cookie_jar = Arc::new(Jar::default()); + let cookie_jar = Self::shared_cookie_jar(); let client = Self::build_client(cookie_jar.clone(), None); let requester = Requester { @@ -122,14 +196,11 @@ impl Requester { self.proxy = proxy; } - pub fn proxy_enabled(&self) -> bool { - self.proxy - } - pub fn set_debug_trace_id(&mut self, debug_trace_id: Option) { self.debug_trace_id = debug_trace_id; } + #[cfg(feature = "debug")] pub fn debug_trace_id(&self) -> Option<&str> { self.debug_trace_id.as_deref() } @@ -176,7 +247,9 @@ impl Requester { } } - request.send().await + let response = request.send().await?; + self.store_response_cookies(url, &response); + Ok(response) } pub async fn get_raw_with_headers( @@ -209,7 +282,9 @@ impl Requester { for (key, value) in headers.iter() { request = request.header(key, value); } - request.send().await + let response = request.send().await?; + self.store_response_cookies(url, &response); + Ok(response) } pub async fn post_json( @@ -246,9 +321,12 @@ impl Requester { } } - request.send().await + let response = request.send().await?; + self.store_response_cookies(url, &response); + Ok(response) } + #[cfg(any(not(hottub_single_provider), hottub_provider = "hypnotube"))] pub async fn post( &mut self, url: &str, @@ -285,7 +363,9 @@ impl Requester { } } - request.send().await + let response = request.send().await?; + self.store_response_cookies(url, &response); + Ok(response) } pub async fn post_multipart( @@ -325,7 +405,9 @@ impl Requester { } } - request.send().await + let response = request.send().await?; + self.store_response_cookies(url, &response); + Ok(response) } pub async fn get( @@ -370,6 +452,7 @@ impl Requester { } } let response = request.send().await?; + self.store_response_cookies(url, &response); crate::flow_debug!( "trace={} requester direct response url={} status={}", self.debug_trace_id().unwrap_or("none"), @@ -424,17 +507,9 @@ impl Requester { .map_err(|e| -> AnyErr { format!("Failed to solve FlareSolverr: {e}").into() })?; // Rebuild client and apply UA/cookies from FlareSolverr - let cookie_origin = url.split('/').take(3).collect::>().join("/"); - let useragent = res.solution.userAgent; self.user_agent = Some(useragent); - - if url::Url::parse(&cookie_origin).is_ok() { - for cookie in res.solution.cookies { - self.cookie_jar - .add_cookie_str(&format!("{}={}", cookie.name, cookie.value), &cookie_origin); - } - } + self.store_flaresolverr_cookies(url, &res.solution.cookies); self.client = Self::build_client(self.cookie_jar.clone(), self.user_agent.as_deref()); crate::flow_debug!( @@ -457,6 +532,7 @@ impl Requester { } let response = request.send().await?; + self.store_response_cookies(url, &response); crate::flow_debug!( "trace={} requester retry response url={} status={}", self.debug_trace_id().unwrap_or("none"), @@ -476,3 +552,26 @@ impl Requester { Ok(res.solution.response) } } + +#[cfg(test)] +mod tests { + use super::Requester; + + #[test] + fn new_requesters_share_cookie_jar() { + let a = Requester::new(); + let b = Requester::new(); + let origin = "https://shared-cookie-requester-test.invalid/"; + + a.cookie_jar.add_cookie_str( + "shared_cookie=1; Path=/; SameSite=Lax", + origin, + ); + + let cookie_header = b + .cookie_header_for_url("https://shared-cookie-requester-test.invalid/path") + .unwrap_or_default(); + + assert!(cookie_header.contains("shared_cookie=1")); + } +} diff --git a/src/videos.rs b/src/videos.rs index e385e3c..3a4c041 100644 --- a/src/videos.rs +++ b/src/videos.rs @@ -114,7 +114,6 @@ pub struct VideoItem { #[serde(skip_serializing_if = "Option::is_none")] pub aspectRatio: Option, } -#[allow(dead_code)] impl VideoItem { pub fn new( id: String, @@ -145,9 +144,11 @@ impl VideoItem { aspectRatio: None, } } + #[cfg(any(not(hottub_single_provider), hottub_provider = "hentaihaven"))] pub fn from(s: String) -> Result { serde_json::from_str::(&s) } + #[cfg(any(not(hottub_single_provider), hottub_provider = "hanime"))] pub fn tags(mut self, tags: Vec) -> Self { if tags.is_empty() { return self; @@ -155,30 +156,113 @@ impl VideoItem { self.tags = Some(tags); self } + #[cfg(any( + not(hottub_single_provider), + hottub_provider = "hanime", + hottub_provider = "heavyfetish", + hottub_provider = "porndish", + hottub_provider = "shooshtime", + hottub_provider = "spankbang", + hottub_provider = "chaturbate", + hottub_provider = "porn4fans", + hottub_provider = "xfree", + hottub_provider = "pornhub", + ))] pub fn uploader(mut self, uploader: String) -> Self { self.uploader = Some(uploader); self } + #[cfg(any( + not(hottub_single_provider), + hottub_provider = "heavyfetish", + hottub_provider = "porndish", + hottub_provider = "shooshtime", + hottub_provider = "spankbang", + hottub_provider = "chaturbate", + ))] pub fn uploader_url(mut self, uploader_url: String) -> Self { self.uploaderUrl = Some(uploader_url); self } - pub fn verified(mut self, verified: bool) -> Self { - self.verified = Some(verified); - self - } + #[cfg(any( + not(hottub_single_provider), + hottub_provider = "beeg", + hottub_provider = "chaturbate", + hottub_provider = "freepornvideosxxx", + hottub_provider = "hanime", + hottub_provider = "heavyfetish", + hottub_provider = "hentaihaven", + hottub_provider = "hypnotube", + hottub_provider = "javtiful", + hottub_provider = "noodlemagazine", + hottub_provider = "okxxx", + hottub_provider = "omgxxx", + hottub_provider = "perfectgirls", + hottub_provider = "pimpbunny", + hottub_provider = "pmvhaven", + hottub_provider = "porn00", + hottub_provider = "porn4fans", + hottub_provider = "porndish", + hottub_provider = "pornhat", + hottub_provider = "pornhub", + hottub_provider = "redtube", + hottub_provider = "rule34gen", + hottub_provider = "rule34video", + hottub_provider = "shooshtime", + hottub_provider = "spankbang", + hottub_provider = "sxyprn", + hottub_provider = "tnaflix", + hottub_provider = "tokyomotion", + hottub_provider = "viralxxxporn", + hottub_provider = "xfree", + hottub_provider = "xxthots", + hottub_provider = "yesporn", + hottub_provider = "youjizz", + ))] pub fn views(mut self, views: u32) -> Self { self.views = Some(views); self } + #[cfg(any( + not(hottub_single_provider), + hottub_provider = "beeg", + hottub_provider = "hanime", + hottub_provider = "heavyfetish", + hottub_provider = "hsex", + hottub_provider = "porn4fans", + hottub_provider = "shooshtime", + hottub_provider = "spankbang", + hottub_provider = "tokyomotion", + hottub_provider = "vrporn", + hottub_provider = "yesporn", + ))] pub fn rating(mut self, rating: f32) -> Self { self.rating = Some(rating); self } + #[cfg(any( + not(hottub_single_provider), + hottub_provider = "porndish", + hottub_provider = "shooshtime", + hottub_provider = "heavyfetish", + hottub_provider = "xfree", + ))] pub fn uploaded_at(mut self, uploaded_at: u64) -> Self { self.uploadedAt = Some(uploaded_at); self } + #[cfg(any( + not(hottub_single_provider), + hottub_provider = "hanime", + hottub_provider = "heavyfetish", + hottub_provider = "hentaihaven", + hottub_provider = "hqporner", + hottub_provider = "javtiful", + hottub_provider = "noodlemagazine", + hottub_provider = "pimpbunny", + hottub_provider = "pmvhaven", + hottub_provider = "shooshtime", + ))] pub fn formats(mut self, formats: Vec) -> Self { if formats.is_empty() { return self; @@ -186,27 +270,48 @@ impl VideoItem { self.formats = Some(formats); self } - pub fn add_format(mut self, format: VideoFormat) { - if let Some(formats) = self.formats.as_mut() { - formats.push(format); - } else { - self.formats = Some(vec![format]); - } - } - pub fn embed(mut self, embed: VideoEmbed) -> Self { - self.embed = Some(embed); - self - } + #[cfg(any( + not(hottub_single_provider), + hottub_provider = "freepornvideosxxx", + hottub_provider = "heavyfetish", + hottub_provider = "homoxxx", + hottub_provider = "javtiful", + hottub_provider = "missav", + hottub_provider = "okxxx", + hottub_provider = "omgxxx", + hottub_provider = "perfectgirls", + hottub_provider = "pimpbunny", + hottub_provider = "pmvhaven", + hottub_provider = "pornhat", + hottub_provider = "redtube", + hottub_provider = "rule34gen", + hottub_provider = "shooshtime", + hottub_provider = "spankbang", + hottub_provider = "sxyprn", + hottub_provider = "tnaflix", + hottub_provider = "xfree", + hottub_provider = "xxdbx", + hottub_provider = "yesporn", + ))] pub fn preview(mut self, preview: String) -> Self { self.preview = Some(preview); self } + #[cfg(any( + not(hottub_single_provider), + hottub_provider = "hentaihaven", + hottub_provider = "hanime", + hottub_provider = "heavyfetish", + hottub_provider = "paradisehill", + hottub_provider = "xfree", + ))] pub fn aspect_ratio(mut self, aspect_ratio: f32) -> Self { self.aspectRatio = Some(aspect_ratio); self } + #[cfg(any(not(hottub_single_provider), hottub_provider = "chaturbate"))] pub fn is_live(mut self, is_live: bool) -> Self { self.isLive = is_live; self @@ -294,6 +399,13 @@ impl VideoFormat { http_headers: None, } } + #[cfg(any( + not(hottub_single_provider), + hottub_provider = "vrporn", + hottub_provider = "perverzija", + hottub_provider = "porndish", + hottub_provider = "spankbang", + ))] pub fn add_http_header(&mut self, key: String, value: String) { if self.http_headers.is_none() { self.http_headers = Some(HashMap::new()); @@ -302,6 +414,14 @@ impl VideoFormat { headers.insert(key, value); } } + #[cfg(any( + not(hottub_single_provider), + hottub_provider = "hentaihaven", + hottub_provider = "noodlemagazine", + hottub_provider = "shooshtime", + hottub_provider = "heavyfetish", + hottub_provider = "hsex", + ))] pub fn http_header(&mut self, key: String, value: String) -> Self { if self.http_headers.is_none() { self.http_headers = Some(HashMap::new());