diff --git a/Cargo.toml b/Cargo.toml index 2846f59..7e035b7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,10 @@ name = "hottub" version = "0.1.0" edition = "2024" +build = "build.rs" + +[features] +debug = [] [dependencies] cute = "0.3.0" diff --git a/build.rs b/build.rs new file mode 100644 index 0000000..7f10a2b --- /dev/null +++ b/build.rs @@ -0,0 +1,328 @@ +use std::env; +use std::fs; +use std::path::PathBuf; + +struct ProviderDef { + id: &'static str, + module: &'static str, + ty: &'static str, +} + +const PROVIDERS: &[ProviderDef] = &[ + ProviderDef { + id: "all", + module: "all", + ty: "AllProvider", + }, + ProviderDef { + id: "perverzija", + module: "perverzija", + ty: "PerverzijaProvider", + }, + ProviderDef { + id: "hanime", + module: "hanime", + ty: "HanimeProvider", + }, + ProviderDef { + id: "pornhub", + module: "pornhub", + ty: "PornhubProvider", + }, + ProviderDef { + id: "spankbang", + module: "spankbang", + ty: "SpankbangProvider", + }, + ProviderDef { + id: "rule34video", + module: "rule34video", + ty: "Rule34videoProvider", + }, + ProviderDef { + id: "redtube", + module: "redtube", + ty: "RedtubeProvider", + }, + ProviderDef { + id: "okporn", + module: "okporn", + ty: "OkpornProvider", + }, + ProviderDef { + id: "pornhat", + module: "pornhat", + ty: "PornhatProvider", + }, + ProviderDef { + id: "perfectgirls", + module: "perfectgirls", + ty: "PerfectgirlsProvider", + }, + ProviderDef { + id: "okxxx", + module: "okxxx", + ty: "OkxxxProvider", + }, + ProviderDef { + id: "homoxxx", + module: "homoxxx", + ty: "HomoxxxProvider", + }, + ProviderDef { + id: "missav", + module: "missav", + ty: "MissavProvider", + }, + ProviderDef { + id: "xxthots", + module: "xxthots", + ty: "XxthotsProvider", + }, + ProviderDef { + id: "yesporn", + module: "yesporn", + ty: "YespornProvider", + }, + ProviderDef { + id: "sxyprn", + module: "sxyprn", + ty: "SxyprnProvider", + }, + ProviderDef { + id: "porn00", + module: "porn00", + ty: "Porn00Provider", + }, + ProviderDef { + id: "youjizz", + module: "youjizz", + ty: "YoujizzProvider", + }, + ProviderDef { + id: "paradisehill", + module: "paradisehill", + ty: "ParadisehillProvider", + }, + ProviderDef { + id: "porn4fans", + module: "porn4fans", + ty: "Porn4fansProvider", + }, + ProviderDef { + id: "porndish", + module: "porndish", + ty: "PorndishProvider", + }, + ProviderDef { + id: "shooshtime", + module: "shooshtime", + ty: "ShooshtimeProvider", + }, + ProviderDef { + id: "pornzog", + module: "pornzog", + ty: "PornzogProvider", + }, + ProviderDef { + id: "omgxxx", + module: "omgxxx", + ty: "OmgxxxProvider", + }, + ProviderDef { + id: "beeg", + module: "beeg", + ty: "BeegProvider", + }, + ProviderDef { + id: "tnaflix", + module: "tnaflix", + ty: "TnaflixProvider", + }, + ProviderDef { + id: "tokyomotion", + module: "tokyomotion", + ty: "TokyomotionProvider", + }, + ProviderDef { + id: "viralxxxporn", + module: "viralxxxporn", + ty: "ViralxxxpornProvider", + }, + ProviderDef { + id: "vrporn", + module: "vrporn", + ty: "VrpornProvider", + }, + ProviderDef { + id: "rule34gen", + module: "rule34gen", + ty: "Rule34genProvider", + }, + ProviderDef { + id: "xxdbx", + module: "xxdbx", + ty: "XxdbxProvider", + }, + ProviderDef { + id: "xfree", + module: "xfree", + ty: "XfreeProvider", + }, + ProviderDef { + id: "hqporner", + module: "hqporner", + ty: "HqpornerProvider", + }, + ProviderDef { + id: "pmvhaven", + module: "pmvhaven", + ty: "PmvhavenProvider", + }, + ProviderDef { + id: "noodlemagazine", + module: "noodlemagazine", + ty: "NoodlemagazineProvider", + }, + ProviderDef { + id: "pimpbunny", + module: "pimpbunny", + ty: "PimpbunnyProvider", + }, + ProviderDef { + id: "javtiful", + module: "javtiful", + ty: "JavtifulProvider", + }, + ProviderDef { + id: "hypnotube", + module: "hypnotube", + ty: "HypnotubeProvider", + }, + ProviderDef { + id: "freepornvideosxxx", + module: "freepornvideosxxx", + ty: "FreepornvideosxxxProvider", + }, + ProviderDef { + id: "heavyfetish", + module: "heavyfetish", + ty: "HeavyfetishProvider", + }, + ProviderDef { + id: "hsex", + module: "hsex", + ty: "HsexProvider", + }, + ProviderDef { + id: "hentaihaven", + module: "hentaihaven", + ty: "HentaihavenProvider", + }, + ProviderDef { + id: "chaturbate", + module: "chaturbate", + ty: "ChaturbateProvider", + }, +]; + +fn main() { + println!("cargo:rerun-if-changed=build.rs"); + println!("cargo:rerun-if-env-changed=HOT_TUB_PROVIDER"); + println!("cargo:rerun-if-env-changed=HOTTUB_PROVIDER"); + println!("cargo:rustc-check-cfg=cfg(hottub_single_provider)"); + + let selected = env::var("HOT_TUB_PROVIDER") + .or_else(|_| env::var("HOTTUB_PROVIDER")) + .ok() + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()); + + let providers = match selected.as_deref() { + Some(selected_id) => { + let provider = PROVIDERS + .iter() + .find(|provider| provider.id == selected_id) + .unwrap_or_else(|| { + panic!("Unknown provider `{selected_id}` from HOT_TUB_PROVIDER/HOTTUB_PROVIDER") + }); + println!("cargo:rustc-cfg=hottub_single_provider"); + vec![provider] + } + None => PROVIDERS.iter().collect(), + }; + + let out_dir = PathBuf::from(env::var("OUT_DIR").expect("OUT_DIR")); + let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR")); + + let modules = providers + .iter() + .map(|provider| { + let module_path = manifest_dir + .join("src/providers") + .join(format!("{}.rs", provider.module)); + format!( + "#[path = r#\"{}\"#]\npub mod {};", + module_path.display(), + provider.module + ) + }) + .collect::>() + .join("\n"); + fs::write(out_dir.join("provider_modules.rs"), format!("{modules}\n")) + .expect("write provider_modules.rs"); + + let registry = providers + .iter() + .map(|provider| { + format!( + "m.insert(\"{id}\", Arc::new({module}::{ty}::new()) as DynProvider);", + id = provider.id, + module = provider.module, + ty = provider.ty + ) + }) + .collect::>() + .join("\n"); + fs::write( + out_dir.join("provider_registry.rs"), + format!("{{\n{registry}\n}}\n"), + ) + .expect("write provider_registry.rs"); + + let metadata_arms = providers + .iter() + .map(|provider| { + if provider.id == "all" { + format!( + "\"all\" | \"hottub\" => Some({module}::CHANNEL_METADATA),", + module = provider.module + ) + } else { + format!( + "\"{id}\" => Some({module}::CHANNEL_METADATA),", + id = provider.id, + module = provider.module + ) + } + }) + .collect::>() + .join("\n"); + fs::write( + out_dir.join("provider_metadata_fn.rs"), + format!("match id {{\n{metadata_arms}\n_ => None,\n}}\n"), + ) + .expect("write provider_metadata_fn.rs"); + + let selection = match selected.as_deref() { + Some(selected_id) => format!( + "pub const COMPILE_TIME_SELECTED_PROVIDER: Option<&str> = Some(\"{selected_id}\");" + ), + None => "pub const COMPILE_TIME_SELECTED_PROVIDER: Option<&str> = None;".to_string(), + }; + fs::write( + out_dir.join("provider_selection.rs"), + format!("{selection}\n"), + ) + .expect("write provider_selection.rs"); +} diff --git a/src/api.rs b/src/api.rs index 1ee9423..7ce10b6 100644 --- a/src/api.rs +++ b/src/api.rs @@ -1,6 +1,6 @@ use crate::providers::{ - ALL_PROVIDERS, DynProvider, build_status_response, panic_payload_to_string, report_provider_error, - run_provider_guarded, + ALL_PROVIDERS, DynProvider, build_status_response, panic_payload_to_string, + report_provider_error, resolve_provider_for_build, run_provider_guarded, }; use crate::util::cache::VideoCache; use crate::util::discord::send_discord_error_report; @@ -146,6 +146,7 @@ pub fn config(cfg: &mut web::ServiceConfig) { } async fn status(req: HttpRequest) -> Result { + let trace_id = crate::util::flow_debug::next_trace_id("status"); let clientversion: ClientVersion = match req.headers().get("User-Agent") { Some(v) => match v.to_str() { Ok(useragent) => ClientVersion::parse(useragent) @@ -159,6 +160,12 @@ async fn status(req: HttpRequest) -> Result { "Received status request with client version: {:?}", clientversion ); + crate::flow_debug!( + "trace={} status request host={} client={:?}", + trace_id, + req.connection_info().host(), + &clientversion + ); let host = req .headers() @@ -168,8 +175,14 @@ async fn status(req: HttpRequest) -> Result { .to_string(); let public_url_base = format!("{}://{}", req.connection_info().scheme(), host); let mut status = Status::new(); + let mut channel_count = 0usize; for (provider_name, provider) in ALL_PROVIDERS.iter() { + crate::flow_debug!( + "trace={} status inspecting provider={}", + trace_id, + provider_name + ); let channel_result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { provider.get_channel(clientversion.clone()) })); @@ -178,17 +191,37 @@ async fn status(req: HttpRequest) -> Result { if channel.favicon.starts_with('/') { channel.favicon = format!("{}{}", public_url_base, channel.favicon); } + channel_count += 1; + crate::flow_debug!( + "trace={} status added channel id={} provider={}", + trace_id, + channel.id.as_str(), + provider_name + ); status.add_channel(channel) } Ok(None) => {} Err(payload) => { let panic_msg = panic_payload_to_string(payload); + crate::flow_debug!( + "trace={} status provider panic provider={} panic={}", + trace_id, + provider_name, + &panic_msg + ); report_provider_error(provider_name, "status.get_channel", &panic_msg).await; } } } status.iconUrl = format!("{}/favicon.ico", public_url_base).to_string(); - Ok(web::HttpResponse::Ok().json(&build_status_response(status))) + let response = build_status_response(status); + crate::flow_debug!( + "trace={} status response channels={} groups={}", + trace_id, + channel_count, + response.channelGroups.len() + ); + Ok(web::HttpResponse::Ok().json(&response)) } async fn videos_post( @@ -198,6 +231,7 @@ async fn videos_post( requester: web::types::State, req: HttpRequest, ) -> Result { + let trace_id = crate::util::flow_debug::next_trace_id("videos"); let clientversion: ClientVersion = match req.headers().get("User-Agent") { Some(v) => match v.to_str() { Ok(useragent) => ClientVersion::parse(useragent) @@ -235,11 +269,12 @@ async fn videos_post( }, items: vec![], }; - let channel: String = video_request + let requested_channel: String = video_request .channel .as_deref() .unwrap_or("all") .to_string(); + let channel = resolve_provider_for_build(requested_channel.as_str()).to_string(); let sort: String = video_request.sort.as_deref().unwrap_or("date").to_string(); let (query, literal_query) = normalize_query(video_request.query.as_deref()); let page: u8 = video_request @@ -294,6 +329,22 @@ async fn videos_post( req.connection_info().scheme(), req.connection_info().host() ); + crate::flow_debug!( + "trace={} videos request requested_channel={} resolved_channel={} sort={} query={:?} page={} per_page={} filter={} category={} sites={} client={:?}", + trace_id, + &requested_channel, + &channel, + &sort, + &query, + page, + perPage, + &filter, + &category, + &sites, + &clientversion + ); + let mut requester = requester; + requester.set_debug_trace_id(Some(trace_id.clone())); let options = ServerOptions { featured: Some(featured), category: Some(category), @@ -309,6 +360,12 @@ async fn videos_post( sort: Some(sort.clone()), sexuality: Some(sexuality), }; + crate::flow_debug!( + "trace={} videos provider dispatch provider={} literal_query={:?}", + trace_id, + &channel, + &literal_query + ); let mut video_items = run_provider_guarded( &channel, "videos_post.get_videos", @@ -323,6 +380,11 @@ async fn videos_post( ), ) .await; + crate::flow_debug!( + "trace={} videos provider returned count={}", + trace_id, + video_items.len() + ); // There is a bug in Hottub38 that makes the client error for a 403-url even though formats work fine if clientversion == ClientVersion::new(38, 0, "Hot%20Tub".to_string()) { @@ -345,7 +407,14 @@ async fn videos_post( } if let Some(literal_query) = literal_query.as_deref() { + let before = video_items.len(); video_items.retain(|video| video_matches_literal_query(video, literal_query)); + crate::flow_debug!( + "trace={} videos literal filter kept={} removed={}", + trace_id, + video_items.len(), + before.saturating_sub(video_items.len()) + ); } videos.items = video_items.clone(); @@ -365,7 +434,14 @@ async fn videos_post( let per_page_clone = perPage.to_string(); let options_clone = options.clone(); let channel_clone = channel.clone(); + let prefetch_trace_id = trace_id.clone(); task::spawn_local(async move { + crate::flow_debug!( + "trace={} videos prefetch spawn next_page={} provider={}", + prefetch_trace_id, + next_page, + &channel_clone + ); // if let AnyProvider::Spankbang(_) = provider_clone { // // Spankbang has a delay for the next page // ntex::time::sleep(ntex::time::Seconds(80)).await; @@ -399,11 +475,23 @@ async fn videos_post( } } + crate::flow_debug!( + "trace={} videos response items={} has_next={}", + trace_id, + videos.items.len(), + videos.pageInfo.hasNextPage + ); Ok(web::HttpResponse::Ok().json(&videos)) } pub fn get_provider(channel: &str) -> Option { - ALL_PROVIDERS.get(channel).cloned() + let provider = ALL_PROVIDERS.get(channel).cloned(); + crate::flow_debug!( + "provider lookup channel={} found={}", + channel, + provider.is_some() + ); + provider } pub async fn test() -> Result { @@ -424,6 +512,7 @@ pub async fn test() -> Result { pub async fn proxies() -> Result { let proxies = all_proxies_snapshot().await.unwrap_or_default(); + crate::flow_debug!("proxies endpoint snapshot_count={}", proxies.len()); let mut by_protocol: std::collections::BTreeMap> = std::collections::BTreeMap::new(); for proxy in proxies { diff --git a/src/main.rs b/src/main.rs index 2c98c42..afcdaa9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -39,6 +39,11 @@ async fn main() -> std::io::Result<()> { } } env_logger::init(); // You need this to actually see logs + crate::flow_debug!( + "startup begin rust_log={} debug_compiled={}", + std::env::var("RUST_LOG").unwrap_or_else(|_| "unset".to_string()), + cfg!(feature = "debug") + ); // set up database connection pool let connspec = std::env::var("DATABASE_URL").expect("DATABASE_URL"); @@ -46,15 +51,25 @@ async fn main() -> std::io::Result<()> { let pool = r2d2::Pool::builder() .build(manager) .expect("Failed to create pool."); + crate::flow_debug!( + "database pool ready database_url={}", + crate::util::flow_debug::preview(&connspec, 96) + ); let mut requester = util::requester::Requester::new(); requester.set_proxy(env::var("PROXY").unwrap_or("0".to_string()) != "0".to_string()); + crate::flow_debug!( + "requester initialized proxy_enabled={}", + requester.proxy_enabled() + ); let cache: util::cache::VideoCache = crate::util::cache::VideoCache::new() .max_size(100_000) .to_owned(); + crate::flow_debug!("video cache initialized max_size=100000"); thread::spawn(move || { + crate::flow_debug!("provider init thread spawned"); // Create a tiny runtime just for these async tasks let rt = tokio::runtime::Builder::new_current_thread() .enable_all() @@ -62,10 +77,13 @@ async fn main() -> std::io::Result<()> { .expect("build tokio runtime"); rt.block_on(async move { + crate::flow_debug!("provider init begin"); providers::init_providers_now(); + crate::flow_debug!("provider init complete"); }); }); + crate::flow_debug!("http server binding addr=0.0.0.0:18080 workers=8"); web::HttpServer::new(move || { web::App::new() .state(pool.clone()) diff --git a/src/providers/mod.rs b/src/providers/mod.rs index 3594616..31da4d8 100644 --- a/src/providers/mod.rs +++ b/src/providers/mod.rs @@ -14,52 +14,8 @@ use crate::{ videos::{ServerOptions, VideoItem}, }; -pub mod all; -pub mod hanime; -pub mod homoxxx; -pub mod okporn; -pub mod okxxx; -pub mod perfectgirls; -pub mod perverzija; -pub mod pmvhaven; -pub mod pornhat; -pub mod pornhub; -pub mod redtube; -pub mod rule34video; -pub mod spankbang; -// pub mod hentaimoon; -pub mod beeg; -pub mod missav; -pub mod omgxxx; -pub mod paradisehill; -pub mod porn00; -pub mod porn4fans; -pub mod porndish; -pub mod pornzog; -pub mod shooshtime; -pub mod sxyprn; -pub mod tnaflix; -pub mod tokyomotion; -pub mod viralxxxporn; -pub mod vrporn; -pub mod xfree; -pub mod xxthots; -pub mod yesporn; -pub mod youjizz; -// pub mod pornxp; -pub mod chaturbate; -pub mod freepornvideosxxx; -pub mod heavyfetish; -pub mod hentaihaven; -pub mod hqporner; -pub mod hsex; -pub mod hypnotube; -pub mod javtiful; -pub mod noodlemagazine; -pub mod pimpbunny; -pub mod rule34gen; -pub mod xxdbx; -// pub mod tube8; +include!(concat!(env!("OUT_DIR"), "/provider_selection.rs")); +include!(concat!(env!("OUT_DIR"), "/provider_modules.rs")); // convenient alias pub type DynProvider = Arc; @@ -72,180 +28,30 @@ pub struct ProviderChannelMetadata { pub static ALL_PROVIDERS: Lazy> = Lazy::new(|| { let mut m = HashMap::default(); - m.insert("all", Arc::new(all::AllProvider::new()) as DynProvider); - m.insert( - "perverzija", - Arc::new(perverzija::PerverzijaProvider::new()) as DynProvider, - ); - m.insert( - "hanime", - Arc::new(hanime::HanimeProvider::new()) as DynProvider, - ); - m.insert( - "pornhub", - Arc::new(pornhub::PornhubProvider::new()) as DynProvider, - ); - m.insert( - "spankbang", - Arc::new(spankbang::SpankbangProvider::new()) as DynProvider, - ); - m.insert( - "rule34video", - Arc::new(rule34video::Rule34videoProvider::new()) as DynProvider, - ); - m.insert( - "redtube", - Arc::new(redtube::RedtubeProvider::new()) as DynProvider, - ); - m.insert( - "okporn", - Arc::new(okporn::OkpornProvider::new()) as DynProvider, - ); - m.insert( - "pornhat", - Arc::new(pornhat::PornhatProvider::new()) as DynProvider, - ); - m.insert( - "perfectgirls", - Arc::new(perfectgirls::PerfectgirlsProvider::new()) as DynProvider, - ); - m.insert( - "okxxx", - Arc::new(okxxx::OkxxxProvider::new()) as DynProvider, - ); - m.insert( - "homoxxx", - Arc::new(homoxxx::HomoxxxProvider::new()) as DynProvider, - ); - m.insert( - "missav", - Arc::new(missav::MissavProvider::new()) as DynProvider, - ); - m.insert( - "xxthots", - Arc::new(xxthots::XxthotsProvider::new()) as DynProvider, - ); - m.insert( - "yesporn", - Arc::new(yesporn::YespornProvider::new()) as DynProvider, - ); - m.insert( - "sxyprn", - Arc::new(sxyprn::SxyprnProvider::new()) as DynProvider, - ); - m.insert( - "porn00", - Arc::new(porn00::Porn00Provider::new()) as DynProvider, - ); - m.insert( - "youjizz", - Arc::new(youjizz::YoujizzProvider::new()) as DynProvider, - ); - m.insert( - "paradisehill", - Arc::new(paradisehill::ParadisehillProvider::new()) as DynProvider, - ); - m.insert( - "porn4fans", - Arc::new(porn4fans::Porn4fansProvider::new()) as DynProvider, - ); - m.insert( - "porndish", - Arc::new(porndish::PorndishProvider::new()) as DynProvider, - ); - m.insert( - "shooshtime", - Arc::new(shooshtime::ShooshtimeProvider::new()) as DynProvider, - ); - m.insert( - "pornzog", - Arc::new(pornzog::PornzogProvider::new()) as DynProvider, - ); - m.insert( - "omgxxx", - Arc::new(omgxxx::OmgxxxProvider::new()) as DynProvider, - ); - m.insert("beeg", Arc::new(beeg::BeegProvider::new()) as DynProvider); - m.insert( - "tnaflix", - Arc::new(tnaflix::TnaflixProvider::new()) as DynProvider, - ); - m.insert( - "tokyomotion", - Arc::new(tokyomotion::TokyomotionProvider::new()) as DynProvider, - ); - m.insert( - "viralxxxporn", - Arc::new(viralxxxporn::ViralxxxpornProvider::new()) as DynProvider, - ); - m.insert( - "vrporn", - Arc::new(vrporn::VrpornProvider::new()) as DynProvider, - ); - // m.insert("pornxp", Arc::new(pornxp::PornxpProvider::new()) as DynProvider); - m.insert( - "rule34gen", - Arc::new(rule34gen::Rule34genProvider::new()) as DynProvider, - ); - m.insert( - "xxdbx", - Arc::new(xxdbx::XxdbxProvider::new()) as DynProvider, - ); - m.insert( - "xfree", - Arc::new(xfree::XfreeProvider::new()) as DynProvider, - ); - m.insert( - "hqporner", - Arc::new(hqporner::HqpornerProvider::new()) as DynProvider, - ); - m.insert( - "pmvhaven", - Arc::new(pmvhaven::PmvhavenProvider::new()) as DynProvider, - ); - m.insert( - "noodlemagazine", - Arc::new(noodlemagazine::NoodlemagazineProvider::new()) as DynProvider, - ); - m.insert( - "pimpbunny", - Arc::new(pimpbunny::PimpbunnyProvider::new()) as DynProvider, - ); - m.insert( - "javtiful", - Arc::new(javtiful::JavtifulProvider::new()) as DynProvider, - ); - m.insert( - "hypnotube", - Arc::new(hypnotube::HypnotubeProvider::new()) as DynProvider, - ); - m.insert( - "freepornvideosxxx", - Arc::new(freepornvideosxxx::FreepornvideosxxxProvider::new()) as DynProvider, - ); - m.insert( - "heavyfetish", - Arc::new(heavyfetish::HeavyfetishProvider::new()) as DynProvider, - ); - m.insert("hsex", Arc::new(hsex::HsexProvider::new()) as DynProvider); - m.insert( - "hentaihaven", - Arc::new(hentaihaven::HentaihavenProvider::new()) as DynProvider, - ); - m.insert( - "chaturbate", - Arc::new(chaturbate::ChaturbateProvider::new()) as DynProvider, - ); - // m.insert("tube8", Arc::new(tube8::Tube8Provider::new()) as DynProvider); - // add more here as you migrate them + include!(concat!(env!("OUT_DIR"), "/provider_registry.rs")); m }); pub fn init_providers_now() { // Idempotent & thread-safe: runs the Lazy init exactly once. + crate::flow_debug!( + "provider init selection={:?}", + compile_time_selected_provider() + ); Lazy::force(&ALL_PROVIDERS); } +pub fn compile_time_selected_provider() -> Option<&'static str> { + COMPILE_TIME_SELECTED_PROVIDER +} + +pub fn resolve_provider_for_build<'a>(channel: &'a str) -> &'a str { + match compile_time_selected_provider() { + Some(selected) if channel == "all" => selected, + _ => channel, + } +} + pub fn panic_payload_to_string(payload: Box) -> String { if let Some(s) = payload.downcast_ref::<&str>() { return (*s).to_string(); @@ -260,10 +66,29 @@ pub async fn run_provider_guarded(provider_name: &str, context: &str, fut: F) where F: Future>, { + crate::flow_debug!( + "provider guard enter provider={} context={}", + provider_name, + context + ); match AssertUnwindSafe(fut).catch_unwind().await { - Ok(videos) => videos, + Ok(videos) => { + crate::flow_debug!( + "provider guard exit provider={} context={} videos={}", + provider_name, + context, + videos.len() + ); + videos + } Err(payload) => { let panic_msg = panic_payload_to_string(payload); + crate::flow_debug!( + "provider guard panic provider={} context={} panic={}", + provider_name, + context, + &panic_msg + ); let _ = send_discord_error_report( format!("Provider panic: {}", provider_name), None, @@ -307,8 +132,21 @@ pub fn requester_or_default( context: &str, ) -> Requester { match options.requester.clone() { - Some(requester) => requester, + Some(requester) => { + crate::flow_debug!( + "provider requester existing provider={} context={} trace={}", + provider_name, + context, + requester.debug_trace_id().unwrap_or("none") + ); + requester + } None => { + crate::flow_debug!( + "provider requester fallback provider={} context={}", + provider_name, + context + ); report_provider_error_background( provider_name, context, @@ -343,52 +181,7 @@ pub fn build_proxy_url(options: &ServerOptions, proxy: &str, target: &str) -> St } fn channel_metadata_for(id: &str) -> Option { - match id { - "all" | "hottub" => Some(all::CHANNEL_METADATA), - "pornhub" => Some(pornhub::CHANNEL_METADATA), - "spankbang" => Some(spankbang::CHANNEL_METADATA), - "rule34video" => Some(rule34video::CHANNEL_METADATA), - "redtube" => Some(redtube::CHANNEL_METADATA), - "okporn" => Some(okporn::CHANNEL_METADATA), - "pornhat" => Some(pornhat::CHANNEL_METADATA), - "perfectgirls" => Some(perfectgirls::CHANNEL_METADATA), - "okxxx" => Some(okxxx::CHANNEL_METADATA), - "homoxxx" => Some(homoxxx::CHANNEL_METADATA), - "missav" => Some(missav::CHANNEL_METADATA), - "xxthots" => Some(xxthots::CHANNEL_METADATA), - "yesporn" => Some(yesporn::CHANNEL_METADATA), - "sxyprn" => Some(sxyprn::CHANNEL_METADATA), - "porn00" => Some(porn00::CHANNEL_METADATA), - "youjizz" => Some(youjizz::CHANNEL_METADATA), - "paradisehill" => Some(paradisehill::CHANNEL_METADATA), - "porn4fans" => Some(porn4fans::CHANNEL_METADATA), - "porndish" => Some(porndish::CHANNEL_METADATA), - "shooshtime" => Some(shooshtime::CHANNEL_METADATA), - "pornzog" => Some(pornzog::CHANNEL_METADATA), - "omgxxx" => Some(omgxxx::CHANNEL_METADATA), - "beeg" => Some(beeg::CHANNEL_METADATA), - "tnaflix" => Some(tnaflix::CHANNEL_METADATA), - "tokyomotion" => Some(tokyomotion::CHANNEL_METADATA), - "viralxxxporn" => Some(viralxxxporn::CHANNEL_METADATA), - "vrporn" => Some(vrporn::CHANNEL_METADATA), - "rule34gen" => Some(rule34gen::CHANNEL_METADATA), - "xxdbx" => Some(xxdbx::CHANNEL_METADATA), - "xfree" => Some(xfree::CHANNEL_METADATA), - "hqporner" => Some(hqporner::CHANNEL_METADATA), - "pmvhaven" => Some(pmvhaven::CHANNEL_METADATA), - "noodlemagazine" => Some(noodlemagazine::CHANNEL_METADATA), - "pimpbunny" => Some(pimpbunny::CHANNEL_METADATA), - "javtiful" => Some(javtiful::CHANNEL_METADATA), - "hypnotube" => Some(hypnotube::CHANNEL_METADATA), - "freepornvideosxxx" => Some(freepornvideosxxx::CHANNEL_METADATA), - "heavyfetish" => Some(heavyfetish::CHANNEL_METADATA), - "hsex" => Some(hsex::CHANNEL_METADATA), - "hentaihaven" => Some(hentaihaven::CHANNEL_METADATA), - "hanime" => Some(hanime::CHANNEL_METADATA), - "perverzija" => Some(perverzija::CHANNEL_METADATA), - "chaturbate" => Some(chaturbate::CHANNEL_METADATA), - _ => None, - } + include!(concat!(env!("OUT_DIR"), "/provider_metadata_fn.rs")) } fn channel_group_title(group_id: &str) -> &'static str { @@ -536,6 +329,11 @@ pub fn build_status_response(status: Status) -> StatusResponse { .collect::>(); assign_channel_sort_order(&mut channels); let channelGroups = build_channel_groups(&channels); + crate::flow_debug!( + "status response build channels={} groups={}", + channels.len(), + channelGroups.len() + ); StatusResponse { id: status.id, @@ -590,7 +388,7 @@ pub trait Provider: Send + Sync { } } -#[cfg(test)] +#[cfg(all(test, not(hottub_single_provider)))] mod tests { use super::*; use crate::status::ChannelOption; diff --git a/src/util/flow_debug.rs b/src/util/flow_debug.rs new file mode 100644 index 0000000..03c4fb3 --- /dev/null +++ b/src/util/flow_debug.rs @@ -0,0 +1,43 @@ +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{SystemTime, UNIX_EPOCH}; + +static NEXT_TRACE_ID: AtomicU64 = AtomicU64::new(1); + +pub fn next_trace_id(prefix: &str) -> String { + let id = NEXT_TRACE_ID.fetch_add(1, Ordering::Relaxed); + format!("{prefix}-{id:06}") +} + +#[cfg(feature = "debug")] +pub fn emit(module: &str, line: u32, message: String) { + let millis = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|duration| duration.as_millis()) + .unwrap_or_default(); + eprintln!("[debug][{millis}][{module}:{line}] {message}"); +} + +#[cfg(not(feature = "debug"))] +pub fn emit(_module: &str, _line: u32, _message: String) {} + +pub fn preview(value: &str, limit: usize) -> String { + if value.len() <= limit { + return value.to_string(); + } + + let mut end = limit; + while !value.is_char_boundary(end) { + end -= 1; + } + format!("{}...", &value[..end]) +} + +#[macro_export] +macro_rules! flow_debug { + ($($arg:tt)*) => {{ + #[cfg(feature = "debug")] + { + $crate::util::flow_debug::emit(module_path!(), line!(), format!($($arg)*)); + } + }}; +} diff --git a/src/util/mod.rs b/src/util/mod.rs index 655f888..03ad843 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -1,6 +1,7 @@ pub mod cache; pub mod discord; pub mod flaresolverr; +pub mod flow_debug; pub mod proxy; pub mod requester; pub mod time; diff --git a/src/util/requester.rs b/src/util/requester.rs index bb38bea..3e5f5e8 100644 --- a/src/util/requester.rs +++ b/src/util/requester.rs @@ -26,6 +26,8 @@ pub struct Requester { client: Client, #[serde(skip)] cookie_jar: Arc, + #[serde(skip)] + debug_trace_id: Option, proxy: bool, flaresolverr_session: Option, user_agent: Option, @@ -35,6 +37,7 @@ impl fmt::Debug for Requester { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("Requester") .field("proxy", &self.proxy) + .field("debug_trace_id", &self.debug_trace_id) .field("flaresolverr_session", &self.flaresolverr_session) .field("user_agent", &self.user_agent) .finish() @@ -67,6 +70,7 @@ impl Requester { let requester = Requester { client, cookie_jar, + debug_trace_id: None, proxy: false, flaresolverr_session: None, user_agent: None, @@ -84,6 +88,18 @@ impl Requester { self.proxy = proxy; } + pub fn proxy_enabled(&self) -> bool { + self.proxy + } + + pub fn set_debug_trace_id(&mut self, debug_trace_id: Option) { + self.debug_trace_id = debug_trace_id; + } + + pub fn debug_trace_id(&self) -> Option<&str> { + self.debug_trace_id.as_deref() + } + pub fn cookie_header_for_url(&self, url: &str) -> Option { let parsed = url.parse::().ok()?; match self.cookie_jar.cookies(&parsed) { @@ -102,6 +118,12 @@ impl Requester { } pub async fn get_raw(&mut self, url: &str) -> Result { + crate::flow_debug!( + "trace={} requester get_raw url={} proxy={}", + self.debug_trace_id().unwrap_or("none"), + crate::util::flow_debug::preview(url, 120), + self.proxy + ); let client = Self::build_client(self.cookie_jar.clone(), self.user_agent.as_deref()); let mut request = client.get(url).version(Version::HTTP_11); @@ -121,6 +143,13 @@ impl Requester { url: &str, headers: Vec<(String, String)>, ) -> Result { + crate::flow_debug!( + "trace={} requester get_raw_with_headers url={} headers={} proxy={}", + self.debug_trace_id().unwrap_or("none"), + crate::util::flow_debug::preview(url, 120), + headers.len(), + self.proxy + ); let client = Self::build_client(self.cookie_jar.clone(), self.user_agent.as_deref()); let mut request = client.get(url).version(Version::HTTP_11); @@ -147,6 +176,13 @@ impl Requester { where S: Serialize + ?Sized, { + crate::flow_debug!( + "trace={} requester post_json url={} headers={} proxy={}", + self.debug_trace_id().unwrap_or("none"), + crate::util::flow_debug::preview(url, 120), + headers.len(), + self.proxy + ); let mut request = self.client.post(url).version(Version::HTTP_11).json(data); // Set custom headers @@ -170,6 +206,14 @@ impl Requester { data: &str, headers: Vec<(&str, &str)>, ) -> Result { + crate::flow_debug!( + "trace={} requester post url={} headers={} body_len={} proxy={}", + self.debug_trace_id().unwrap_or("none"), + crate::util::flow_debug::preview(url, 120), + headers.len(), + data.len(), + self.proxy + ); let mut request = self .client .post(url) @@ -198,6 +242,13 @@ impl Requester { headers: Vec<(String, String)>, _http_version: Option, ) -> Result { + crate::flow_debug!( + "trace={} requester post_multipart url={} headers={} proxy={}", + self.debug_trace_id().unwrap_or("none"), + crate::util::flow_debug::preview(url, 120), + headers.len(), + self.proxy + ); let http_version = match _http_version { Some(v) => v, None => Version::HTTP_11, @@ -234,6 +285,14 @@ impl Requester { headers: Vec<(String, String)>, _http_version: Option, ) -> Result { + crate::flow_debug!( + "trace={} requester get_with_headers start url={} headers={} http_version={:?} proxy={}", + self.debug_trace_id().unwrap_or("none"), + crate::util::flow_debug::preview(url, 120), + headers.len(), + _http_version, + self.proxy + ); let http_version = match _http_version { Some(v) => v, None => Version::HTTP_11, @@ -250,10 +309,21 @@ impl Requester { } } let response = request.send().await?; + crate::flow_debug!( + "trace={} requester direct response url={} status={}", + self.debug_trace_id().unwrap_or("none"), + crate::util::flow_debug::preview(url, 120), + response.status() + ); if response.status().is_success() || response.status().as_u16() == 404 { return Ok(response.text().await?); } if response.status().as_u16() == 429 { + crate::flow_debug!( + "trace={} requester direct retry url={} status=429", + self.debug_trace_id().unwrap_or("none"), + crate::util::flow_debug::preview(url, 120) + ); tokio::time::sleep(std::time::Duration::from_secs(1)).await; continue; } else { @@ -276,6 +346,12 @@ impl Requester { if self.proxy && env::var("BURP_URL").is_ok() { flare.set_proxy(true); } + crate::flow_debug!( + "trace={} requester flaresolverr url={} proxy={}", + self.debug_trace_id().unwrap_or("none"), + crate::util::flow_debug::preview(url, 120), + self.proxy + ); let res = flare .solve(FlareSolverrRequest { @@ -300,6 +376,12 @@ impl Requester { } self.client = Self::build_client(self.cookie_jar.clone(), self.user_agent.as_deref()); + crate::flow_debug!( + "trace={} requester flaresolverr solved url={} user_agent={}", + self.debug_trace_id().unwrap_or("none"), + crate::util::flow_debug::preview(url, 120), + crate::util::flow_debug::preview(self.user_agent.as_deref().unwrap_or("unknown"), 96) + ); // Retry the original URL with the updated client & (optional) proxy let mut request = self.client.get(url).version(Version::HTTP_11); @@ -314,11 +396,22 @@ impl Requester { } let response = request.send().await?; + crate::flow_debug!( + "trace={} requester retry response url={} status={}", + self.debug_trace_id().unwrap_or("none"), + crate::util::flow_debug::preview(url, 120), + response.status() + ); if response.status().is_success() { return Ok(response.text().await?); } // Fall back to FlareSolverr-provided body + crate::flow_debug!( + "trace={} requester fallback body url={}", + self.debug_trace_id().unwrap_or("none"), + crate::util::flow_debug::preview(url, 120) + ); Ok(res.solution.response) } }