simon/hottub
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

debugging and single provider compime

Browse Source
This commit is contained in:
Simon
2026-03-21 21:18:43 +00:00
parent 05ea90405b
commit 7b66e5b28a
8 changed files with 640 additions and 266 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
Cargo.toml
View File

@@ -2,6 +2,10 @@
name = "hottub"
version = "0.1.0"
edition = "2024"
build = "build.rs"
[features]
debug = []
[dependencies]
cute = "0.3.0"

328
build.rs Normal file
View File

@@ -0,0 +1,328 @@
use std::env;
use std::fs;
use std::path::PathBuf;
struct ProviderDef {
id: &'static str,
module: &'static str,
ty: &'static str,
}
const PROVIDERS: &[ProviderDef] = &[
ProviderDef {
id: "all",
module: "all",
ty: "AllProvider",
},
ProviderDef {
id: "perverzija",
module: "perverzija",
ty: "PerverzijaProvider",
},
ProviderDef {
id: "hanime",
module: "hanime",
ty: "HanimeProvider",
},
ProviderDef {
id: "pornhub",
module: "pornhub",
ty: "PornhubProvider",
},
ProviderDef {
id: "spankbang",
module: "spankbang",
ty: "SpankbangProvider",
},
ProviderDef {
id: "rule34video",
module: "rule34video",
ty: "Rule34videoProvider",
},
ProviderDef {
id: "redtube",
module: "redtube",
ty: "RedtubeProvider",
},
ProviderDef {
id: "okporn",
module: "okporn",
ty: "OkpornProvider",
},
ProviderDef {
id: "pornhat",
module: "pornhat",
ty: "PornhatProvider",
},
ProviderDef {
id: "perfectgirls",
module: "perfectgirls",
ty: "PerfectgirlsProvider",
},
ProviderDef {
id: "okxxx",
module: "okxxx",
ty: "OkxxxProvider",
},
ProviderDef {
id: "homoxxx",
module: "homoxxx",
ty: "HomoxxxProvider",
},
ProviderDef {
id: "missav",
module: "missav",
ty: "MissavProvider",
},
ProviderDef {
id: "xxthots",
module: "xxthots",
ty: "XxthotsProvider",
},
ProviderDef {
id: "yesporn",
module: "yesporn",
ty: "YespornProvider",
},
ProviderDef {
id: "sxyprn",
module: "sxyprn",
ty: "SxyprnProvider",
},
ProviderDef {
id: "porn00",
module: "porn00",
ty: "Porn00Provider",
},
ProviderDef {
id: "youjizz",
module: "youjizz",
ty: "YoujizzProvider",
},
ProviderDef {
id: "paradisehill",
module: "paradisehill",
ty: "ParadisehillProvider",
},
ProviderDef {
id: "porn4fans",
module: "porn4fans",
ty: "Porn4fansProvider",
},
ProviderDef {
id: "porndish",
module: "porndish",
ty: "PorndishProvider",
},
ProviderDef {
id: "shooshtime",
module: "shooshtime",
ty: "ShooshtimeProvider",
},
ProviderDef {
id: "pornzog",
module: "pornzog",
ty: "PornzogProvider",
},
ProviderDef {
id: "omgxxx",
module: "omgxxx",
ty: "OmgxxxProvider",
},
ProviderDef {
id: "beeg",
module: "beeg",
ty: "BeegProvider",
},
ProviderDef {
id: "tnaflix",
module: "tnaflix",
ty: "TnaflixProvider",
},
ProviderDef {
id: "tokyomotion",
module: "tokyomotion",
ty: "TokyomotionProvider",
},
ProviderDef {
id: "viralxxxporn",
module: "viralxxxporn",
ty: "ViralxxxpornProvider",
},
ProviderDef {
id: "vrporn",
module: "vrporn",
ty: "VrpornProvider",
},
ProviderDef {
id: "rule34gen",
module: "rule34gen",
ty: "Rule34genProvider",
},
ProviderDef {
id: "xxdbx",
module: "xxdbx",
ty: "XxdbxProvider",
},
ProviderDef {
id: "xfree",
module: "xfree",
ty: "XfreeProvider",
},
ProviderDef {
id: "hqporner",
module: "hqporner",
ty: "HqpornerProvider",
},
ProviderDef {
id: "pmvhaven",
module: "pmvhaven",
ty: "PmvhavenProvider",
},
ProviderDef {
id: "noodlemagazine",
module: "noodlemagazine",
ty: "NoodlemagazineProvider",
},
ProviderDef {
id: "pimpbunny",
module: "pimpbunny",
ty: "PimpbunnyProvider",
},
ProviderDef {
id: "javtiful",
module: "javtiful",
ty: "JavtifulProvider",
},
ProviderDef {
id: "hypnotube",
module: "hypnotube",
ty: "HypnotubeProvider",
},
ProviderDef {
id: "freepornvideosxxx",
module: "freepornvideosxxx",
ty: "FreepornvideosxxxProvider",
},
ProviderDef {
id: "heavyfetish",
module: "heavyfetish",
ty: "HeavyfetishProvider",
},
ProviderDef {
id: "hsex",
module: "hsex",
ty: "HsexProvider",
},
ProviderDef {
id: "hentaihaven",
module: "hentaihaven",
ty: "HentaihavenProvider",
},
ProviderDef {
id: "chaturbate",
module: "chaturbate",
ty: "ChaturbateProvider",
},
];
fn main() {
println!("cargo:rerun-if-changed=build.rs");
println!("cargo:rerun-if-env-changed=HOT_TUB_PROVIDER");
println!("cargo:rerun-if-env-changed=HOTTUB_PROVIDER");
println!("cargo:rustc-check-cfg=cfg(hottub_single_provider)");
let selected = env::var("HOT_TUB_PROVIDER")
.or_else(|_| env::var("HOTTUB_PROVIDER"))
.ok()
.map(|value| value.trim().to_string())
.filter(|value| !value.is_empty());
let providers = match selected.as_deref() {
Some(selected_id) => {
let provider = PROVIDERS
.iter()
.find(|provider| provider.id == selected_id)
.unwrap_or_else(|| {
panic!("Unknown provider `{selected_id}` from HOT_TUB_PROVIDER/HOTTUB_PROVIDER")
});
println!("cargo:rustc-cfg=hottub_single_provider");
vec![provider]
}
None => PROVIDERS.iter().collect(),
};
let out_dir = PathBuf::from(env::var("OUT_DIR").expect("OUT_DIR"));
let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR"));
let modules = providers
.iter()
.map(|provider| {
let module_path = manifest_dir
.join("src/providers")
.join(format!("{}.rs", provider.module));
format!(
"#[path = r#\"{}\"#]\npub mod {};",
module_path.display(),
provider.module
)
})
.collect::<Vec<_>>()
.join("\n");
fs::write(out_dir.join("provider_modules.rs"), format!("{modules}\n"))
.expect("write provider_modules.rs");
let registry = providers
.iter()
.map(|provider| {
format!(
"m.insert(\"{id}\", Arc::new({module}::{ty}::new()) as DynProvider);",
id = provider.id,
module = provider.module,
ty = provider.ty
)
})
.collect::<Vec<_>>()
.join("\n");
fs::write(
out_dir.join("provider_registry.rs"),
format!("{{\n{registry}\n}}\n"),
)
.expect("write provider_registry.rs");
let metadata_arms = providers
.iter()
.map(|provider| {
if provider.id == "all" {
format!(
"\"all\" | \"hottub\" => Some({module}::CHANNEL_METADATA),",
module = provider.module
)
} else {
format!(
"\"{id}\" => Some({module}::CHANNEL_METADATA),",
id = provider.id,
module = provider.module
)
}
})
.collect::<Vec<_>>()
.join("\n");
fs::write(
out_dir.join("provider_metadata_fn.rs"),
format!("match id {{\n{metadata_arms}\n_ => None,\n}}\n"),
)
.expect("write provider_metadata_fn.rs");
let selection = match selected.as_deref() {
Some(selected_id) => format!(
"pub const COMPILE_TIME_SELECTED_PROVIDER: Option<&str> = Some(\"{selected_id}\");"
),
None => "pub const COMPILE_TIME_SELECTED_PROVIDER: Option<&str> = None;".to_string(),
};
fs::write(
out_dir.join("provider_selection.rs"),
format!("{selection}\n"),
)
.expect("write provider_selection.rs");
}

99
src/api.rs
View File

@@ -1,6 +1,6 @@
use crate::providers::{
ALL_PROVIDERS, DynProvider, build_status_response, panic_payload_to_string, report_provider_error,
run_provider_guarded,
ALL_PROVIDERS, DynProvider, build_status_response, panic_payload_to_string,
report_provider_error, resolve_provider_for_build, run_provider_guarded,
};
use crate::util::cache::VideoCache;
use crate::util::discord::send_discord_error_report;
@@ -146,6 +146,7 @@ pub fn config(cfg: &mut web::ServiceConfig) {
}
async fn status(req: HttpRequest) -> Result<impl web::Responder, web::Error> {
let trace_id = crate::util::flow_debug::next_trace_id("status");
let clientversion: ClientVersion = match req.headers().get("User-Agent") {
Some(v) => match v.to_str() {
Ok(useragent) => ClientVersion::parse(useragent)
@@ -159,6 +160,12 @@ async fn status(req: HttpRequest) -> Result<impl web::Responder, web::Error> {
"Received status request with client version: {:?}",
clientversion
);
crate::flow_debug!(
"trace={} status request host={} client={:?}",
trace_id,
req.connection_info().host(),
&clientversion
);
let host = req
.headers()
@@ -168,8 +175,14 @@ async fn status(req: HttpRequest) -> Result<impl web::Responder, web::Error> {
.to_string();
let public_url_base = format!("{}://{}", req.connection_info().scheme(), host);
let mut status = Status::new();
let mut channel_count = 0usize;
for (provider_name, provider) in ALL_PROVIDERS.iter() {
crate::flow_debug!(
"trace={} status inspecting provider={}",
trace_id,
provider_name
);
let channel_result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
provider.get_channel(clientversion.clone())
}));
@@ -178,17 +191,37 @@ async fn status(req: HttpRequest) -> Result<impl web::Responder, web::Error> {
if channel.favicon.starts_with('/') {
channel.favicon = format!("{}{}", public_url_base, channel.favicon);
}
channel_count += 1;
crate::flow_debug!(
"trace={} status added channel id={} provider={}",
trace_id,
channel.id.as_str(),
provider_name
);
status.add_channel(channel)
}
Ok(None) => {}
Err(payload) => {
let panic_msg = panic_payload_to_string(payload);
crate::flow_debug!(
"trace={} status provider panic provider={} panic={}",
trace_id,
provider_name,
&panic_msg
);
report_provider_error(provider_name, "status.get_channel", &panic_msg).await;
}
}
}
status.iconUrl = format!("{}/favicon.ico", public_url_base).to_string();
Ok(web::HttpResponse::Ok().json(&build_status_response(status)))
let response = build_status_response(status);
crate::flow_debug!(
"trace={} status response channels={} groups={}",
trace_id,
channel_count,
response.channelGroups.len()
);
Ok(web::HttpResponse::Ok().json(&response))
}
async fn videos_post(
@@ -198,6 +231,7 @@ async fn videos_post(
requester: web::types::State<Requester>,
req: HttpRequest,
) -> Result<impl web::Responder, web::Error> {
let trace_id = crate::util::flow_debug::next_trace_id("videos");
let clientversion: ClientVersion = match req.headers().get("User-Agent") {
Some(v) => match v.to_str() {
Ok(useragent) => ClientVersion::parse(useragent)
@@ -235,11 +269,12 @@ async fn videos_post(
},
items: vec![],
};
let channel: String = video_request
let requested_channel: String = video_request
.channel
.as_deref()
.unwrap_or("all")
.to_string();
let channel = resolve_provider_for_build(requested_channel.as_str()).to_string();
let sort: String = video_request.sort.as_deref().unwrap_or("date").to_string();
let (query, literal_query) = normalize_query(video_request.query.as_deref());
let page: u8 = video_request
@@ -294,6 +329,22 @@ async fn videos_post(
req.connection_info().scheme(),
req.connection_info().host()
);
crate::flow_debug!(
"trace={} videos request requested_channel={} resolved_channel={} sort={} query={:?} page={} per_page={} filter={} category={} sites={} client={:?}",
trace_id,
&requested_channel,
&channel,
&sort,
&query,
page,
perPage,
&filter,
&category,
&sites,
&clientversion
);
let mut requester = requester;
requester.set_debug_trace_id(Some(trace_id.clone()));
let options = ServerOptions {
featured: Some(featured),
category: Some(category),
@@ -309,6 +360,12 @@ async fn videos_post(
sort: Some(sort.clone()),
sexuality: Some(sexuality),
};
crate::flow_debug!(
"trace={} videos provider dispatch provider={} literal_query={:?}",
trace_id,
&channel,
&literal_query
);
let mut video_items = run_provider_guarded(
&channel,
"videos_post.get_videos",
@@ -323,6 +380,11 @@ async fn videos_post(
),
)
.await;
crate::flow_debug!(
"trace={} videos provider returned count={}",
trace_id,
video_items.len()
);
// There is a bug in Hottub38 that makes the client error for a 403-url even though formats work fine
if clientversion == ClientVersion::new(38, 0, "Hot%20Tub".to_string()) {
@@ -345,7 +407,14 @@ async fn videos_post(
}
if let Some(literal_query) = literal_query.as_deref() {
let before = video_items.len();
video_items.retain(|video| video_matches_literal_query(video, literal_query));
crate::flow_debug!(
"trace={} videos literal filter kept={} removed={}",
trace_id,
video_items.len(),
before.saturating_sub(video_items.len())
);
}
videos.items = video_items.clone();
@@ -365,7 +434,14 @@ async fn videos_post(
let per_page_clone = perPage.to_string();
let options_clone = options.clone();
let channel_clone = channel.clone();
let prefetch_trace_id = trace_id.clone();
task::spawn_local(async move {
crate::flow_debug!(
"trace={} videos prefetch spawn next_page={} provider={}",
prefetch_trace_id,
next_page,
&channel_clone
);
// if let AnyProvider::Spankbang(_) = provider_clone {
// // Spankbang has a delay for the next page
// ntex::time::sleep(ntex::time::Seconds(80)).await;
@@ -399,11 +475,23 @@ async fn videos_post(
}
}
crate::flow_debug!(
"trace={} videos response items={} has_next={}",
trace_id,
videos.items.len(),
videos.pageInfo.hasNextPage
);
Ok(web::HttpResponse::Ok().json(&videos))
}
pub fn get_provider(channel: &str) -> Option<DynProvider> {
ALL_PROVIDERS.get(channel).cloned()
let provider = ALL_PROVIDERS.get(channel).cloned();
crate::flow_debug!(
"provider lookup channel={} found={}",
channel,
provider.is_some()
);
provider
}
pub async fn test() -> Result<impl web::Responder, web::Error> {
@@ -424,6 +512,7 @@ pub async fn test() -> Result<impl web::Responder, web::Error> {
pub async fn proxies() -> Result<impl web::Responder, web::Error> {
let proxies = all_proxies_snapshot().await.unwrap_or_default();
crate::flow_debug!("proxies endpoint snapshot_count={}", proxies.len());
let mut by_protocol: std::collections::BTreeMap<String, Vec<Proxy>> =
std::collections::BTreeMap::new();
for proxy in proxies {

18
src/main.rs
View File

@@ -39,6 +39,11 @@ async fn main() -> std::io::Result<()> {
}
}
env_logger::init(); // You need this to actually see logs
crate::flow_debug!(
"startup begin rust_log={} debug_compiled={}",
std::env::var("RUST_LOG").unwrap_or_else(|_| "unset".to_string()),
cfg!(feature = "debug")
);
// set up database connection pool
let connspec = std::env::var("DATABASE_URL").expect("DATABASE_URL");
@@ -46,15 +51,25 @@ async fn main() -> std::io::Result<()> {
let pool = r2d2::Pool::builder()
.build(manager)
.expect("Failed to create pool.");
crate::flow_debug!(
"database pool ready database_url={}",
crate::util::flow_debug::preview(&connspec, 96)
);
let mut requester = util::requester::Requester::new();
requester.set_proxy(env::var("PROXY").unwrap_or("0".to_string()) != "0".to_string());
crate::flow_debug!(
"requester initialized proxy_enabled={}",
requester.proxy_enabled()
);
let cache: util::cache::VideoCache = crate::util::cache::VideoCache::new()
.max_size(100_000)
.to_owned();
crate::flow_debug!("video cache initialized max_size=100000");
thread::spawn(move || {
crate::flow_debug!("provider init thread spawned");
// Create a tiny runtime just for these async tasks
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
@@ -62,10 +77,13 @@ async fn main() -> std::io::Result<()> {
.expect("build tokio runtime");
rt.block_on(async move {
crate::flow_debug!("provider init begin");
providers::init_providers_now();
crate::flow_debug!("provider init complete");
});
});
crate::flow_debug!("http server binding addr=0.0.0.0:18080 workers=8");
web::HttpServer::new(move || {
web::App::new()
.state(pool.clone())

320
src/providers/mod.rs
View File

@@ -14,52 +14,8 @@ use crate::{
videos::{ServerOptions, VideoItem},
};
pub mod all;
pub mod hanime;
pub mod homoxxx;
pub mod okporn;
pub mod okxxx;
pub mod perfectgirls;
pub mod perverzija;
pub mod pmvhaven;
pub mod pornhat;
pub mod pornhub;
pub mod redtube;
pub mod rule34video;
pub mod spankbang;
// pub mod hentaimoon;
pub mod beeg;
pub mod missav;
pub mod omgxxx;
pub mod paradisehill;
pub mod porn00;
pub mod porn4fans;
pub mod porndish;
pub mod pornzog;
pub mod shooshtime;
pub mod sxyprn;
pub mod tnaflix;
pub mod tokyomotion;
pub mod viralxxxporn;
pub mod vrporn;
pub mod xfree;
pub mod xxthots;
pub mod yesporn;
pub mod youjizz;
// pub mod pornxp;
pub mod chaturbate;
pub mod freepornvideosxxx;
pub mod heavyfetish;
pub mod hentaihaven;
pub mod hqporner;
pub mod hsex;
pub mod hypnotube;
pub mod javtiful;
pub mod noodlemagazine;
pub mod pimpbunny;
pub mod rule34gen;
pub mod xxdbx;
// pub mod tube8;
include!(concat!(env!("OUT_DIR"), "/provider_selection.rs"));
include!(concat!(env!("OUT_DIR"), "/provider_modules.rs"));
// convenient alias
pub type DynProvider = Arc<dyn Provider>;
@@ -72,180 +28,30 @@ pub struct ProviderChannelMetadata {
pub static ALL_PROVIDERS: Lazy<HashMap<&'static str, DynProvider>> = Lazy::new(|| {
let mut m = HashMap::default();
m.insert("all", Arc::new(all::AllProvider::new()) as DynProvider);
m.insert(
"perverzija",
Arc::new(perverzija::PerverzijaProvider::new()) as DynProvider,
);
m.insert(
"hanime",
Arc::new(hanime::HanimeProvider::new()) as DynProvider,
);
m.insert(
"pornhub",
Arc::new(pornhub::PornhubProvider::new()) as DynProvider,
);
m.insert(
"spankbang",
Arc::new(spankbang::SpankbangProvider::new()) as DynProvider,
);
m.insert(
"rule34video",
Arc::new(rule34video::Rule34videoProvider::new()) as DynProvider,
);
m.insert(
"redtube",
Arc::new(redtube::RedtubeProvider::new()) as DynProvider,
);
m.insert(
"okporn",
Arc::new(okporn::OkpornProvider::new()) as DynProvider,
);
m.insert(
"pornhat",
Arc::new(pornhat::PornhatProvider::new()) as DynProvider,
);
m.insert(
"perfectgirls",
Arc::new(perfectgirls::PerfectgirlsProvider::new()) as DynProvider,
);
m.insert(
"okxxx",
Arc::new(okxxx::OkxxxProvider::new()) as DynProvider,
);
m.insert(
"homoxxx",
Arc::new(homoxxx::HomoxxxProvider::new()) as DynProvider,
);
m.insert(
"missav",
Arc::new(missav::MissavProvider::new()) as DynProvider,
);
m.insert(
"xxthots",
Arc::new(xxthots::XxthotsProvider::new()) as DynProvider,
);
m.insert(
"yesporn",
Arc::new(yesporn::YespornProvider::new()) as DynProvider,
);
m.insert(
"sxyprn",
Arc::new(sxyprn::SxyprnProvider::new()) as DynProvider,
);
m.insert(
"porn00",
Arc::new(porn00::Porn00Provider::new()) as DynProvider,
);
m.insert(
"youjizz",
Arc::new(youjizz::YoujizzProvider::new()) as DynProvider,
);
m.insert(
"paradisehill",
Arc::new(paradisehill::ParadisehillProvider::new()) as DynProvider,
);
m.insert(
"porn4fans",
Arc::new(porn4fans::Porn4fansProvider::new()) as DynProvider,
);
m.insert(
"porndish",
Arc::new(porndish::PorndishProvider::new()) as DynProvider,
);
m.insert(
"shooshtime",
Arc::new(shooshtime::ShooshtimeProvider::new()) as DynProvider,
);
m.insert(
"pornzog",
Arc::new(pornzog::PornzogProvider::new()) as DynProvider,
);
m.insert(
"omgxxx",
Arc::new(omgxxx::OmgxxxProvider::new()) as DynProvider,
);
m.insert("beeg", Arc::new(beeg::BeegProvider::new()) as DynProvider);
m.insert(
"tnaflix",
Arc::new(tnaflix::TnaflixProvider::new()) as DynProvider,
);
m.insert(
"tokyomotion",
Arc::new(tokyomotion::TokyomotionProvider::new()) as DynProvider,
);
m.insert(
"viralxxxporn",
Arc::new(viralxxxporn::ViralxxxpornProvider::new()) as DynProvider,
);
m.insert(
"vrporn",
Arc::new(vrporn::VrpornProvider::new()) as DynProvider,
);
// m.insert("pornxp", Arc::new(pornxp::PornxpProvider::new()) as DynProvider);
m.insert(
"rule34gen",
Arc::new(rule34gen::Rule34genProvider::new()) as DynProvider,
);
m.insert(
"xxdbx",
Arc::new(xxdbx::XxdbxProvider::new()) as DynProvider,
);
m.insert(
"xfree",
Arc::new(xfree::XfreeProvider::new()) as DynProvider,
);
m.insert(
"hqporner",
Arc::new(hqporner::HqpornerProvider::new()) as DynProvider,
);
m.insert(
"pmvhaven",
Arc::new(pmvhaven::PmvhavenProvider::new()) as DynProvider,
);
m.insert(
"noodlemagazine",
Arc::new(noodlemagazine::NoodlemagazineProvider::new()) as DynProvider,
);
m.insert(
"pimpbunny",
Arc::new(pimpbunny::PimpbunnyProvider::new()) as DynProvider,
);
m.insert(
"javtiful",
Arc::new(javtiful::JavtifulProvider::new()) as DynProvider,
);
m.insert(
"hypnotube",
Arc::new(hypnotube::HypnotubeProvider::new()) as DynProvider,
);
m.insert(
"freepornvideosxxx",
Arc::new(freepornvideosxxx::FreepornvideosxxxProvider::new()) as DynProvider,
);
m.insert(
"heavyfetish",
Arc::new(heavyfetish::HeavyfetishProvider::new()) as DynProvider,
);
m.insert("hsex", Arc::new(hsex::HsexProvider::new()) as DynProvider);
m.insert(
"hentaihaven",
Arc::new(hentaihaven::HentaihavenProvider::new()) as DynProvider,
);
m.insert(
"chaturbate",
Arc::new(chaturbate::ChaturbateProvider::new()) as DynProvider,
);
// m.insert("tube8", Arc::new(tube8::Tube8Provider::new()) as DynProvider);
// add more here as you migrate them
include!(concat!(env!("OUT_DIR"), "/provider_registry.rs"));
m
});
pub fn init_providers_now() {
// Idempotent & thread-safe: runs the Lazy init exactly once.
crate::flow_debug!(
"provider init selection={:?}",
compile_time_selected_provider()
);
Lazy::force(&ALL_PROVIDERS);
}
pub fn compile_time_selected_provider() -> Option<&'static str> {
COMPILE_TIME_SELECTED_PROVIDER
}
pub fn resolve_provider_for_build<'a>(channel: &'a str) -> &'a str {
match compile_time_selected_provider() {
Some(selected) if channel == "all" => selected,
_ => channel,
}
}
pub fn panic_payload_to_string(payload: Box<dyn std::any::Any + Send>) -> String {
if let Some(s) = payload.downcast_ref::<&str>() {
return (*s).to_string();
@@ -260,10 +66,29 @@ pub async fn run_provider_guarded<F>(provider_name: &str, context: &str, fut: F)
where
F: Future<Output = Vec<VideoItem>>,
{
crate::flow_debug!(
"provider guard enter provider={} context={}",
provider_name,
context
);
match AssertUnwindSafe(fut).catch_unwind().await {
Ok(videos) => videos,
Ok(videos) => {
crate::flow_debug!(
"provider guard exit provider={} context={} videos={}",
provider_name,
context,
videos.len()
);
videos
}
Err(payload) => {
let panic_msg = panic_payload_to_string(payload);
crate::flow_debug!(
"provider guard panic provider={} context={} panic={}",
provider_name,
context,
&panic_msg
);
let _ = send_discord_error_report(
format!("Provider panic: {}", provider_name),
None,
@@ -307,8 +132,21 @@ pub fn requester_or_default(
context: &str,
) -> Requester {
match options.requester.clone() {
Some(requester) => requester,
Some(requester) => {
crate::flow_debug!(
"provider requester existing provider={} context={} trace={}",
provider_name,
context,
requester.debug_trace_id().unwrap_or("none")
);
requester
}
None => {
crate::flow_debug!(
"provider requester fallback provider={} context={}",
provider_name,
context
);
report_provider_error_background(
provider_name,
context,
@@ -343,52 +181,7 @@ pub fn build_proxy_url(options: &ServerOptions, proxy: &str, target: &str) -> St
}
fn channel_metadata_for(id: &str) -> Option<ProviderChannelMetadata> {
match id {
"all" | "hottub" => Some(all::CHANNEL_METADATA),
"pornhub" => Some(pornhub::CHANNEL_METADATA),
"spankbang" => Some(spankbang::CHANNEL_METADATA),
"rule34video" => Some(rule34video::CHANNEL_METADATA),
"redtube" => Some(redtube::CHANNEL_METADATA),
"okporn" => Some(okporn::CHANNEL_METADATA),
"pornhat" => Some(pornhat::CHANNEL_METADATA),
"perfectgirls" => Some(perfectgirls::CHANNEL_METADATA),
"okxxx" => Some(okxxx::CHANNEL_METADATA),
"homoxxx" => Some(homoxxx::CHANNEL_METADATA),
"missav" => Some(missav::CHANNEL_METADATA),
"xxthots" => Some(xxthots::CHANNEL_METADATA),
"yesporn" => Some(yesporn::CHANNEL_METADATA),
"sxyprn" => Some(sxyprn::CHANNEL_METADATA),
"porn00" => Some(porn00::CHANNEL_METADATA),
"youjizz" => Some(youjizz::CHANNEL_METADATA),
"paradisehill" => Some(paradisehill::CHANNEL_METADATA),
"porn4fans" => Some(porn4fans::CHANNEL_METADATA),
"porndish" => Some(porndish::CHANNEL_METADATA),
"shooshtime" => Some(shooshtime::CHANNEL_METADATA),
"pornzog" => Some(pornzog::CHANNEL_METADATA),
"omgxxx" => Some(omgxxx::CHANNEL_METADATA),
"beeg" => Some(beeg::CHANNEL_METADATA),
"tnaflix" => Some(tnaflix::CHANNEL_METADATA),
"tokyomotion" => Some(tokyomotion::CHANNEL_METADATA),
"viralxxxporn" => Some(viralxxxporn::CHANNEL_METADATA),
"vrporn" => Some(vrporn::CHANNEL_METADATA),
"rule34gen" => Some(rule34gen::CHANNEL_METADATA),
"xxdbx" => Some(xxdbx::CHANNEL_METADATA),
"xfree" => Some(xfree::CHANNEL_METADATA),
"hqporner" => Some(hqporner::CHANNEL_METADATA),
"pmvhaven" => Some(pmvhaven::CHANNEL_METADATA),
"noodlemagazine" => Some(noodlemagazine::CHANNEL_METADATA),
"pimpbunny" => Some(pimpbunny::CHANNEL_METADATA),
"javtiful" => Some(javtiful::CHANNEL_METADATA),
"hypnotube" => Some(hypnotube::CHANNEL_METADATA),
"freepornvideosxxx" => Some(freepornvideosxxx::CHANNEL_METADATA),
"heavyfetish" => Some(heavyfetish::CHANNEL_METADATA),
"hsex" => Some(hsex::CHANNEL_METADATA),
"hentaihaven" => Some(hentaihaven::CHANNEL_METADATA),
"hanime" => Some(hanime::CHANNEL_METADATA),
"perverzija" => Some(perverzija::CHANNEL_METADATA),
"chaturbate" => Some(chaturbate::CHANNEL_METADATA),
_ => None,
}
include!(concat!(env!("OUT_DIR"), "/provider_metadata_fn.rs"))
}
fn channel_group_title(group_id: &str) -> &'static str {
@@ -536,6 +329,11 @@ pub fn build_status_response(status: Status) -> StatusResponse {
.collect::<Vec<_>>();
assign_channel_sort_order(&mut channels);
let channelGroups = build_channel_groups(&channels);
crate::flow_debug!(
"status response build channels={} groups={}",
channels.len(),
channelGroups.len()
);
StatusResponse {
id: status.id,
@@ -590,7 +388,7 @@ pub trait Provider: Send + Sync {
}
}
#[cfg(test)]
#[cfg(all(test, not(hottub_single_provider)))]
mod tests {
use super::*;
use crate::status::ChannelOption;

43
src/util/flow_debug.rs Normal file
View File

@@ -0,0 +1,43 @@
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{SystemTime, UNIX_EPOCH};
static NEXT_TRACE_ID: AtomicU64 = AtomicU64::new(1);
pub fn next_trace_id(prefix: &str) -> String {
let id = NEXT_TRACE_ID.fetch_add(1, Ordering::Relaxed);
format!("{prefix}-{id:06}")
}
#[cfg(feature = "debug")]
pub fn emit(module: &str, line: u32, message: String) {
let millis = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|duration| duration.as_millis())
.unwrap_or_default();
eprintln!("[debug][{millis}][{module}:{line}] {message}");
}
#[cfg(not(feature = "debug"))]
pub fn emit(_module: &str, _line: u32, _message: String) {}
pub fn preview(value: &str, limit: usize) -> String {
if value.len() <= limit {
return value.to_string();
}
let mut end = limit;
while !value.is_char_boundary(end) {
end -= 1;
}
format!("{}...", &value[..end])
}
#[macro_export]
macro_rules! flow_debug {
($($arg:tt)*) => {{
#[cfg(feature = "debug")]
{
$crate::util::flow_debug::emit(module_path!(), line!(), format!($($arg)*));
}
}};
}

1
src/util/mod.rs
View File

@@ -1,6 +1,7 @@
pub mod cache;
pub mod discord;
pub mod flaresolverr;
pub mod flow_debug;
pub mod proxy;
pub mod requester;
pub mod time;

93
src/util/requester.rs
View File

@@ -26,6 +26,8 @@ pub struct Requester {
client: Client,
#[serde(skip)]
cookie_jar: Arc<Jar>,
#[serde(skip)]
debug_trace_id: Option<String>,
proxy: bool,
flaresolverr_session: Option<String>,
user_agent: Option<String>,
@@ -35,6 +37,7 @@ impl fmt::Debug for Requester {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Requester")
.field("proxy", &self.proxy)
.field("debug_trace_id", &self.debug_trace_id)
.field("flaresolverr_session", &self.flaresolverr_session)
.field("user_agent", &self.user_agent)
.finish()
@@ -67,6 +70,7 @@ impl Requester {
let requester = Requester {
client,
cookie_jar,
debug_trace_id: None,
proxy: false,
flaresolverr_session: None,
user_agent: None,
@@ -84,6 +88,18 @@ impl Requester {
self.proxy = proxy;
}
pub fn proxy_enabled(&self) -> bool {
self.proxy
}
pub fn set_debug_trace_id(&mut self, debug_trace_id: Option<String>) {
self.debug_trace_id = debug_trace_id;
}
pub fn debug_trace_id(&self) -> Option<&str> {
self.debug_trace_id.as_deref()
}
pub fn cookie_header_for_url(&self, url: &str) -> Option<String> {
let parsed = url.parse::<Uri>().ok()?;
match self.cookie_jar.cookies(&parsed) {
@@ -102,6 +118,12 @@ impl Requester {
}
pub async fn get_raw(&mut self, url: &str) -> Result<Response, wreq::Error> {
crate::flow_debug!(
"trace={} requester get_raw url={} proxy={}",
self.debug_trace_id().unwrap_or("none"),
crate::util::flow_debug::preview(url, 120),
self.proxy
);
let client = Self::build_client(self.cookie_jar.clone(), self.user_agent.as_deref());
let mut request = client.get(url).version(Version::HTTP_11);
@@ -121,6 +143,13 @@ impl Requester {
url: &str,
headers: Vec<(String, String)>,
) -> Result<Response, wreq::Error> {
crate::flow_debug!(
"trace={} requester get_raw_with_headers url={} headers={} proxy={}",
self.debug_trace_id().unwrap_or("none"),
crate::util::flow_debug::preview(url, 120),
headers.len(),
self.proxy
);
let client = Self::build_client(self.cookie_jar.clone(), self.user_agent.as_deref());
let mut request = client.get(url).version(Version::HTTP_11);
@@ -147,6 +176,13 @@ impl Requester {
where
S: Serialize + ?Sized,
{
crate::flow_debug!(
"trace={} requester post_json url={} headers={} proxy={}",
self.debug_trace_id().unwrap_or("none"),
crate::util::flow_debug::preview(url, 120),
headers.len(),
self.proxy
);
let mut request = self.client.post(url).version(Version::HTTP_11).json(data);
// Set custom headers
@@ -170,6 +206,14 @@ impl Requester {
data: &str,
headers: Vec<(&str, &str)>,
) -> Result<Response, wreq::Error> {
crate::flow_debug!(
"trace={} requester post url={} headers={} body_len={} proxy={}",
self.debug_trace_id().unwrap_or("none"),
crate::util::flow_debug::preview(url, 120),
headers.len(),
data.len(),
self.proxy
);
let mut request = self
.client
.post(url)
@@ -198,6 +242,13 @@ impl Requester {
headers: Vec<(String, String)>,
_http_version: Option<Version>,
) -> Result<Response, wreq::Error> {
crate::flow_debug!(
"trace={} requester post_multipart url={} headers={} proxy={}",
self.debug_trace_id().unwrap_or("none"),
crate::util::flow_debug::preview(url, 120),
headers.len(),
self.proxy
);
let http_version = match _http_version {
Some(v) => v,
None => Version::HTTP_11,
@@ -234,6 +285,14 @@ impl Requester {
headers: Vec<(String, String)>,
_http_version: Option<Version>,
) -> Result<String, AnyErr> {
crate::flow_debug!(
"trace={} requester get_with_headers start url={} headers={} http_version={:?} proxy={}",
self.debug_trace_id().unwrap_or("none"),
crate::util::flow_debug::preview(url, 120),
headers.len(),
_http_version,
self.proxy
);
let http_version = match _http_version {
Some(v) => v,
None => Version::HTTP_11,
@@ -250,10 +309,21 @@ impl Requester {
}
}
let response = request.send().await?;
crate::flow_debug!(
"trace={} requester direct response url={} status={}",
self.debug_trace_id().unwrap_or("none"),
crate::util::flow_debug::preview(url, 120),
response.status()
);
if response.status().is_success() || response.status().as_u16() == 404 {
return Ok(response.text().await?);
}
if response.status().as_u16() == 429 {
crate::flow_debug!(
"trace={} requester direct retry url={} status=429",
self.debug_trace_id().unwrap_or("none"),
crate::util::flow_debug::preview(url, 120)
);
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
continue;
} else {
@@ -276,6 +346,12 @@ impl Requester {
if self.proxy && env::var("BURP_URL").is_ok() {
flare.set_proxy(true);
}
crate::flow_debug!(
"trace={} requester flaresolverr url={} proxy={}",
self.debug_trace_id().unwrap_or("none"),
crate::util::flow_debug::preview(url, 120),
self.proxy
);
let res = flare
.solve(FlareSolverrRequest {
@@ -300,6 +376,12 @@ impl Requester {
}
self.client = Self::build_client(self.cookie_jar.clone(), self.user_agent.as_deref());
crate::flow_debug!(
"trace={} requester flaresolverr solved url={} user_agent={}",
self.debug_trace_id().unwrap_or("none"),
crate::util::flow_debug::preview(url, 120),
crate::util::flow_debug::preview(self.user_agent.as_deref().unwrap_or("unknown"), 96)
);
// Retry the original URL with the updated client & (optional) proxy
let mut request = self.client.get(url).version(Version::HTTP_11);
@@ -314,11 +396,22 @@ impl Requester {
}
let response = request.send().await?;
crate::flow_debug!(
"trace={} requester retry response url={} status={}",
self.debug_trace_id().unwrap_or("none"),
crate::util::flow_debug::preview(url, 120),
response.status()
);
if response.status().is_success() {
return Ok(response.text().await?);
}
// Fall back to FlareSolverr-provided body
crate::flow_debug!(
"trace={} requester fallback body url={}",
self.debug_trace_id().unwrap_or("none"),
crate::util::flow_debug::preview(url, 120)
);
Ok(res.solution.response)
}
}