From cacd45d893d8f1b8c8563540bf7b66b8bf346895 Mon Sep 17 00:00:00 2001 From: Simon Date: Wed, 14 Jan 2026 11:49:27 +0000 Subject: [PATCH] upgrades --- archive/spankbang.rs | 4 +- src/providers/hqporner.rs | 541 ++++++++++++++++++----------------- src/providers/javtiful.rs | 6 +- src/providers/rule34gen.rs | 2 +- src/providers/rule34video.rs | 2 +- 5 files changed, 278 insertions(+), 277 deletions(-) diff --git a/archive/spankbang.rs b/archive/spankbang.rs index 0f3091a..393276c 100644 --- a/archive/spankbang.rs +++ b/archive/spankbang.rs @@ -38,7 +38,7 @@ impl SpankbangProvider { let old_items = match cache.get(&url) { Some((time, items)) => { if time.elapsed().unwrap_or_default().as_secs() < 60 * 60 { - println!("Cache hit for URL: {}", url); + // println!("Cache hit for URL: {}", url); return Ok(items.clone()); } else{ @@ -123,7 +123,7 @@ impl SpankbangProvider { let old_items = match cache.get(&url) { Some((time, items)) => { if time.elapsed().unwrap_or_default().as_secs() < 60 * 60 { - println!("Cache hit for URL: {}", url); + // println!("Cache hit for URL: {}", url); return Ok(items.clone()); } else{ diff --git a/src/providers/hqporner.rs b/src/providers/hqporner.rs index 62b8c7f..80df10c 100644 --- a/src/providers/hqporner.rs +++ b/src/providers/hqporner.rs @@ -3,6 +3,7 @@ use crate::api::ClientVersion; use crate::providers::Provider; use crate::status::*; use crate::util::cache::VideoCache; +use crate::util::discord::{format_error_chain, send_discord_error_report}; use crate::util::requester::Requester; use crate::util::time::parse_time_to_seconds; use crate::videos::{ServerOptions, VideoFormat, VideoItem}; @@ -18,6 +19,13 @@ error_chain! { foreign_links { Io(std::io::Error); HttpRequest(wreq::Error); + Json(serde_json::Error); + } + errors { + Parse(msg: String) { + description("parse error") + display("parse error: {}", msg) + } } } @@ -27,6 +35,7 @@ pub struct HqpornerProvider { stars: Arc>>, categories: Arc>>, } + impl HqpornerProvider { pub fn new() -> Self { let provider = HqpornerProvider { @@ -38,95 +47,101 @@ impl HqpornerProvider { provider } - fn spawn_initial_load(&self) { + fn spawn_initial_load(&self) { let url = self.url.clone(); let stars = Arc::clone(&self.stars); let categories = Arc::clone(&self.categories); thread::spawn(move || { - // Create a tiny runtime just for these async tasks let rt = tokio::runtime::Builder::new_current_thread() .enable_all() - .build() - .expect("build tokio runtime"); + .build(); - rt.block_on(async move { - if let Err(e) = Self::load_stars(&url, stars).await { - eprintln!("load_stars failed: {e}"); - } - if let Err(e) = Self::load_categories(&url, categories).await { - eprintln!("load_categories failed: {e}"); - } - }); + if let Ok(runtime) = rt { + runtime.block_on(async move { + if let Err(e) = Self::load_stars(&url, stars).await { + eprintln!("load_stars failed: {e}"); + } + if let Err(e) = Self::load_categories(&url, categories).await { + eprintln!("load_categories failed: {e}"); + } + }); + } }); } async fn load_stars(base_url: &str, stars: Arc>>) -> Result<()> { let mut requester = Requester::new(); let text = requester - .get(format!("{}/girls", &base_url).as_str(), None) + .get(&format!("{}/girls", base_url), None) .await - .unwrap(); + .map_err(|e| Error::from(format!("Request failed: {}", e)))?; + let stars_div = text .split("Girls") - .collect::>().last().unwrap() - .split("") - .collect::>()[0]; - for stars_element in stars_div.split(">()[1..].to_vec() { - let star_id = stars_element.split("href=\"/actress/").collect::>()[1] - .split("\"") - .collect::>()[0] - .to_string(); - let star_name = stars_element.split(">()[1] - .split(">").collect::>()[1] - .split("<") - .collect::>()[0] - .to_string(); - Self::push_unique( - &stars, - FilterOption { - id: star_id, - title: star_name, - }, - ); + .last() + .and_then(|s| s.split("").next()) + .ok_or_else(|| Error::from("Could not find stars div"))?; + + for stars_element in stars_div.split("').nth(1)) + .and_then(|s| s.split('<').next()) + .map(|s| s.to_string()); + + if let (Some(id), Some(name)) = (star_id, star_name) { + Self::push_unique(&stars, FilterOption { id, title: name }); + } } - return Ok(()); + Ok(()) } - async fn load_categories(base_url: &str, categories: Arc>>) -> Result<()> { + async fn load_categories( + base_url: &str, + categories: Arc>>, + ) -> Result<()> { let mut requester = Requester::new(); let text = requester - .get(format!("{}/categories", &base_url).as_str(), None) + .get(&format!("{}/categories", base_url), None) .await - .unwrap(); + .map_err(|e| Error::from(format!("Request failed: {}", e)))?; + let categories_div = text .split("Categories") - .collect::>().last().unwrap() - .split("") - .collect::>()[0]; - for categories_element in categories_div.split(">()[1..].to_vec() { - let category_id = categories_element.split("href=\"/category/").collect::>()[1] - .split("\"") - .collect::>()[0] - .to_string(); - let category_name = categories_element.split(">()[1] - .split(">").collect::>()[1] - .split("<") - .collect::>()[0] - .titlecase(); - Self::push_unique( - &categories, - FilterOption { - id: category_id, - title: category_name, - }, - ); + .last() + .and_then(|s| s.split("").next()) + .ok_or_else(|| Error::from("Could not find categories div"))?; + + for categories_element in categories_div.split("').nth(1)) + .and_then(|s| s.split('<').next()) + .map(|s| s.titlecase()); + + if let (Some(id), Some(name)) = (category_id, category_name) { + Self::push_unique(&categories, FilterOption { id, title: name }); + } } - return Ok(()); + Ok(()) } - fn build_channel(&self, clientversion: ClientVersion) -> Channel { - let _ = clientversion; + fn build_channel(&self, _clientversion: ClientVersion) -> Channel { Channel { id: "hqporner".to_string(), name: "HQPorner".to_string(), @@ -134,20 +149,21 @@ impl HqpornerProvider { premium: false, favicon: "https://www.google.com/s2/favicons?sz=64&domain=hqporner.com".to_string(), status: "active".to_string(), - categories: self.categories.read().unwrap().iter().map(|c| c.title.clone()).collect(), + categories: self + .categories + .read() + .map(|c| c.iter().map(|o| o.title.clone()).collect()) + .unwrap_or_default(), options: vec![], nsfw: true, cacheDuration: None, } } - // Push one item with minimal lock time and dedup by id fn push_unique(target: &Arc>>, item: FilterOption) { if let Ok(mut vec) = target.write() { if !vec.iter().any(|x| x.id == item.id) { vec.push(item); - // Optional: keep it sorted for nicer UX - // vec.sort_by(|a,b| a.title.cmp(&b.title)); } } } @@ -156,34 +172,25 @@ impl HqpornerProvider { &self, cache: VideoCache, page: u8, - sort: &str, + _sort: &str, options: ServerOptions, ) -> Result> { - let _ = sort; let video_url = format!("{}/hdporn/{}", self.url, page); - let old_items = match cache.get(&video_url) { - Some((time, items)) => { - if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { - return Ok(items.clone()); - } else { - items.clone() - } + if let Some((time, items)) = cache.get(&video_url) { + if time.elapsed().unwrap_or_default().as_secs() < 300 { + return Ok(items.clone()); } - None => { - vec![] - } - }; + } - let mut requester = options.requester.clone().unwrap(); - let text = requester.get(&video_url, None).await.unwrap(); - let video_items: Vec = self - .get_video_items_from_html(text.clone(), &mut requester) - .await; + let mut requester = options.requester.clone().ok_or("No requester")?; + let text = requester + .get(&video_url, None) + .await + .map_err(|e| Error::from(format!("Request failed: {}", e)))?; + + let video_items = self.get_video_items_from_html(text, &mut requester).await; if !video_items.is_empty() { - cache.remove(&video_url); - cache.insert(video_url.clone(), video_items.clone()); - } else { - return Ok(old_items); + cache.insert(video_url, video_items.clone()); } Ok(video_items) } @@ -195,42 +202,41 @@ impl HqpornerProvider { query: &str, options: ServerOptions, ) -> Result> { - let search_string = query.trim().to_string(); - + let search_string = query.trim().to_lowercase(); let mut video_url = format!("{}/?q={}&p={}", self.url, search_string, page); - if let Some(star) = self.stars.read().unwrap().iter().find(|s| s.title.to_ascii_lowercase() == search_string.to_ascii_lowercase()) { - video_url = format!("{}/actress/{}/{}", self.url, star.id, page); - } - if let Some(cat) = self.categories.read().unwrap().iter().find(|c| c.title.to_ascii_lowercase() == search_string.to_ascii_lowercase()) { - video_url = format!("{}/category/{}/{}", self.url, cat.id, page); - } - // Check our Video Cache. If the result is younger than 1 hour, we return it. - let old_items = match cache.get(&video_url) { - Some((time, items)) => { - if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { - return Ok(items.clone()); - } else { - let _ = cache.check().await; - return Ok(items.clone()); - } + if let Ok(stars) = self.stars.read() { + if let Some(star) = stars + .iter() + .find(|s| s.title.to_lowercase() == search_string) + { + video_url = format!("{}/actress/{}/{}", self.url, star.id, page); } - None => { - vec![] + } + if let Ok(cats) = self.categories.read() { + if let Some(cat) = cats + .iter() + .find(|c| c.title.to_lowercase() == search_string) + { + video_url = format!("{}/category/{}/{}", self.url, cat.id, page); } - }; + } - let mut requester = options.requester.clone().unwrap(); + if let Some((time, items)) = cache.get(&video_url) { + if time.elapsed().unwrap_or_default().as_secs() < 300 { + return Ok(items.clone()); + } + } - let text = requester.get(&video_url, None).await.unwrap(); - let video_items: Vec = self - .get_video_items_from_html(text.clone(), &mut requester) - .await; + let mut requester = options.requester.clone().ok_or("No requester")?; + let text = requester + .get(&video_url, None) + .await + .map_err(|e| Error::from(format!("Request failed: {}", e)))?; + + let video_items = self.get_video_items_from_html(text, &mut requester).await; if !video_items.is_empty() { - cache.remove(&video_url); - cache.insert(video_url.clone(), video_items.clone()); - } else { - return Ok(old_items); + cache.insert(video_url, video_items.clone()); } Ok(video_items) } @@ -243,185 +249,188 @@ impl HqpornerProvider { if html.is_empty() || html.contains("404 Not Found") { return vec![]; } - let raw_videos = html.split("id=\"footer\"").collect::>()[0] - .split("") - .collect::>()[2] - .split("") - .collect::>()[1..] - .to_vec(); + + let raw_videos: Vec = html + .split("id=\"footer\"") + .next() + .and_then(|s| s.split("").nth(2)) + .map(|s| { + s.split("") + .skip(1) + .map(|v| v.to_string()) + .collect() + }) + .unwrap_or_default(); let futures = raw_videos .into_iter() - .map(|el| self.get_video_item(el.to_string(), requester.clone())); - let results: Vec> = join_all(futures).await; - let video_items: Vec = results.into_iter().filter_map(Result::ok).collect(); - return video_items; + .map(|el| self.get_video_item(el, requester.clone())); + + join_all(futures) + .await + .into_iter() + .inspect(|r| { + if let Err(e) = r { + let msg = e.to_string(); + let chain = format_error_chain(e); + tokio::spawn(async move { + let _ = send_discord_error_report( + msg, + Some(chain), + Some("Hqporner Provider"), + None, + file!(), + line!(), + module_path!(), + ) + .await; + }); + } + }) + .filter_map(Result::ok) + .collect() } - async fn get_video_item( - &self, - video_segment: String, - mut requester: Requester, - ) -> Result { - let video_url: String = format!( + async fn get_video_item(&self, seg: String, mut requester: Requester) -> Result { + let video_url = format!( "{}{}", self.url, - video_segment.split(">()[1] - .split("\"") - .collect::>()[0] - .to_string() + seg.split("") - .collect::>()[1] - .split(">") - .collect::>()[1] - .split("<") - .collect::>()[0] - .trim() - .to_string(); - // html decode - title = decode(title.as_bytes()) + .nth(1) + .and_then(|s| s.split('>').nth(1)) + .and_then(|s| s.split('<').next()) + .ok_or_else(|| ErrorKind::Parse(format!("title \n{seg}").into()))?; + let title = decode(title_raw.as_bytes()) .to_string() - .unwrap_or(title) + .unwrap_or_else(|_| title_raw.to_string()) .titlecase(); - let id = video_url.split("/").collect::>()[4] - .split(".") - .collect::>()[0] - .to_string(); + let id = video_url + .split('/') + .nth(4) + .and_then(|s| s.split('.').next()) + .ok_or_else(|| ErrorKind::Parse(format!("id \n{seg}").into()))? + .to_string(); let thumb = format!( "https:{}", - video_segment - .split("onmouseleave='defaultImage(\"") - .collect::>()[1] - .split("\"") - .collect::>()[0] - .to_string() + seg.split("onmouseleave='defaultImage(\"") + .nth(1) + .and_then(|s| s.split('"').next()) + .ok_or_else(|| ErrorKind::Parse(format!("thumb \n{seg}").into()))? ); - let raw_duration = video_segment + let raw_duration = seg .split("") - .collect::>()[1] - .split("s<") - .collect::>()[0] - .replace("m ", ":") - .to_string(); - let duration = parse_time_to_seconds(raw_duration.as_str()).unwrap_or(0) as u32; + .nth(1) + .and_then(|s| s.split("s<").next()) + .map(|s| s.replace("m ", ":")) + .unwrap_or_default(); + let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32; - let (tags, formats) = match self.extract_media(&video_url, &mut requester).await{ - Ok((t, f)) => (t, f), - Err(_) => return Err(Error::from("Video media extraction failed")), - }; - if formats.is_empty() { - return Err(Error::from("No formats found for video")); - } - let video_item = VideoItem::new( - id, - title, - video_url, - "hqporner".to_string(), - thumb, - duration, + let (tags, formats) = self.extract_media(&video_url, &mut requester).await?; + + Ok( + VideoItem::new(id, title, video_url, "hqporner".into(), thumb, duration) + .formats(formats) + .tags(tags), ) - .formats(formats) - .tags(tags); - return Ok(video_item); } async fn extract_media( &self, - video_page_url: &str, + url: &str, requester: &mut Requester, ) -> Result<(Vec, Vec)> { let mut formats = vec![]; let mut tags = vec![]; - let text = requester.get_raw_with_headers(&video_page_url, vec![("Referer".to_string(), "https://hqporner.com".to_string())]) - .await.unwrap().text().await.unwrap(); + let resp = requester + .get_raw_with_headers( + url, + vec![("Referer".to_string(), "https://hqporner.com".into())], + ) + .await + .map_err(|e| Error::from(format!("Request failed: {}", e)))?; + let text = resp + .text() + .await + .map_err(|e| Error::from(format!("Text conversion failed: {}", e)))?; + if text.contains("Why do I see it?") { return Ok((tags, formats)); } - let stars_elements = text.split("icon fa-star-o").collect::>()[1] - .split("") - .collect::>()[0] - .split("href=\"/actress/") - .collect::>()[1..] - .to_vec(); - for star_el in stars_elements { - let star_id = star_el.split("\"").collect::>()[0].to_string(); - let star_name = star_el.split("\">").collect::>()[1] - .split("<") - .collect::>()[0] - .to_string(); - tags.push(star_name.clone()); - Self::push_unique(&self.stars, FilterOption { - id: star_id, - title: star_name.clone(), - }); - } - let categories_elements = text.split("This video belongs to the following categories").collect::>()[1] - .split("