use crate::DbPool; use crate::api::ClientVersion; use crate::providers::Provider; use crate::status::*; use crate::util::cache::VideoCache; use crate::util::discord::{format_error_chain, send_discord_error_report}; use crate::util::requester::Requester; use crate::util::time::parse_time_to_seconds; use crate::videos::{ServerOptions, VideoFormat, VideoItem}; use async_trait::async_trait; use error_chain::error_chain; use futures::stream::{FuturesUnordered, StreamExt}; use htmlentity::entity::{ICodedDataTrait, decode}; use std::sync::{Arc, RwLock}; use std::{thread, vec}; use titlecase::Titlecase; pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata = crate::providers::ProviderChannelMetadata { group_id: "studio-network", tags: &["studio", "hd", "scenes"], }; error_chain! { foreign_links { Io(std::io::Error); HttpRequest(wreq::Error); Json(serde_json::Error); } errors { Parse(msg: String) { description("parse error") display("parse error: {}", msg) } } } #[derive(Debug, Clone)] pub struct HqpornerProvider { url: String, stars: Arc>>, categories: Arc>>, } impl HqpornerProvider { pub fn new() -> Self { let provider = HqpornerProvider { url: "https://hqporner.com".to_string(), stars: Arc::new(RwLock::new(vec![])), categories: Arc::new(RwLock::new(vec![])), }; provider.spawn_initial_load(); provider } fn spawn_initial_load(&self) { let url = self.url.clone(); let stars = Arc::clone(&self.stars); let categories = Arc::clone(&self.categories); thread::spawn(move || { let rt = tokio::runtime::Builder::new_current_thread() .enable_all() .build(); if let Ok(runtime) = rt { runtime.block_on(async move { if let Err(e) = Self::load_stars(&url, stars).await { eprintln!("load_stars failed: {e}"); } if let Err(e) = Self::load_categories(&url, categories).await { eprintln!("load_categories failed: {e}"); } }); } }); } async fn load_stars(base_url: &str, stars: Arc>>) -> Result<()> { let mut requester = Requester::new(); let text = requester .get(&format!("{}/girls", base_url), None) .await .map_err(|e| Error::from(format!("Request failed: {}", e)))?; let stars_div = text .split("Girls") .last() .and_then(|s| s.split("").next()) .ok_or_else(|| Error::from("Could not find stars div"))?; for stars_element in stars_div.split("
  • ').nth(1)) .and_then(|s| s.split('<').next()) .map(|s| s.to_string()); if let (Some(id), Some(name)) = (star_id, star_name) { Self::push_unique(&stars, FilterOption { id, title: name }); } } Ok(()) } async fn load_categories( base_url: &str, categories: Arc>>, ) -> Result<()> { let mut requester = Requester::new(); let text = requester .get(&format!("{}/categories", base_url), None) .await .map_err(|e| Error::from(format!("Request failed: {}", e)))?; let categories_div = text .split("Categories") .last() .and_then(|s| s.split("").next()) .ok_or_else(|| Error::from("Could not find categories div"))?; for categories_element in categories_div.split("
  • ').nth(1)) .and_then(|s| s.split('<').next()) .map(|s| s.titlecase()); if let (Some(id), Some(name)) = (category_id, category_name) { Self::push_unique(&categories, FilterOption { id, title: name }); } } Ok(()) } fn build_channel(&self, _clientversion: ClientVersion) -> Channel { Channel { id: "hqporner".to_string(), name: "HQPorner".to_string(), description: "HD Porn Videos Tube".to_string(), premium: false, favicon: "https://www.google.com/s2/favicons?sz=64&domain=hqporner.com".to_string(), status: "active".to_string(), categories: self .categories .read() .map(|c| c.iter().map(|o| o.title.clone()).collect()) .unwrap_or_default(), options: vec![], nsfw: true, cacheDuration: None, } } fn push_unique(target: &Arc>>, item: FilterOption) { if let Ok(mut vec) = target.write() { if !vec.iter().any(|x| x.id == item.id) { vec.push(item); } } } async fn get( &self, cache: VideoCache, page: u8, _sort: &str, options: ServerOptions, ) -> Result> { let video_url = format!("{}/hdporn/{}", self.url, page); if let Some((time, items)) = cache.get(&video_url) { if time.elapsed().unwrap_or_default().as_secs() < 300 { return Ok(items.clone()); } } let mut requester = options.requester.clone().ok_or("No requester")?; let text = requester .get(&video_url, None) .await .map_err(|e| Error::from(format!("Request failed: {}", e)))?; let video_items = self .get_video_items_from_html(text, &mut requester, &options) .await; if !video_items.is_empty() { cache.insert(video_url, video_items.clone()); } Ok(video_items) } async fn query( &self, cache: VideoCache, page: u8, query: &str, options: ServerOptions, ) -> Result> { let search_string = query.trim().to_lowercase(); let mut video_url = format!("{}/?q={}&p={}", self.url, search_string, page); if let Ok(stars) = self.stars.read() { if let Some(star) = stars .iter() .find(|s| s.title.to_lowercase() == search_string) { video_url = format!("{}/actress/{}/{}", self.url, star.id, page); } } if let Ok(cats) = self.categories.read() { if let Some(cat) = cats .iter() .find(|c| c.title.to_lowercase() == search_string) { video_url = format!("{}/category/{}/{}", self.url, cat.id, page); } } if let Some((time, items)) = cache.get(&video_url) { if time.elapsed().unwrap_or_default().as_secs() < 300 { return Ok(items.clone()); } } let mut requester = options.requester.clone().ok_or("No requester")?; let text = requester .get(&video_url, None) .await .map_err(|e| Error::from(format!("Request failed: {}", e)))?; let video_items = self .get_video_items_from_html(text, &mut requester, &options) .await; if !video_items.is_empty() { cache.insert(video_url, video_items.clone()); } Ok(video_items) } async fn get_video_items_from_html( &self, html: String, requester: &mut Requester, options: &ServerOptions, ) -> Vec { if html.is_empty() || html.contains("404 Not Found") { return vec![]; } let raw_videos: Vec = html .split("id=\"footer\"") .next() .and_then(|s| s.split("
    ").nth(2)) .map(|s| { s.split("
    ") .skip(1) .map(|v| v.to_string()) .collect() }) .unwrap_or_default(); // Limit concurrent detail-page requests to reduce transient connect errors. let mut in_flight = FuturesUnordered::new(); let mut iter = raw_videos.into_iter(); let mut items = Vec::new(); const MAX_IN_FLIGHT: usize = 6; loop { while in_flight.len() < MAX_IN_FLIGHT { let Some(seg) = iter.next() else { break; }; in_flight.push(self.get_video_item(seg, requester.clone(), options)); } let Some(result) = in_flight.next().await else { break; }; match result { Ok(item) if item .formats .as_ref() .map(|formats| !formats.is_empty()) .unwrap_or(false) => { items.push(item); } Ok(_) => {} Err(e) => { let msg = e.to_string(); let chain = format_error_chain(&e); tokio::spawn(async move { let _ = send_discord_error_report( msg, Some(chain), Some("Hqporner Provider"), None, file!(), line!(), module_path!(), ) .await; }); } } } items } async fn get_video_item( &self, seg: String, mut requester: Requester, options: &ServerOptions, ) -> Result { let video_url = format!( "{}{}", self.url, seg.split("