use crate::api::ClientVersion; use crate::providers::Provider; use crate::status::*; use crate::util::cache::VideoCache; use crate::util::discord::{format_error_chain, send_discord_error_report}; use crate::util::requester::Requester; use crate::videos::{ServerOptions, VideoFormat, VideoItem}; use crate::{DbPool, db}; use async_trait::async_trait; use error_chain::error_chain; use futures::future::join_all; use htmlentity::entity::{ICodedDataTrait, decode}; use std::sync::{Arc, RwLock}; use std::vec; use titlecase::Titlecase; use wreq::Version; error_chain! { foreign_links { Io(std::io::Error); HttpRequest(wreq::Error); Json(serde_json::Error); } errors { Parse(msg: String) { description("parse error") display("parse error: {}", msg) } } } #[derive(Debug, Clone)] pub struct HentaihavenProvider { url: String, categories: Arc>>, } impl HentaihavenProvider { pub fn new() -> Self { let provider = Self { url: "https://hentaihaven.xxx".to_string(), categories: Arc::new(RwLock::new(vec![])), }; provider } fn build_channel(&self, clientversion: ClientVersion) -> Channel { let _ = clientversion; Channel { id: "hentaihaven".to_string(), name: "Hentai Haven".to_string(), description: "Watch Free Hentai Videos HD!".to_string(), premium: false, favicon: "https://www.google.com/s2/favicons?sz=64&domain=hentaihaven.xxx".to_string(), status: "active".to_string(), categories: self .categories .read() .unwrap() .iter() .map(|c| c.title.clone()) .collect(), options: vec![], nsfw: true, cacheDuration: None, } } fn push_unique(target: &Arc>>, item: FilterOption) { if let Ok(mut vec) = target.write() { if !vec.iter().any(|x| x.id == item.id) { vec.push(item); } } } async fn get( &self, cache: VideoCache, page: u8, sort: &str, options: ServerOptions, pool: DbPool, ) -> Result> { let _ = sort; let video_url = format!("{}/hentai/page/{}/", self.url, page); let old_items = match cache.get(&video_url) { Some((time, items)) => { if time.elapsed().unwrap_or_default().as_secs() < 60 * 60 * 24 { return Ok(items.clone()); } else { items.clone() } } None => { vec![] } }; let mut requester = options.requester.clone().unwrap(); let text = requester .get(&video_url, Some(Version::HTTP_2)) .await .unwrap(); if page > 1 && !text.contains(&format!( "
  • {}", page )) { return Ok(vec![]); } let video_items: Vec = self .get_video_items_from_html(text.clone(), &mut requester, pool.clone()) .await; if !video_items.is_empty() { cache.remove(&video_url); cache.insert(video_url.clone(), video_items.clone()); } else { return Ok(old_items); } Ok(video_items) } async fn query( &self, cache: VideoCache, page: u8, query: &str, options: ServerOptions, pool: DbPool, ) -> Result> { let video_url = format!( "{}/?s={}", self.url, query.replace(" ", "+"), ); // Check our Video Cache. If the result is younger than 1 hour, we return it. let old_items = match cache.get(&video_url) { Some((time, items)) => { if time.elapsed().unwrap_or_default().as_secs() < 60 * 60 * 24 { return Ok(items.clone()); } else { let _ = cache.check().await; return Ok(items.clone()); } } None => { vec![] } }; let mut requester = options.requester.clone().unwrap(); let text = requester .get(&video_url, Some(Version::HTTP_2)) .await .unwrap(); if page > 1 { return Ok(vec![]); } let video_items: Vec = self .get_video_items_from_html_search(text.clone(), &mut requester, pool) .await; if !video_items.is_empty() { cache.remove(&video_url); cache.insert(video_url.clone(), video_items.clone()); } else { return Ok(old_items); } Ok(video_items) } async fn get_video_items_from_html( &self, html: String, requester: &mut Requester, pool: DbPool, ) -> Vec { if html.is_empty() || html.contains("404 Not Found") { return vec![]; } let block = match html .split("previouspostslink") .next() .and_then(|s| s.split("vraven_manga_list").nth(1)) { Some(b) => b, None => { eprint!("Hentai Haven Provider: Failed to get block from html"); let e = Error::from(ErrorKind::Parse("html".into())); send_discord_error_report( e.to_string(), Some(format_error_chain(&e)), Some("Hentai Haven Provider"), Some(&format!("Failed to get block from html:\n```{html}\n```")), file!(), line!(), module_path!(), ) .await; return vec![]; } }; let futures = block .split("id=\"manga-item-") .skip(1) .map(|el| self.get_video_item(el.to_string(), pool.clone(), requester.clone())); join_all(futures) .await .into_iter() .inspect(|r| { if let Err(e) = r { eprint!("Hentai Haven Provider: Failed to get video item:{}\n", e); // Prepare data to move into the background task let msg = e.to_string(); let chain = format_error_chain(&e); // Spawn the report into the background - NO .await here tokio::spawn(async move { let _ = send_discord_error_report( msg, Some(chain), Some("Hentai Haven Provider"), Some("Failed to get video item"), file!(), // Note: these might report the utility line line!(), // better to hardcode or pass from outside module_path!(), ) .await; }); } }) .filter_map(Result::ok) .collect() } async fn get_video_items_from_html_search( &self, html: String, requester: &mut Requester, pool: DbPool, ) -> Vec { if html.is_empty() || html.contains("404 Not Found") { return vec![]; } let block = match html .split(" b, None => { eprint!("Hentai Haven Provider: Failed to get block from html"); let e = Error::from(ErrorKind::Parse("html".into())); send_discord_error_report( e.to_string(), Some(format_error_chain(&e)), Some("Hentai Haven Provider"), Some(&format!("Failed to get block from html:\n```{html}\n```")), file!(), line!(), module_path!(), ) .await; return vec![]; } }; let futures = block .split("c-tabs-item__content col-6 col-md-12") .skip(1) .map(|el| self.get_video_item(el.to_string(), pool.clone(), requester.clone())); join_all(futures) .await .into_iter() .inspect(|r| { if let Err(e) = r { eprint!("Hentai Haven Provider: Failed to get video item:{}\n", e); // Prepare data to move into the background task let msg = e.to_string(); let chain = format_error_chain(&e); // Spawn the report into the background - NO .await here tokio::spawn(async move { let _ = send_discord_error_report( msg, Some(chain), Some("Hentai Haven Provider"), Some("Failed to get video item"), file!(), // Note: these might report the utility line line!(), // better to hardcode or pass from outside module_path!(), ) .await; }); } }) .filter_map(Result::ok) .collect() } async fn get_video_item( &self, seg: String, pool: DbPool, mut requester: Requester, ) -> Result { let video_url = seg .split("a href=\"") .nth(1) .and_then(|s| s.split('"').next()) .ok_or_else(|| ErrorKind::Parse("video url\n\n{seg}".into()))? .to_string(); let mut conn = pool.get().expect("couldn't get db connection from pool"); let db_result = db::get_video(&mut conn, video_url.clone()); drop(conn); match db_result { Ok(Some(video)) => { let video_item = VideoItem::from(video); match video_item { Ok(item) => return Ok(item), Err(e) => { eprint!("Failed to convert video from DB result: {}\n", e); } } } Ok(None) => { // continue to fetch and parse the video } Err(e) => { eprint!("Database error: {}\n", e); // continue to fetch and parse the video even if there's a DB error } } let html = requester .get(&video_url, Some(Version::HTTP_2)) .await .map_err(|e| Error::from(format!("Failed to fetch video page: {}", e)))?; let mut title = html .split("

    ") .nth(1) .and_then(|s| s.split("

    ").next()) .ok_or_else(|| ErrorKind::Parse(format!("video title\n\n{seg}").into()))? .trim() .to_string(); title = decode(title.as_bytes()) .to_string() .unwrap_or(title) .titlecase(); let id = video_url .split('/') .nth(4) .and_then(|s| s.split('.').next()) .ok_or_else(|| ErrorKind::Parse("video id\n\n{seg}".into()))? .to_string(); let thumb = html .split("og:image\" content=\"") .nth(1) .and_then(|s| s.split('"').next()) .unwrap_or("") .to_string(); let raw_tags: Vec = html .split("Genre(s)") .nth(1) .unwrap_or_default() .split("Release") .nth(0) .unwrap_or_default() .split("a href=\"") .skip(1) .map(|tag_block| { let id = tag_block .split("\"") .nth(1) .and_then(|s| s.split('"').next()) .unwrap_or("") .to_string(); let title = tag_block .split('>') .nth(1) .and_then(|s| s.split('<').next()) .map(|s| { decode(s.as_bytes()) .to_string() .unwrap_or(s.to_string()) .titlecase() }) .unwrap_or("".to_string()); FilterOption { id: id.to_ascii_lowercase().replace(" ", "+"), title: title.clone(), } }) .collect::>(); for tag in &raw_tags { Self::push_unique(&self.categories, tag.clone()); } let tags = raw_tags.into_iter().map(|t| t.title).collect(); let views = html .split("Viewed") .last() .and_then(|s| s.split("summary-content\">").nth(1)) .and_then(|s| s.split(" Total").nth(0)) .map(|s| s.trim().parse::().unwrap_or(0)) .unwrap_or(0); let mut formats = vec![]; let episode_block = html .split("manga-chapters-holder") .nth(1) .unwrap_or_default() .split("vraven_read") .nth(0) .unwrap_or_default(); for episode in episode_block.split("wp-manga-chapter").skip(1) { let ep_thumbnail = episode .split(" src=\"") .nth(1) .and_then(|s| s.split('"').next()) .unwrap_or_default(); let episode_title = episode .split("
    ") .nth(1) .and_then(|s| s.split('<').next()) .unwrap_or_default() .trim() .to_string(); let episode_id = ep_thumbnail.split('/').nth(5).unwrap_or_default(); let episode_url = format!( "https://master-lengs.org/api/v3/hh/{}/master.m3u8", episode_id ); let format = VideoFormat::new(episode_url, "1080p".to_string(), "m3u8".to_string()) .format_id(episode_title) .http_header("Connection".to_string(), "keep-alive".to_string()) .http_header("User-Agent".to_string(), "Mozilla/5.0 (X11; Linux x86_64; rv:140.0) Gecko/20100101 Firefox/140.0".to_string()) .http_header("Accept".to_string(), "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8".to_string()) .http_header("Accept-Language".to_string(), "en-US,en;q=0.5".to_string()) .http_header("Accept-Encoding".to_string(), "gzip, deflate, br".to_string()) .http_header("Sec-Fetch-Mode".to_string(), "navigate".to_string()) .http_header("Origin".to_string(), self.url.clone()) ; formats.push(format); } if formats.is_empty() { let e = Error::from(format!("No formats found for video URL: {}", video_url)); return Err(e); } if formats.len() > 1 { title = format!("{} ({} Episodes)", title, formats.len()); } let video_item = VideoItem::new(id, title, video_url.clone(), "hentaihaven".into(), thumb, 0) .formats(formats) .tags(tags) .views(views); let mut conn = pool.get().expect("couldn't get db connection from pool"); let _ = db::insert_video( &mut conn, &video_url, &serde_json::to_string(&video_item).unwrap_or_default(), ); drop(conn); Ok(video_item) } } #[async_trait] impl Provider for HentaihavenProvider { async fn get_videos( &self, cache: VideoCache, pool: DbPool, sort: String, query: Option, page: String, _per_page: String, options: ServerOptions, ) -> Vec { let page = page.parse::().unwrap_or(1); let res = match query { Some(q) => self.to_owned().query(cache, page, &q, options, pool).await, None => self.get(cache, page, &sort, options, pool).await, }; res.unwrap_or_else(|e| { eprintln!("hentai haven error: {e}"); vec![] }) } fn get_channel(&self, v: ClientVersion) -> Option { Some(self.build_channel(v)) } }