diff --git a/archive/tube8.rs b/archive/tube8.rs new file mode 100644 index 0000000..3c49de9 --- /dev/null +++ b/archive/tube8.rs @@ -0,0 +1,448 @@ +use crate::DbPool; +use crate::api::ClientVersion; +use crate::providers::Provider; +use crate::status::*; +use crate::util::cache::VideoCache; +use crate::util::parse_abbreviated_number; +use crate::util::time::parse_time_to_seconds; +use crate::videos::{ServerOptions, VideoItem}; +use async_trait::async_trait; +use error_chain::error_chain; +use htmlentity::entity::{ICodedDataTrait, decode}; +use std::sync::{Arc, RwLock}; +use std::vec; + +error_chain! { + foreign_links { + Io(std::io::Error); + HttpRequest(wreq::Error); + } +} + +#[derive(Debug, Clone)] +pub struct Tube8Provider { + url: String, + sites: Arc>>, + stars: Arc>>, +} +impl Tube8Provider { + pub fn new() -> Self { + let provider = Tube8Provider { + url: "https://www.tube8.com".to_string(), + sites: Arc::new(RwLock::new(vec![FilterOption { + id: "all".to_string(), + title: "All".to_string(), + }])), + stars: Arc::new(RwLock::new(vec![FilterOption { + id: "all".to_string(), + title: "All".to_string(), + }])), + }; + + // Kick off the background load but return immediately + provider + } + + // Push one item with minimal lock time and dedup by id + fn push_unique(target: &Arc>>, item: FilterOption) { + if let Ok(mut vec) = target.write() { + if !vec.iter().any(|x| x.id == item.id) { + vec.push(item); + // Optional: keep it sorted for nicer UX + // vec.sort_by(|a,b| a.title.cmp(&b.title)); + } + } + } + + fn build_channel(&self, clientversion: ClientVersion) -> Channel { + let _ = clientversion; + let sites: Vec = self + .sites + .read() + .map(|g| g.clone()) // or: .map(|g| g.to_vec()) + .unwrap_or_default(); // or: .unwrap_or_else(|_| Vec::new()) + + let stars: Vec = self + .stars + .read() + .map(|g| g.clone()) // or: .map(|g| g.to_vec()) + .unwrap_or_default(); // or: .unwrap_or_else(|_| Vec::new()) + + Channel { + id: "tube8".to_string(), + name: "Tube8".to_string(), + description: "Tube8 Videos".to_string(), + premium: false, + favicon: "https://www.google.com/s2/favicons?sz=64&domain=www.tube8.com".to_string(), + status: "active".to_string(), + categories: vec![], + options: vec![ + ChannelOption { + id: "sort".to_string(), + title: "Sort".to_string(), + description: "Sort the Videos".to_string(), + systemImage: "list.number".to_string(), + colorName: "blue".to_string(), + options: vec![ + FilterOption { + id: "rating".into(), + title: "Rating".into(), + }, + FilterOption { + id: "mostviewed.html".into(), + title: "Most Viewed".into(), + }, + FilterOption { + id: "longest.html".into(), + title: "Duration".into(), + }, + FilterOption { + id: "newest.html".into(), + title: "Newest".into(), + }, + ], + multiSelect: false, + }, + ChannelOption { + id: "sites".to_string(), + title: "Sites".to_string(), + description: "Filter for different Sites".to_string(), + systemImage: "rectangle.stack".to_string(), + colorName: "green".to_string(), + options: sites, + multiSelect: false, + }, + ChannelOption { + id: "stars".to_string(), + title: "Stars".to_string(), + description: "Filter for different Pornstars".to_string(), + systemImage: "star.fill".to_string(), + colorName: "yellow".to_string(), + options: stars, + multiSelect: false, + }, + ], + nsfw: true, + cacheDuration: None, + } + } + + async fn get( + &self, + cache: VideoCache, + page: u8, + sort: &str, + options: ServerOptions, + ) -> Result> { + let mut sort_string: String = match sort { + "mostviewed.html" => "mostviewed.html/".to_string(), + "longest.html" => "longest.html/".to_string(), + "newest.html" => "newest.html/".to_string(), + _ => "".to_string(), + }; + if options.sites.is_some() + && !options.sites.as_ref().unwrap().is_empty() + && options.sites.as_ref().unwrap() != "all" + { + sort_string = match sort { + "mostviewed.html" => "?orderBy=mv&page=".to_string(), + "longest.html" => "?orderBy=ln&page=".to_string(), + "newest.html" => "?page=".to_string(), + _ => "?orderBy=tr&page=".to_string(), + }; + } + if options.stars.is_some() + && !options.stars.as_ref().unwrap().is_empty() + && options.stars.as_ref().unwrap() != "all" + { + sort_string = match sort { + "mostviewed.html" => "views/?page=".to_string(), + "longest.html" => "duration/?page=".to_string(), + "newest.html" => "?page=".to_string(), + _ => "rating/?page=".to_string(), + }; + } + let video_url = format!("{}/{}{}", self.url, sort_string, page); + let old_items = match cache.get(&video_url) { + Some((time, items)) => { + if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { + return Ok(items.clone()); + } else { + items.clone() + } + } + None => { + vec![] + } + }; + + let mut requester = options.requester.clone().unwrap(); + let text = requester.get(&video_url, None).await.unwrap(); + let video_items: Vec = self.get_video_items_from_html(text.clone()); + if !video_items.is_empty() { + cache.remove(&video_url); + cache.insert(video_url.clone(), video_items.clone()); + } else { + return Ok(old_items); + } + Ok(video_items) + } + + async fn query( + &self, + cache: VideoCache, + page: u8, + query: &str, + options: ServerOptions, + ) -> Result> { + let mut sort_string: String = match options.sort.as_ref().unwrap().as_str() { + "mostviewed.html" => "&orderby=views&page=".to_string(), + "longest.html" => "&orderby=longest&page=".to_string(), + "newest.html" => "&orderby=newest&page=".to_string(), + _ => "&orderby=rating&page=".to_string(), + }; + let mut search_string = query.to_string().to_ascii_lowercase().trim().to_string(); + let mut video_url = format!( + "{}/searches.html/?q={}{}{}", + self.url, query, sort_string, page + ); + video_url = video_url.replace(" ", "+"); + match self + .stars + .read() + .unwrap() + .iter() + .find(|s| s.title.to_ascii_lowercase() == search_string) + { + Some(star) => { + sort_string = match options.sort.as_ref().unwrap().as_str() { + "mostviewed.html" => "views/?page=".to_string(), + "longest.html" => "duration/?page=".to_string(), + "newest.html" => "?page=".to_string(), + _ => "rating/?page=".to_string(), + }; + video_url = format!("{}/{}{}{}", self.url, star.id, sort_string, page); + } + _ => {} + } + match self + .sites + .read() + .unwrap() + .iter() + .find(|s| s.title.to_ascii_lowercase() == search_string) + { + Some(site) => { + sort_string = match options.sort.as_ref().unwrap().as_str() { + "mostviewed.html" => "?orderBy=mv&page=".to_string(), + "longest.html" => "?orderBy=ln&page=".to_string(), + "newest.html" => "?page=".to_string(), + _ => "?orderBy=tr&page=".to_string(), + }; + video_url = format!("{}/{}{}{}", self.url, site.id, sort_string, page); + } + _ => {} + } + // Check our Video Cache. If the result is younger than 1 hour, we return it. + let old_items = match cache.get(&video_url) { + Some((time, items)) => { + if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { + return Ok(items.clone()); + } else { + let _ = cache.check().await; + return Ok(items.clone()); + } + } + None => { + vec![] + } + }; + + let mut requester = options.requester.clone().unwrap(); + + let text = requester.get(&video_url, None).await.unwrap(); + let video_items: Vec = self.get_video_items_from_html(text.clone()); + if !video_items.is_empty() { + cache.remove(&video_url); + cache.insert(video_url.clone(), video_items.clone()); + } else { + return Ok(old_items); + } + Ok(video_items) + } + + fn get_video_items_from_html(&self, html: String) -> Vec { + if html.is_empty() { + println!("HTML is empty"); + return vec![]; + } + let mut items: Vec = Vec::new(); + if !html.contains("video-box ") { + return items; + } + let raw_videos = html.split("id=\"pagination\"").collect::>()[0] + .split("-thumbs") + .collect::>()[1] + .split("video-box ") + .collect::>()[1..] + .to_vec(); + for video_segment in &raw_videos { + // let vid = video_segment.split("\n").collect::>(); + // for (index, line) in vid.iter().enumerate() { + // println!("Line {}: {}", index, line); + // } + let video_url: String = format!("{}{}", self.url, video_segment.split(">()[1] + .split("\"") + .collect::>()[0] + .to_string()); + let mut title = video_segment.split("alt=\"").collect::>()[1] + .split("\"") + .collect::>()[0] + .to_string(); + // html decode + title = decode(title.as_bytes()).to_string().unwrap_or(title); + let id = video_url.split("/").collect::>()[4].to_string(); + + let thumb = match video_segment.split("thumb-image ").collect::>()[1] + .contains("data-src=\"") + { + true => video_segment.split("thumb-image ").collect::>()[1] + .split("data-src=\"") + .collect::>()[1] + .split("\"") + .collect::>()[0] + .to_string(), + false => video_segment.split("thumb-image ").collect::>()[1] + .split("data-poster=\"") + .collect::>()[1] + .split("\"") + .collect::>()[0] + .to_string(), + }; + let raw_duration = video_segment + .split("video-duration ") + .collect::>()[1] + .split("") + .collect::>()[0] + .split("") + .collect::>() + .last() + .unwrap_or(&"") + .replace("\n", "") + .trim() + .to_string(); + let duration = parse_time_to_seconds(raw_duration.as_str()).unwrap_or(0) as u32; + let views = parse_abbreviated_number( + video_segment + .split("") + .collect::>()[1] + .split("<") + .collect::>()[0] + .to_string() + .as_str(), + ) + .unwrap_or(0) as u32; + let site_name = title + .split("]") + .collect::>() + .first() + .unwrap_or(&"") + .trim_start_matches("["); + let site_id = self + .get_site_id_from_name(site_name) + .unwrap_or("".to_string()); + let mut tags = match video_segment.contains("class=\"models\">") { + true => video_segment + .split("class=\"models\">") + .collect::>()[1] + .split("") + .collect::>()[0] + .split("href=\"") + .collect::>()[1..] + .into_iter() + .map(|s| { + Self::push_unique( + &self.stars, + FilterOption { + id: s.split("/").collect::>()[4].to_string(), + title: s.split(">").collect::>()[1] + .split("<") + .collect::>()[0] + .trim() + .to_string(), + }, + ); + s.split(">").collect::>()[1] + .split("<") + .collect::>()[0] + .trim() + .to_string() + }) + .collect::>() + .to_vec(), + false => vec![], + }; + if !site_id.is_empty() { + Self::push_unique( + &self.sites, + FilterOption { + id: site_id, + title: site_name.to_string(), + }, + ); + tags.push(site_name.to_string()); + } + + let video_item = VideoItem::new( + id, + title, + video_url.to_string(), + "omgxxx".to_string(), + thumb, + duration, + ) + .views(views) + .preview(preview) + .tags(tags); + items.push(video_item); + } + return items; + } +} + +#[async_trait] +impl Provider for Tube8Provider { + async fn get_videos( + &self, + cache: VideoCache, + pool: DbPool, + sort: String, + query: Option, + page: String, + per_page: String, + options: ServerOptions, + ) -> Vec { + let _ = per_page; + let _ = pool; + let videos: std::result::Result, Error> = match query { + Some(q) => { + self.query(cache, page.parse::().unwrap_or(1), &q, options) + .await + } + None => { + self.get(cache, page.parse::().unwrap_or(1), &sort, options) + .await + } + }; + match videos { + Ok(v) => v, + Err(e) => { + println!("Error fetching videos: {}", e); + vec![] + } + } + } + fn get_channel(&self, clientversion: ClientVersion) -> Option { + Some(self.build_channel(clientversion)) + } +} diff --git a/src/providers/hentaihaven.rs b/src/providers/hentaihaven.rs index 0ded2ab..2f92bad 100644 --- a/src/providers/hentaihaven.rs +++ b/src/providers/hentaihaven.rs @@ -1,62 +1,93 @@ use crate::api::ClientVersion; use crate::providers::Provider; -use crate::schema::videos::url; -use crate::status::{Channel, ChannelOption, FilterOption}; +use crate::status::*; use crate::util::cache::VideoCache; -use crate::util::flaresolverr::{FlareSolverrRequest, Flaresolverr}; -use crate::util::parse_abbreviated_number; -use crate::util::time::parse_time_to_seconds; -use crate::videos::{ServerOptions, VideoItem}; -use crate::{DbPool, videos}; +use crate::util::discord::{format_error_chain, send_discord_error_report}; +use crate::util::requester::Requester; +use crate::videos::{ServerOptions, VideoFormat, VideoItem}; +use crate::{DbPool, db}; + use async_trait::async_trait; use error_chain::error_chain; +use futures::future::join_all; use htmlentity::entity::{ICodedDataTrait, decode}; -use std::env; +use std::sync::{Arc, RwLock}; use std::vec; -use wreq::{Client, Proxy}; -use wreq_util::Emulation; +use titlecase::Titlecase; +use wreq::Version; error_chain! { foreign_links { Io(std::io::Error); HttpRequest(wreq::Error); + Json(serde_json::Error); + } + errors { + Parse(msg: String) { + description("parse error") + display("parse error: {}", msg) + } } } #[derive(Debug, Clone)] pub struct HentaihavenProvider { url: String, + categories: Arc>>, } + impl HentaihavenProvider { pub fn new() -> Self { - HentaihavenProvider { + let provider = Self { url: "https://hentaihaven.xxx".to_string(), - } + categories: Arc::new(RwLock::new(vec![])), + }; + provider } fn build_channel(&self, clientversion: ClientVersion) -> Channel { let _ = clientversion; - Channel { id: "hentaihaven".to_string(), - name: "WORK IN PROGRESS Hentai Haven".to_string(), - description: "".to_string(), + name: "Hentai Haven".to_string(), + description: "Watch Free Hentai Videos HD!".to_string(), premium: false, favicon: "https://www.google.com/s2/favicons?sz=64&domain=hentaihaven.xxx".to_string(), status: "active".to_string(), - categories: vec![], + categories: self + .categories + .read() + .unwrap() + .iter() + .map(|c| c.title.clone()) + .collect(), options: vec![], nsfw: true, cacheDuration: None, } } - async fn get(&self, cache: VideoCache, page: u8, sort: &str) -> Result> { - let video_url = format!("{}/page/{}/?m_orderby={}", self.url, page, sort); + fn push_unique(target: &Arc>>, item: FilterOption) { + if let Ok(mut vec) = target.write() { + if !vec.iter().any(|x| x.id == item.id) { + vec.push(item); + } + } + } + + async fn get( + &self, + cache: VideoCache, + page: u8, + sort: &str, + options: ServerOptions, + pool: DbPool, + ) -> Result> { + let _ = sort; + let video_url = format!("{}/hentai/page/{}/", self.url, page); let old_items = match cache.get(&video_url) { Some((time, items)) => { - if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { - println!("Cache hit for URL: {}", video_url); + if time.elapsed().unwrap_or_default().as_secs() < 60 * 60 * 24 { return Ok(items.clone()); } else { items.clone() @@ -67,79 +98,48 @@ impl HentaihavenProvider { } }; - let proxy = Proxy::all("http://192.168.0.103:8081").unwrap(); - let client = Client::builder() - .cert_verification(false) - .emulation(Emulation::Firefox136) - .build()?; - - let mut response = client - .get(video_url.clone()) - .proxy(proxy.clone()) - .send() - .await?; - if response.status().is_redirection() { - println!( - "Redirection detected, following to: {}", - response.headers()["Location"].to_str().unwrap() - ); - response = client - .get(response.headers()["Location"].to_str().unwrap()) - // .proxy(proxy) - .send() - .await?; + let mut requester = options.requester.clone().unwrap(); + let text = requester + .get(&video_url, Some(Version::HTTP_2)) + .await + .unwrap(); + if page > 1 + && !text.contains(&format!( + "
  • {}", + page + )) + { + return Ok(vec![]); } - if response.status().is_success() { - let text = response.text().await?; - let video_items: Vec = self.get_video_items_from_html(text.clone()); - if !video_items.is_empty() { - cache.remove(&video_url); - cache.insert(video_url.clone(), video_items.clone()); - } else { - return Ok(old_items); - } - Ok(video_items) + let video_items: Vec = self + .get_video_items_from_html(text.clone(), &mut requester, pool.clone()) + .await; + if !video_items.is_empty() { + cache.remove(&video_url); + cache.insert(video_url.clone(), video_items.clone()); } else { - let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set"); - let flare = Flaresolverr::new(flare_url); - let result = flare - .solve(FlareSolverrRequest { - cmd: "request.get".to_string(), - url: video_url.clone(), - maxTimeout: 60000, - }) - .await; - let video_items = match result { - Ok(res) => { - // println!("FlareSolverr response: {}", res); - self.get_video_items_from_html(res.solution.response) - } - Err(e) => { - println!("Error solving FlareSolverr: {}", e); - return Err("Failed to solve FlareSolverr".into()); - } - }; - if !video_items.is_empty() { - cache.remove(&video_url); - cache.insert(video_url.clone(), video_items.clone()); - } else { - return Ok(old_items); - } - Ok(video_items) + return Ok(old_items); } + Ok(video_items) } - async fn query(&self, cache: VideoCache, page: u8, query: &str) -> Result> { - let search_string = query.to_lowercase().trim().replace(" ", "-"); - let mut video_url = format!("{}/search/{}/{}/", self.url, search_string, page); - if search_string.starts_with("@") { - let url_part = search_string.split("@").collect::>()[1].replace(":", "/"); - video_url = format!("{}/{}/", self.url, url_part); - } + async fn query( + &self, + cache: VideoCache, + page: u8, + query: &str, + options: ServerOptions, + pool: DbPool, + ) -> Result> { + let video_url = format!( + "{}/?s={}", + self.url, + query.replace(" ", "+"), + ); // Check our Video Cache. If the result is younger than 1 hour, we return it. let old_items = match cache.get(&video_url) { Some((time, items)) => { - if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { + if time.elapsed().unwrap_or_default().as_secs() < 60 * 60 * 24 { return Ok(items.clone()); } else { let _ = cache.check().await; @@ -151,180 +151,318 @@ impl HentaihavenProvider { } }; - let proxy = Proxy::all("http://192.168.0.103:8081").unwrap(); - let client = Client::builder() - .cert_verification(false) - .emulation(Emulation::Firefox136) - .build()?; - - let mut response = client - .get(video_url.clone()) - .proxy(proxy.clone()) - .send() - .await?; - - if response.status().is_redirection() { - response = client - .get(self.url.clone() + response.headers()["Location"].to_str().unwrap()) - // .proxy(proxy) - .send() - .await?; + let mut requester = options.requester.clone().unwrap(); + let text = requester + .get(&video_url, Some(Version::HTTP_2)) + .await + .unwrap(); + if page > 1 + { + return Ok(vec![]); } - - if response.status().is_success() { - let text = response.text().await?; - let video_items: Vec = self.get_video_items_from_html(text.clone()); - if !video_items.is_empty() { - cache.remove(&video_url); - cache.insert(video_url.clone(), video_items.clone()); - } else { - return Ok(old_items); - } - Ok(video_items) + let video_items: Vec = self + .get_video_items_from_html_search(text.clone(), &mut requester, pool) + .await; + if !video_items.is_empty() { + cache.remove(&video_url); + cache.insert(video_url.clone(), video_items.clone()); } else { - let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set"); - let flare = Flaresolverr::new(flare_url); - let result = flare - .solve(FlareSolverrRequest { - cmd: "request.get".to_string(), - url: video_url.clone(), - maxTimeout: 60000, - }) - .await; - let video_items = match result { - Ok(res) => self.get_video_items_from_html(res.solution.response), - Err(e) => { - println!("Error solving FlareSolverr: {}", e); - return Err("Failed to solve FlareSolverr".into()); - } - }; - if !video_items.is_empty() { - cache.remove(&video_url); - cache.insert(video_url.clone(), video_items.clone()); - } else { - return Ok(old_items); - } - Ok(video_items) + return Ok(old_items); } + Ok(video_items) } - fn get_video_items_from_html(&self, html: String) -> Vec { - if html.is_empty() { - println!("HTML is empty"); + async fn get_video_items_from_html( + &self, + html: String, + requester: &mut Requester, + pool: DbPool, + ) -> Vec { + if html.is_empty() || html.contains("404 Not Found") { return vec![]; } - let mut items: Vec = Vec::new(); - let raw_videos = html.split("\"wp-pagenavi\"").collect::>()[0] - .split("page-item-detail video") - .collect::>()[1..] - .to_vec(); - for video_segment in &raw_videos { - let vid = video_segment.split("\n").collect::>(); - for (index, line) in vid.iter().enumerate() { - println!("Line {}: {}", index, line); - } - let episode_count = video_segment - .split("chapter font-meta") - .collect::>()[1] - .split("class=\"btn-link\">") - .collect::>()[1] - .split("<") - .collect::>()[0] - .split(" ") - .collect::>()[2] - .to_string() - .parse::() - .unwrap(); - let season = video_segment - .split("chapter font-meta") - .collect::>()[1] - .split("class=\"btn-link\">") - .collect::>()[1] - .split("<") - .collect::>()[0] - .split(" ") - .collect::>()[1] - == "Season"; - let mut url_part_list = video_segment - .split("chapter font-meta") - .collect::>()[1] - .split("href=\"") - .collect::>()[1] - .split("\"") - .collect::>()[0] - .split("/") - .collect::>()[4] - .split("-") - .collect::>(); - if url_part_list.len() > 5 { - if let Some(pos) = url_part_list.iter().rposition(|x| *x == "no") { - url_part_list.remove(pos); + let block = match html + .split("previouspostslink") + .next() + .and_then(|s| s.split("vraven_manga_list").nth(1)) + { + Some(b) => b, + None => { + eprint!("Hentai Haven Provider: Failed to get block from html"); + let e = Error::from(ErrorKind::Parse("html".into())); + send_discord_error_report( + e.to_string(), + Some(format_error_chain(&e)), + Some("Hentai Haven Provider"), + Some(&format!("Failed to get block from html:\n```{html}\n```")), + file!(), + line!(), + module_path!(), + ) + .await; + return vec![]; + } + }; + + let futures = block + .split("id=\"manga-item-") + .skip(1) + .map(|el| self.get_video_item(el.to_string(), pool.clone(), requester.clone())); + join_all(futures) + .await + .into_iter() + .inspect(|r| { + if let Err(e) = r { + eprint!("Hentai Haven Provider: Failed to get video item:{}\n", e); + // Prepare data to move into the background task + let msg = e.to_string(); + let chain = format_error_chain(&e); + + // Spawn the report into the background - NO .await here + tokio::spawn(async move { + let _ = send_discord_error_report( + msg, + Some(chain), + Some("Hentai Haven Provider"), + Some("Failed to get video item"), + file!(), // Note: these might report the utility line + line!(), // better to hardcode or pass from outside + module_path!(), + ) + .await; + }); + } + }) + .filter_map(Result::ok) + .collect() + } + + async fn get_video_items_from_html_search( + &self, + html: String, + requester: &mut Requester, + pool: DbPool, + ) -> Vec { + if html.is_empty() || html.contains("404 Not Found") { + return vec![]; + } + + let block = match html + .split(" b, + None => { + eprint!("Hentai Haven Provider: Failed to get block from html"); + let e = Error::from(ErrorKind::Parse("html".into())); + send_discord_error_report( + e.to_string(), + Some(format_error_chain(&e)), + Some("Hentai Haven Provider"), + Some(&format!("Failed to get block from html:\n```{html}\n```")), + file!(), + line!(), + module_path!(), + ) + .await; + return vec![]; + } + }; + + let futures = block + .split("c-tabs-item__content col-6 col-md-12") + .skip(1) + .map(|el| self.get_video_item(el.to_string(), pool.clone(), requester.clone())); + join_all(futures) + .await + .into_iter() + .inspect(|r| { + if let Err(e) = r { + eprint!("Hentai Haven Provider: Failed to get video item:{}\n", e); + // Prepare data to move into the background task + let msg = e.to_string(); + let chain = format_error_chain(&e); + + // Spawn the report into the background - NO .await here + tokio::spawn(async move { + let _ = send_discord_error_report( + msg, + Some(chain), + Some("Hentai Haven Provider"), + Some("Failed to get video item"), + file!(), // Note: these might report the utility line + line!(), // better to hardcode or pass from outside + module_path!(), + ) + .await; + }); + } + }) + .filter_map(Result::ok) + .collect() + } + + async fn get_video_item( + &self, + seg: String, + pool: DbPool, + mut requester: Requester, + ) -> Result { + let video_url = seg + .split("a href=\"") + .nth(1) + .and_then(|s| s.split('"').next()) + .ok_or_else(|| ErrorKind::Parse("video url\n\n{seg}".into()))? + .to_string(); + let mut conn = pool.get().expect("couldn't get db connection from pool"); + let db_result = db::get_video(&mut conn, video_url.clone()); + drop(conn); + match db_result { + Ok(Some(video)) => { + let video_item = VideoItem::from(video); + match video_item { + Ok(item) => return Ok(item), + Err(e) => { + eprint!("Failed to convert video from DB result: {}\n", e); + } } } - url_part_list.truncate(5); - let url_part = url_part_list.join("-"); - for i in 1..=episode_count { - let mut video_url = format!( - "https://master-lengs.org/api/v3/hh/{}-{}-eng/master.m3u8", - url_part, i - ); - if season { - video_url = format!( - "https://master-lengs.org/api/v3/hh/{}-season-eng/master.m3u8", - url_part - ); - } - let title = format!( - "{} - {}", - video_segment.split("title=\"").collect::>()[1] - .split("\"") - .collect::>()[0] - .to_string(), - i - ); - let id = format!("{}-{}", url_part, i); - - let thumb = match video_segment.split(">()[1] - .split("") - .collect::>()[0] - .contains("data-src=\"") - { - true => video_segment.split(">()[1] - .split("data-src=\"") - .collect::>()[1] - .split("\"") - .collect::>()[0] - .replace(" ", "%20") - .to_string(), - false => video_segment.split(">()[1] - .split("src=\"") - .collect::>()[1] - .split("\"") - .collect::>()[0] - .replace(" ", "%20") - .to_string(), - }; - items.push( - VideoItem::new( - id, - title, - video_url.clone(), - "hentaihaven".to_string(), - thumb, - 0, // duration is not available - ) - .formats(vec![videos::VideoFormat::new( - video_url.clone(), - "1080".to_string(), - "m3u8".to_string(), - )]) - .aspect_ratio(0.73), - ); + Ok(None) => { + // continue to fetch and parse the video + } + Err(e) => { + eprint!("Database error: {}\n", e); + // continue to fetch and parse the video even if there's a DB error } } - return items; - //return items; + let html = requester + .get(&video_url, Some(Version::HTTP_2)) + .await + .map_err(|e| Error::from(format!("Failed to fetch video page: {}", e)))?; + + let mut title = html + .split("

    ") + .nth(1) + .and_then(|s| s.split("

    ").next()) + .ok_or_else(|| ErrorKind::Parse(format!("video title\n\n{seg}").into()))? + .trim() + .to_string(); + title = decode(title.as_bytes()) + .to_string() + .unwrap_or(title) + .titlecase(); + let id = video_url + .split('/') + .nth(4) + .and_then(|s| s.split('.').next()) + .ok_or_else(|| ErrorKind::Parse("video id\n\n{seg}".into()))? + .to_string(); + let thumb = html + .split("og:image\" content=\"") + .nth(1) + .and_then(|s| s.split('"').next()) + .unwrap_or("") + .to_string(); + let raw_tags: Vec = html + .split("Genre(s)") + .nth(1) + .unwrap_or_default() + .split("Release") + .nth(0) + .unwrap_or_default() + .split("a href=\"") + .skip(1) + .map(|tag_block| { + let id = tag_block + .split("\"") + .nth(1) + .and_then(|s| s.split('"').next()) + .unwrap_or("") + .to_string(); + let title = tag_block + .split('>') + .nth(1) + .and_then(|s| s.split('<').next()) + .map(|s| { + decode(s.as_bytes()) + .to_string() + .unwrap_or(s.to_string()) + .titlecase() + }) + .unwrap_or("".to_string()); + FilterOption { + id: id.to_ascii_lowercase().replace(" ", "+"), + title: title.clone(), + } + }) + .collect::>(); + for tag in &raw_tags { + Self::push_unique(&self.categories, tag.clone()); + } + let tags = raw_tags.into_iter().map(|t| t.title).collect(); + let views = html + .split("Viewed") + .last() + .and_then(|s| s.split("summary-content\">").nth(1)) + .and_then(|s| s.split(" Total").nth(0)) + .map(|s| + s.trim().parse::().unwrap_or(0)) + .unwrap_or(0); + let mut formats = vec![]; + let episode_block = html + .split("manga-chapters-holder") + .nth(1) + .unwrap_or_default() + .split("vraven_read") + .nth(0) + .unwrap_or_default(); + for episode in episode_block.split("wp-manga-chapter").skip(1) { + let ep_thumbnail = episode + .split(" src=\"") + .nth(1) + .and_then(|s| s.split('"').next()) + .unwrap_or_default(); + let episode_title = episode + .split("
    ") + .nth(1) + .and_then(|s| s.split('<').next()) + .unwrap_or_default() + .trim() + .to_string(); + let episode_id = ep_thumbnail.split('/').nth(5).unwrap_or_default(); + let episode_url = format!( + "https://master-lengs.org/api/v3/hh/{}/master.m3u8", + episode_id + ); + let format = VideoFormat::new(episode_url, "1080p".to_string(), "m3u8".to_string()) + .format_id(episode_title); + formats.push(format); + } + if formats.is_empty() { + let e = Error::from(format!("No formats found for video URL: {}", video_url)); + return Err(e); + } + if formats.len() > 1 { + title = format!("{} ({} Episodes)", title, formats.len()); + } + + let video_item = + VideoItem::new(id, title, video_url.clone(), "hentaihaven".into(), thumb, 0) + .formats(formats) + .tags(tags) + .views(views); + + let mut conn = pool.get().expect("couldn't get db connection from pool"); + let _ = db::insert_video( + &mut conn, + &video_url, + &serde_json::to_string(&video_item).unwrap_or_default(), + ); + drop(conn); + + Ok(video_item) } } @@ -337,29 +475,23 @@ impl Provider for HentaihavenProvider { sort: String, query: Option, page: String, - per_page: String, + _per_page: String, options: ServerOptions, ) -> Vec { - let _ = options; - let _ = per_page; - let _ = pool; - let videos: std::result::Result, Error> = match query { - Some(q) => self.query(cache, page.parse::().unwrap_or(1), &q).await, - None => { - self.get(cache, page.parse::().unwrap_or(1), &sort) - .await - } + let page = page.parse::().unwrap_or(1); + + let res = match query { + Some(q) => self.to_owned().query(cache, page, &q, options, pool).await, + None => self.get(cache, page, &sort, options, pool).await, }; - match videos { - Ok(v) => v, - Err(e) => { - println!("Error fetching videos: {}", e); - vec![] - } - } + + res.unwrap_or_else(|e| { + eprintln!("hentai haven error: {e}"); + vec![] + }) } - fn get_channel(&self, clientversion: ClientVersion) -> Option { - Some(self.build_channel(clientversion)) + fn get_channel(&self, v: ClientVersion) -> Option { + Some(self.build_channel(v)) } } diff --git a/src/providers/mod.rs b/src/providers/mod.rs index 33f5f19..2018b28 100644 --- a/src/providers/mod.rs +++ b/src/providers/mod.rs @@ -42,6 +42,7 @@ pub mod hypnotube; pub mod freepornvideosxxx; pub mod hentaihaven; pub mod chaturbate; +// pub mod tube8; // convenient alias pub type DynProvider = Arc; @@ -63,6 +64,7 @@ pub static ALL_PROVIDERS: Lazy> = Lazy::new(| m.insert("freepornvideosxxx", Arc::new(freepornvideosxxx::FreepornvideosxxxProvider::new()) as DynProvider); m.insert("hentaihaven", Arc::new(hentaihaven::HentaihavenProvider::new()) as DynProvider); m.insert("chaturbate", Arc::new(chaturbate::ChaturbateProvider::new()) as DynProvider); + // m.insert("tube8", Arc::new(tube8::Tube8Provider::new()) as DynProvider); // add more here as you migrate them m }); diff --git a/src/videos.rs b/src/videos.rs index ab09430..788a0f2 100644 --- a/src/videos.rs +++ b/src/videos.rs @@ -144,6 +144,9 @@ impl VideoItem { aspectRatio: None, } } + pub fn from(s: String) -> Result { + serde_json::from_str::(&s) + } pub fn tags(mut self, tags: Vec) -> Self { if tags.is_empty(){ return self;