use crate::DbPool; use crate::api::ClientVersion; use crate::providers::Provider; use crate::status::*; use crate::util::cache::VideoCache; use crate::util::discord::{format_error_chain, send_discord_error_report}; use crate::util::requester::Requester; use crate::util::time::parse_time_to_seconds; use crate::videos::{ServerOptions, VideoFormat, VideoItem}; use async_trait::async_trait; use error_chain::error_chain; use futures::future::join_all; use htmlentity::entity::{ICodedDataTrait, decode}; use std::sync::{Arc, RwLock}; use std::vec; use titlecase::Titlecase; use wreq::Version; error_chain! { foreign_links { Io(std::io::Error); HttpRequest(wreq::Error); Json(serde_json::Error); } errors { Parse(msg: String) { description("parse error") display("parse error: {}", msg) } } } #[derive(Debug, Clone)] pub struct JavtifulProvider { url: String, categories: Arc>>, } impl JavtifulProvider { pub fn new() -> Self { let provider = Self { url: "https://javtiful.com".to_string(), categories: Arc::new(RwLock::new(vec![])), }; provider } fn build_channel(&self, clientversion: ClientVersion) -> Channel { let _ = clientversion; Channel { id: "javtiful".to_string(), name: "Javtiful".to_string(), description: "Watch Porn!".to_string(), premium: false, favicon: "https://www.google.com/s2/favicons?sz=64&domain=javtiful.com".to_string(), status: "active".to_string(), categories: self .categories .read() .map(|categories| categories.iter().map(|c| c.title.clone()).collect()) .unwrap_or_else(|e| { crate::providers::report_provider_error_background( "javtiful", "build_channel.categories_read", &e.to_string(), ); vec![] }), options: vec![ChannelOption { id: "sort".to_string(), title: "Sort".to_string(), description: "Sort the Videos".to_string(), systemImage: "list.number".to_string(), colorName: "blue".to_string(), options: vec![ FilterOption { id: "newest".into(), title: "Newest".into(), }, FilterOption { id: "top rated".into(), title: "Top Rated".into(), }, FilterOption { id: "most viewed".into(), title: "Most Viewed".into(), }, FilterOption { id: "top favorites".into(), title: "Top Favorites".into(), }, ], multiSelect: false, }], nsfw: true, cacheDuration: Some(1800), } } fn push_unique(target: &Arc>>, item: FilterOption) { if let Ok(mut vec) = target.write() { if !vec.iter().any(|x| x.id == item.id) { vec.push(item); } } } async fn get( &self, cache: VideoCache, page: u8, sort: &str, options: ServerOptions, ) -> Result> { let sort_string = match sort { "top rated" => "/sort=top_rated", "most viewed" => "/sort=most_viewed", _ => "", }; let video_url = format!("{}/videos{}?page={}", self.url, sort_string, page); let old_items = match cache.get(&video_url) { Some((time, items)) => { if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { return Ok(items.clone()); } else { items.clone() } } None => { vec![] } }; let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); let text = match requester.get(&video_url, Some(Version::HTTP_2)).await { Ok(text) => text, Err(e) => { crate::providers::report_provider_error( "javtiful", "get.request", &format!("url={video_url}; error={e}"), ) .await; return Ok(old_items); } }; if page > 1 && !text.contains(&format!( "
  • {}", page )) { return Ok(vec![]); } let video_items: Vec = self .get_video_items_from_html(text.clone(), &mut requester) .await; if !video_items.is_empty() { cache.remove(&video_url); cache.insert(video_url.clone(), video_items.clone()); } else { return Ok(old_items); } Ok(video_items) } async fn query( &self, cache: VideoCache, page: u8, query: &str, options: ServerOptions, ) -> Result> { let sort_string = match options.sort.as_deref().unwrap_or("") { "top rated" => "/sort=top_rated", "most viewed" => "/sort=most_viewed", _ => "", }; let video_url = format!( "{}/search/videos{}?search_query={}&page={}", self.url, sort_string, query.replace(" ", "+"), page ); // Check our Video Cache. If the result is younger than 1 hour, we return it. let old_items = match cache.get(&video_url) { Some((time, items)) => { if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { return Ok(items.clone()); } else { let _ = cache.check().await; return Ok(items.clone()); } } None => { vec![] } }; let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); let text = match requester.get(&video_url, Some(Version::HTTP_2)).await { Ok(text) => text, Err(e) => { crate::providers::report_provider_error( "javtiful", "query.request", &format!("url={video_url}; error={e}"), ) .await; return Ok(old_items); } }; if page > 1 && !text.contains(&format!( "
  • {}", page )) { return Ok(vec![]); } let video_items: Vec = self .get_video_items_from_html(text.clone(), &mut requester) .await; if !video_items.is_empty() { cache.remove(&video_url); cache.insert(video_url.clone(), video_items.clone()); } else { return Ok(old_items); } Ok(video_items) } async fn get_video_items_from_html( &self, html: String, requester: &mut Requester, ) -> Vec { if html.is_empty() || html.contains("404 Not Found") { return vec![]; } let block = match html.split("pagination ").next().and_then(|s| { s.split("row row-cols-1 row-cols-sm-2 row-cols-lg-3 row-cols-xl-4") .nth(1) }) { Some(b) => b, None => { eprint!("Javtiful Provider: Failed to get block from html"); let e = Error::from(ErrorKind::Parse("html".into())); send_discord_error_report( e.to_string(), Some(format_error_chain(&e)), Some("Javtiful Provider"), Some(&format!("Failed to get block from html:\n```{html}\n```")), file!(), line!(), module_path!(), ) .await; return vec![]; } }; let futures = block .split("card ") .skip(1) .filter(|seg| !seg.contains("SPONSOR")) .map(|el| self.get_video_item(el.to_string(), requester.clone())); join_all(futures) .await .into_iter() .inspect(|r| { if let Err(e) = r { eprint!("Javtiful Provider: Failed to get video item:{}\n", e); // Prepare data to move into the background task let msg = e.to_string(); let chain = format_error_chain(&e); // Spawn the report into the background - NO .await here tokio::spawn(async move { let _ = send_discord_error_report( msg, Some(chain), Some("Javtiful Provider"), Some("Failed to get video item"), file!(), // Note: these might report the utility line line!(), // better to hardcode or pass from outside module_path!(), ) .await; }); } }) .filter_map(Result::ok) .collect() } async fn get_video_item(&self, seg: String, mut requester: Requester) -> Result { let video_url = seg .split(" href=\"") .nth(1) .and_then(|s| s.split('"').next()) .ok_or_else(|| ErrorKind::Parse("video url\n\n{seg}".into()))? .to_string(); let mut title = seg .split(" alt=\"") .nth(1) .and_then(|s| s.split('"').next()) .ok_or_else(|| ErrorKind::Parse(format!("video title\n\n{seg}").into()))? .trim() .to_string(); title = decode(title.as_bytes()) .to_string() .unwrap_or(title) .titlecase(); let id = video_url .split('/') .nth(5) .and_then(|s| s.split('.').next()) .ok_or_else(|| ErrorKind::Parse("video id\n\n{seg}".into()))? .to_string(); let thumb_block = seg .split("") .nth(1) .and_then(|s| s.split('<').next()) .unwrap_or("") .to_string(); let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32; let (tags, formats, views) = self.extract_media(&video_url, &mut requester).await?; if preview.len() == 0 { preview = format!("https://trailers.jav.si/preview/{id}.mp4"); } let video_item = VideoItem::new(id, title, video_url, "javtiful".into(), thumb, duration) .formats(formats) .tags(tags) .preview(preview) .views(views); Ok(video_item) } async fn extract_media( &self, url: &str, requester: &mut Requester, ) -> Result<(Vec, Vec, u32)> { let text = requester .get(url, Some(Version::HTTP_2)) .await .map_err(|e| Error::from(format!("{}", e)))?; let tags = text .split("related-actress") .next() .and_then(|s| s.split("video-comments").next()) .and_then(|s| s.split(">Tags<").nth(1)) .map(|tag_block| { tag_block .split("') .nth(1) .and_then(|s| s.split('<').next()) .map(|s| { decode(s.as_bytes()) .to_string() .unwrap_or(s.to_string()) .titlecase() }) }) .collect() }) .unwrap_or_else(|| vec![]); for tag in &tags { Self::push_unique( &self.categories, FilterOption { id: tag.to_ascii_lowercase().replace(" ", "+"), title: tag.to_string(), }, ); } let views = text .split(" Views ") .next() .and_then(|s| s.split(" ").last()) .and_then(|s| s.replace(".", "").parse::().ok()) .unwrap_or(0); let quality = "1080p".to_string(); let video_url = url.replace("javtiful.com", "hottub.spacemoehre.de/proxy/javtiful"); Ok(( tags, vec![VideoFormat::new(video_url, quality, "video/mp4".into())], views, )) } } #[async_trait] impl Provider for JavtifulProvider { async fn get_videos( &self, cache: VideoCache, _pool: DbPool, sort: String, query: Option, page: String, _per_page: String, options: ServerOptions, ) -> Vec { let page = page.parse::().unwrap_or(1); let res = match query { Some(q) => self.to_owned().query(cache, page, &q, options).await, None => self.get(cache, page, &sort, options).await, }; res.unwrap_or_else(|e| { eprintln!("javtiful error: {e}"); vec![] }) } fn get_channel(&self, v: ClientVersion) -> Option { Some(self.build_channel(v)) } }