diff --git a/src/api.rs b/src/api.rs index 0fa291e..6db2e44 100644 --- a/src/api.rs +++ b/src/api.rs @@ -1,27 +1,15 @@ -use capitalize::Capitalize; use ntex::http::header; use ntex::web; use ntex::web::HttpRequest; use std::cmp::Ordering; -use std::{fs, io}; +use std::io; use tokio::task; - -use crate::providers::all::AllProvider; -use crate::providers::hanime::HanimeProvider; -use crate::providers::okporn::OkpornProvider; -use crate::providers::perverzija::PerverzijaProvider; -use crate::providers::pornhub::PornhubProvider; -use crate::providers::redtube::RedtubeProvider; -use crate::providers::rule34video::Rule34videoProvider; -// use crate::providers::spankbang::SpankbangProvider; -use crate::providers::{ALL_PROVIDERS, DynProvider}; +use crate::providers::{ALL_PROVIDERS, DynProvider, panic_payload_to_string, report_provider_error, run_provider_guarded}; use crate::util::cache::VideoCache; use crate::util::discord::send_discord_error_report; use crate::util::proxy::{Proxy, all_proxies_snapshot}; use crate::util::requester::Requester; use crate::{DbPool, db, status::*, videos::*}; -use cute::c; -use std::sync::Arc; #[derive(Debug, Clone)] pub struct ClientVersion { @@ -130,827 +118,18 @@ async fn status(req: HttpRequest) -> Result { .to_string(); let mut status = Status::new(); - // pronhub - status.add_channel(Channel { - id: "pornhub".to_string(), - name: "Pornhub".to_string(), - description: "Pornhub Free Videos".to_string(), - premium: false, - favicon: "https://www.google.com/s2/favicons?sz=64&domain=pornhub.com".to_string(), - status: "active".to_string(), - categories: vec![], - options: vec![ChannelOption { - id: "sort".to_string(), - title: "Sort".to_string(), - description: "Sort the Videos".to_string(), //"Sort the videos by Date or Name.".to_string(), - systemImage: "list.number".to_string(), - colorName: "blue".to_string(), - options: vec![ - FilterOption { - id: "mr".to_string(), - title: "Most Recent".to_string(), - }, - FilterOption { - id: "mv".to_string(), - title: "Most Viewed".to_string(), - }, - FilterOption { - id: "tr".to_string(), - title: "Top Rated".to_string(), - }, - FilterOption { - id: "lg".to_string(), - title: "Longest".to_string(), - }, - FilterOption { - id: "cm".to_string(), - title: "Newest".to_string(), - }, - ], - multiSelect: false, - }], - nsfw: true, - cacheDuration: Some(1800), - }); - // perverzija - status.add_channel(Channel { - id: "perverzija".to_string(), - name: "Perverzija".to_string(), - description: "Free videos from Perverzija".to_string(), - premium: false, - favicon: "https://www.google.com/s2/favicons?sz=64&domain=tube.perverzija.com".to_string(), - status: "active".to_string(), - categories: vec![], - options: vec![ - // ChannelOption { - // id: "sort".to_string(), - // title: "Sort".to_string(), - // description: "Sort the Videos".to_string(), //"Sort the videos by Date or Name.".to_string(), - // systemImage: "list.number".to_string(), - // colorName: "blue".to_string(), - // options: vec![ - // FilterOption { - // id: "date".to_string(), - // title: "Date".to_string(), - // }, - // FilterOption { - // id: "name".to_string(), - // title: "Name".to_string(), - // }, - // ], - // multiSelect: false, - // }, - ChannelOption { - id: "featured".to_string(), - title: "Featured".to_string(), - description: "Filter Featured Videos.".to_string(), - systemImage: "star".to_string(), - colorName: "red".to_string(), - options: vec![ - FilterOption { - id: "all".to_string(), - title: "No".to_string(), - }, - FilterOption { - id: "featured".to_string(), - title: "Yes".to_string(), - }, - ], - multiSelect: false, - }, - // ChannelOption { - // id: "duration".to_string(), - // title: "Duration".to_string(), - // description: "Filter the videos by duration.".to_string(), - // systemImage: "timer".to_string(), - // colorName: "green".to_string(), - // options: vec![ - // FilterOption { - // id: "short".to_string(), - // title: "< 1h".to_string(), - // }, - // FilterOption { - // id: "long".to_string(), - // title: "> 1h".to_string(), - // }, - // ], - // multiSelect: true, - // }, - ], - nsfw: true, - cacheDuration: None, - }); - - // pornzog - status.add_channel(Channel { - id: "pornzog".to_string(), - name: "Pornzog".to_string(), - description: "Watch free porn videos at PornZog Free Porn Clips. More than 1 million videos, watch for free now!".to_string(), - premium: false, - favicon: "https://www.google.com/s2/favicons?sz=64&domain=pornzog.com".to_string(), - status: "active".to_string(), - categories: vec![], - options: vec![ChannelOption { - id: "sort".to_string(), - title: "Sort".to_string(), - description: "Sort the Videos".to_string(), //"Sort the videos by Date or Name.".to_string(), - systemImage: "list.number".to_string(), - colorName: "blue".to_string(), - options: vec![ - FilterOption { - id: "recent".to_string(), - title: "Recent".to_string(), - }, - FilterOption { - id: "relevance".to_string(), - title: "Relevance".to_string(), - }, - FilterOption { - id: "viewed".to_string(), - title: "Most Viewed".to_string(), - }, - FilterOption { - id: "rated".to_string(), - title: "Most Rated".to_string(), - }, - FilterOption { - id: "longest".to_string(), - title: "Longest".to_string(), - } - ], - multiSelect: false, - }], - nsfw: true, - cacheDuration: None, - }); - - // Hanime - status.add_channel(Channel { - id: "hanime".to_string(), - name: "Hanime".to_string(), - description: "Free Hentai from Hanime".to_string(), - premium: false, - favicon: "https://www.google.com/s2/favicons?sz=64&domain=hanime.tv".to_string(), - status: "active".to_string(), - categories: vec![], - options: vec![ChannelOption { - id: "sort".to_string(), - title: "Sort".to_string(), - description: "Sort the Videos".to_string(), //"Sort the videos by Date or Name.".to_string(), - systemImage: "list.number".to_string(), - colorName: "blue".to_string(), - options: vec![ - FilterOption { - id: "created_at_unix.desc".to_string(), - title: "Recent Upload".to_string(), - }, - FilterOption { - id: "created_at_unix.asc".to_string(), - title: "Old Upload".to_string(), - }, - FilterOption { - id: "views.desc".to_string(), - title: "Most Views".to_string(), - }, - FilterOption { - id: "views.asc".to_string(), - title: "Least Views".to_string(), - }, - FilterOption { - id: "likes.desc".to_string(), - title: "Most Likes".to_string(), - }, - FilterOption { - id: "likes.asc".to_string(), - title: "Least Likes".to_string(), - }, - FilterOption { - id: "released_at_unix.desc".to_string(), - title: "New".to_string(), - }, - FilterOption { - id: "released_at_unix.asc".to_string(), - title: "Old".to_string(), - }, - FilterOption { - id: "title_sortable.asc".to_string(), - title: "A - Z".to_string(), - }, - FilterOption { - id: "title_sortable.desc".to_string(), - title: "Z - A".to_string(), - }, - ], - multiSelect: false, - }], - nsfw: true, - cacheDuration: None, - }); - - // rule34video - status.add_channel(Channel { - id: "rule34video".to_string(), - name: "Rule34Video".to_string(), - description: "If it exists, there is porn".to_string(), - premium: false, - favicon: "https://www.google.com/s2/favicons?sz=64&domain=rule34video.com".to_string(), - status: "active".to_string(), - categories: vec![], - options: vec![ChannelOption { - id: "sort".to_string(), - title: "Sort".to_string(), - description: "Sort the Videos".to_string(), //"Sort the videos by Date or Name.".to_string(), - systemImage: "list.number".to_string(), - colorName: "blue".to_string(), - options: vec![ - FilterOption { - id: "post_date".to_string(), - title: "Newest".to_string(), - }, - FilterOption { - id: "video_viewed".to_string(), - title: "Most Viewed".to_string(), - }, - FilterOption { - id: "rating".to_string(), - title: "Top Rated".to_string(), - }, - FilterOption { - id: "duration".to_string(), - title: "Longest".to_string(), - }, - FilterOption { - id: "pseudo_random".to_string(), - title: "Random".to_string(), - }, - ], - multiSelect: false, - }], - nsfw: true, - cacheDuration: Some(1800), - }); - - let files = fs::read_dir("./src/providers").unwrap(); - let providers = files - .map(|entry| entry.unwrap().file_name()) - .filter(|name| name.to_str().unwrap().ends_with(".rs")) - .filter(|name| { - !name.to_str().unwrap().contains("mod.rs") && !name.to_str().unwrap().contains("all.rs") - }) - .map(|name| name.to_str().unwrap().replace(".rs", "")) - .collect::>(); - let sites = c![FilterOption { - id: x.to_string(), - title: x.capitalize().to_string(), - }, for x in providers.iter()]; - - // All - status.add_channel(Channel { - id: "all".to_string(), - name: "All".to_string(), - description: "Query from all sites of this Server".to_string(), - premium: false, - favicon: "https://hottub.spacemoehre.de/favicon.ico".to_string(), - status: "active".to_string(), - categories: vec![], - options: vec![ChannelOption { - id: "sites".to_string(), - title: "Sites".to_string(), - description: "What Sites to use".to_string(), //"Sort the videos by Date or Name.".to_string(), - systemImage: "list.number".to_string(), - colorName: "green".to_string(), - options: sites, - multiSelect: true, - }], - nsfw: true, - cacheDuration: Some(1800), - }); - - // Redtube - status.add_channel(Channel { - id: "redtube".to_string(), - name: "Redtube".to_string(), - description: "Redtube brings you NEW porn videos every day for free".to_string(), - premium: false, - favicon: "https://www.google.com/s2/favicons?sz=64&domain=www.redtube.com".to_string(), - status: "active".to_string(), - categories: vec![], - options: vec![], - nsfw: true, - cacheDuration: Some(1800), - }); - - // ok.porn - status.add_channel(Channel { - id: "okporn".to_string(), - name: "Ok.porn".to_string(), - description: "Tons of HD porno movies".to_string(), - premium: false, - favicon: "https://www.google.com/s2/favicons?sz=64&domain=ok.porn".to_string(), - status: "active".to_string(), - categories: vec![], - options: vec![ChannelOption { - id: "sort".to_string(), - title: "Sort".to_string(), - description: "Sort the Videos".to_string(), //"Sort the videos by Date or Name.".to_string(), - systemImage: "list.number".to_string(), - colorName: "blue".to_string(), - options: vec![ - FilterOption { - id: "new".to_string(), - title: "New".to_string(), - }, - FilterOption { - id: "popular".to_string(), - title: "Popular".to_string(), - }, - FilterOption { - id: "trending".to_string(), - title: "Trending".to_string(), - }, - ], - multiSelect: false, - }], - nsfw: true, - cacheDuration: Some(1800), - }); - - // pornhat - status.add_channel(Channel { - id: "pornhat".to_string(), - name: "Pornhat".to_string(), - description: "free HD porn videos".to_string(), - premium: false, - favicon: "https://www.google.com/s2/favicons?sz=64&domain=pornhat.com".to_string(), - status: "active".to_string(), - categories: vec![], - options: vec![ChannelOption { - id: "sort".to_string(), - title: "Sort".to_string(), - description: "Sort the Videos".to_string(), //"Sort the videos by Date or Name.".to_string(), - systemImage: "list.number".to_string(), - colorName: "blue".to_string(), - options: vec![ - FilterOption { - id: "new".to_string(), - title: "New".to_string(), - }, - FilterOption { - id: "popular".to_string(), - title: "Popular".to_string(), - }, - FilterOption { - id: "trending".to_string(), - title: "Trending".to_string(), - }, - ], - multiSelect: false, - }], - nsfw: true, - cacheDuration: Some(1800), - }); - - //perfectgirls - status.add_channel(Channel { - id: "perfectgirls".to_string(), - name: "Perfectgirls".to_string(), - description: "Perfect Girls Tube".to_string(), - premium: false, - favicon: "https://www.google.com/s2/favicons?sz=64&domain=perfectgirls.xxx".to_string(), - status: "active".to_string(), - categories: vec![], - options: vec![ChannelOption { - id: "sort".to_string(), - title: "Sort".to_string(), - description: "Sort the Videos".to_string(), //"Sort the videos by Date or Name.".to_string(), - systemImage: "list.number".to_string(), - colorName: "blue".to_string(), - options: vec![ - FilterOption { - id: "new".to_string(), - title: "New".to_string(), - }, - FilterOption { - id: "popular".to_string(), - title: "Popular".to_string(), - }, - FilterOption { - id: "trending".to_string(), - title: "Trending".to_string(), - }, - ], - multiSelect: false, - }], - nsfw: true, - cacheDuration: Some(1800), - }); - - // okxxx - status.add_channel(Channel { - id: "okxxx".to_string(), - name: "Ok.xxx".to_string(), - description: "free porn tube!".to_string(), - premium: false, - favicon: "https://www.google.com/s2/favicons?sz=64&domain=ok.xxx".to_string(), - status: "active".to_string(), - categories: vec![], - options: vec![ChannelOption { - id: "sort".to_string(), - title: "Sort".to_string(), - description: "Sort the Videos".to_string(), //"Sort the videos by Date or Name.".to_string(), - systemImage: "list.number".to_string(), - colorName: "blue".to_string(), - options: vec![ - FilterOption { - id: "new".to_string(), - title: "New".to_string(), - }, - FilterOption { - id: "popular".to_string(), - title: "Popular".to_string(), - }, - FilterOption { - id: "trending".to_string(), - title: "Trending".to_string(), - }, - ], - multiSelect: false, - }], - nsfw: true, - cacheDuration: Some(1800), - }); - - // homoxxx - status.add_channel(Channel { - id: "homoxxx".to_string(), - name: "Homo.xxx".to_string(), - description: "Best Gay Porn".to_string(), - premium: false, - favicon: "https://www.google.com/s2/favicons?sz=64&domain=homo.xxx".to_string(), - status: "active".to_string(), - categories: vec![], - options: vec![ChannelOption { - id: "sort".to_string(), - title: "Sort".to_string(), - description: "Sort the Videos".to_string(), //"Sort the videos by Date or Name.".to_string(), - systemImage: "list.number".to_string(), - colorName: "blue".to_string(), - options: vec![ - FilterOption { - id: "new".to_string(), - title: "New".to_string(), - }, - FilterOption { - id: "popular".to_string(), - title: "Popular".to_string(), - }, - FilterOption { - id: "trending".to_string(), - title: "Trending".to_string(), - }, - ], - multiSelect: false, - }], - nsfw: true, - cacheDuration: Some(1800), - }); - - // xxthots - status.add_channel(Channel { - id: "xxthots".to_string(), - name: "XXTHOTS".to_string(), - description: "Free XXX Onlyfans Leaks Videos".to_string(), - premium: false, - favicon: "https://www.google.com/s2/favicons?sz=64&domain=xxthots.com".to_string(), - status: "active".to_string(), - categories: vec![], - options: vec![ChannelOption { - id: "sort".to_string(), - title: "Sort".to_string(), - description: "Sort the Videos".to_string(), //"Sort the videos by Date or Name.".to_string(), - systemImage: "list.number".to_string(), - colorName: "blue".to_string(), - options: vec![ - FilterOption { - id: "new".to_string(), - title: "New".to_string(), - }, - FilterOption { - id: "popular".to_string(), - title: "Popular".to_string(), - }, - FilterOption { - id: "top-rated".to_string(), - title: "Top Rated".to_string(), - }, - ], - multiSelect: false, - }], - nsfw: true, - cacheDuration: Some(1800), - }); - - // porn00 - status.add_channel(Channel { - id: "porn00".to_string(), - name: "Porn00".to_string(), - description: "HD Porn".to_string(), - premium: false, - favicon: "https://www.google.com/s2/favicons?sz=64&domain=www.porn00.org".to_string(), - status: "active".to_string(), - categories: vec![], - options: vec![ChannelOption { - id: "sort".to_string(), - title: "Sort".to_string(), - description: "Sort the Videos".to_string(), //"Sort the videos by Date or Name.".to_string(), - systemImage: "list.number".to_string(), - colorName: "blue".to_string(), - options: vec![ - FilterOption { - id: "new".to_string(), - title: "New".to_string(), - }, - FilterOption { - id: "popular".to_string(), - title: "Popular".to_string(), - }, - FilterOption { - id: "top-rated".to_string(), - title: "Top Rated".to_string(), - }, - ], - multiSelect: false, - }], - nsfw: true, - cacheDuration: Some(1800), - }); - - // paradisehill - status.add_channel(Channel { - id: "paradisehill".to_string(), - name: "Paradisehill".to_string(), - description: "Porn Movies on Paradise Hill".to_string(), - premium: false, - favicon: "https://www.google.com/s2/favicons?sz=64&domain=en.paradisehill.cc".to_string(), - status: "active".to_string(), - categories: vec![], - options: vec![], - nsfw: true, - cacheDuration: None, - }); - - // youjizz - status.add_channel(Channel { - id: "youjizz".to_string(), - name: "YouJizz".to_string(), - description: "YouJizz Porntube".to_string(), - premium: false, - favicon: "https://www.google.com/s2/favicons?sz=64&domain=www.youjizz.com".to_string(), - status: "active".to_string(), - categories: vec![], - options: vec![ChannelOption { - id: "sort".to_string(), - title: "Sort".to_string(), - description: "Sort the Videos".to_string(), //"Sort the videos by Date or Name.".to_string(), - systemImage: "list.number".to_string(), - colorName: "blue".to_string(), - options: vec![ - FilterOption { - id: "new".to_string(), - title: "New".to_string(), - }, - FilterOption { - id: "popular".to_string(), - title: "Popular".to_string(), - }, - FilterOption { - id: "top-rated".to_string(), - title: "Top Rated".to_string(), - }, - FilterOption { - id: "top-rated-week".to_string(), - title: "Top Rated (Week)".to_string(), - }, - FilterOption { - id: "top-rated-month".to_string(), - title: "Top Rated (Month)".to_string(), - }, - FilterOption { - id: "trending".to_string(), - title: "Trending".to_string(), - }, - FilterOption { - id: "random".to_string(), - title: "Random".to_string(), - }, - ], - multiSelect: false, - }], - nsfw: true, - cacheDuration: None, - }); - - //missav - status.add_channel(Channel { - id: "missav".to_string(), - name: "MissAV".to_string(), - description: "Watch HD JAV Online".to_string(), - premium: false, - favicon: "https://www.google.com/s2/favicons?sz=64&domain=missav.ws".to_string(), - status: "active".to_string(), - categories: vec![], - options: vec![ - ChannelOption { - id: "sort".to_string(), - title: "Sort".to_string(), - description: "Sort the Videos".to_string(), //"Sort the videos by Date or Name.".to_string(), - systemImage: "list.number".to_string(), - colorName: "blue".to_string(), - options: vec![ - FilterOption { - id: "released_at".to_string(), - title: "Release Date".to_string(), - }, - FilterOption { - id: "published_at".to_string(), - title: "Recent Update".to_string(), - }, - FilterOption { - id: "today_views".to_string(), - title: "Today Views".to_string(), - }, - FilterOption { - id: "weekly_views".to_string(), - title: "Weekly Views".to_string(), - }, - FilterOption { - id: "monthly_views".to_string(), - title: "Monthly Views".to_string(), - }, - FilterOption { - id: "views".to_string(), - title: "Total Views".to_string(), - }, - ], - multiSelect: false, - }, - ChannelOption { - id: "filter".to_string(), - title: "Filter".to_string(), - description: "Filter the Videos".to_string(), //"Sort the videos by Date or Name.".to_string(), - systemImage: "line.horizontal.3.decrease.circle".to_string(), - colorName: "green".to_string(), - options: vec![ - FilterOption { - id: "new".to_string(), - title: "Recent update".to_string(), - }, - FilterOption { - id: "release".to_string(), - title: "New Releases".to_string(), - }, - FilterOption { - id: "uncensored-leak".to_string(), - title: "Uncensored".to_string(), - }, - FilterOption { - id: "english-subtitle".to_string(), - title: "English subtitle".to_string(), - }, - ], - multiSelect: false, - }, - ChannelOption { - id: "language".to_string(), - title: "Language".to_string(), - description: "What Language to fetch".to_string(), //"Sort the videos by Date or Name.".to_string(), - systemImage: "flag.fill".to_string(), - colorName: "gray".to_string(), - options: vec![ - FilterOption { - id: "en".to_string(), - title: "English".to_string(), - }, - FilterOption { - id: "cn".to_string(), - title: "简体中文".to_string(), - }, - FilterOption { - id: "ja".to_string(), - title: "日本語".to_string(), - }, - FilterOption { - id: "ko".to_string(), - title: "한국의".to_string(), - }, - FilterOption { - id: "ms".to_string(), - title: "Melayu".to_string(), - }, - FilterOption { - id: "th".to_string(), - title: "ไทย".to_string(), - }, - FilterOption { - id: "de".to_string(), - title: "Deutsch".to_string(), - }, - FilterOption { - id: "fr".to_string(), - title: "Français".to_string(), - }, - FilterOption { - id: "vi".to_string(), - title: "Tiếng Việt".to_string(), - }, - FilterOption { - id: "id".to_string(), - title: "Bahasa Indonesia".to_string(), - }, - FilterOption { - id: "fil".to_string(), - title: "Filipino".to_string(), - }, - FilterOption { - id: "pt".to_string(), - title: "Português".to_string(), - }, - ], - multiSelect: false, - }, - ], - nsfw: true, - cacheDuration: None, - }); - - // if clientversion >= ClientVersion::new(22, 105, "22i".to_string()) { - //sxyprn - status.add_channel(Channel { - id: "sxyprn".to_string(), - name: "SexyPorn".to_string(), - description: "Free Porn Site".to_string(), - premium: false, - favicon: "https://www.google.com/s2/favicons?sz=64&domain=sxyprn.com".to_string(), - status: "active".to_string(), - categories: vec![], - options: vec![ - ChannelOption { - id: "sort".to_string(), - title: "Sort".to_string(), - description: "Sort the Videos".to_string(), //"Sort the videos by Date or Name.".to_string(), - systemImage: "list.number".to_string(), - colorName: "blue".to_string(), - options: vec![ - FilterOption { - id: "latest".to_string(), - title: "Latest".to_string(), - }, - FilterOption { - id: "views".to_string(), - title: "Views".to_string(), - }, - FilterOption { - id: "rating".to_string(), - title: "Rating".to_string(), - }, - FilterOption { - id: "orgasmic".to_string(), - title: "Orgasmic".to_string(), - }, - ], - multiSelect: false, - }, - ChannelOption { - id: "filter".to_string(), - title: "Filter".to_string(), - description: "Filter the Videos".to_string(), //"Sort the videos by Date or Name.".to_string(), - systemImage: "line.horizontal.3.decrease.circle".to_string(), - colorName: "green".to_string(), - options: vec![ - FilterOption { - id: "top".to_string(), - title: "Top".to_string(), - }, - FilterOption { - id: "other".to_string(), - title: "Other".to_string(), - }, - FilterOption { - id: "all".to_string(), - title: "All".to_string(), - }, - ], - multiSelect: false, - }, - ], - nsfw: true, - cacheDuration: Some(1800), - }); - // } - - for provider in ALL_PROVIDERS.values() { - if let Some(channel) = provider.get_channel(clientversion.clone()) { - status.add_channel(channel); + for (provider_name, provider) in ALL_PROVIDERS.iter() { + let channel_result = + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + provider.get_channel(clientversion.clone()) + })); + match channel_result { + Ok(Some(channel)) => status.add_channel(channel), + Ok(None) => {} + Err(payload) => { + let panic_msg = panic_payload_to_string(payload); + report_provider_error(provider_name, "status.get_channel", &panic_msg).await; + } } } status.iconUrl = format!("http://{}/favicon.ico", host).to_string(); @@ -979,13 +158,25 @@ async fn videos_post( _ => {} } let requester = requester.get_ref().clone(); - let mut conn = pool.get().expect("couldn't get db connection from pool"); - // Ensure "videos" table exists with two string columns - if !(db::has_table(&mut conn, "videos").unwrap()) { - let _ = db::create_table( - &mut conn, - "CREATE TABLE videos (id TEXT NOT NULL, url TEXT NOT NULL);", - ); + // Ensure "videos" table exists with two string columns. + match pool.get() { + Ok(mut conn) => match db::has_table(&mut conn, "videos") { + Ok(false) => { + if let Err(e) = db::create_table( + &mut conn, + "CREATE TABLE videos (id TEXT NOT NULL, url TEXT NOT NULL);", + ) { + report_provider_error("db", "videos_post.create_table", &e.to_string()).await; + } + } + Ok(true) => {} + Err(e) => { + report_provider_error("db", "videos_post.has_table", &e.to_string()).await; + } + }, + Err(e) => { + report_provider_error("db", "videos_post.pool_get", &e.to_string()).await; + } } let mut videos = Videos { @@ -1055,8 +246,10 @@ async fn videos_post( duration: Some(duration), sort: Some(sort.clone()), }; - let mut video_items = provider - .get_videos( + let mut video_items = run_provider_guarded( + &channel, + "videos_post.get_videos", + provider.get_videos( cache.get_ref().clone(), pool.get_ref().clone(), sort.clone(), @@ -1064,16 +257,21 @@ async fn videos_post( page.to_string(), perPage.to_string(), options.clone(), - ) - .await; + ), + ) + .await; // There is a bug in Hottub38 that makes the client error for a 403-url even though formats work fine if clientversion == ClientVersion::new(38, 0, "Hot%20Tub".to_string()) { // filter out videos without preview for old clients video_items = video_items.into_iter().filter_map(|video| { - if !video.formats.is_none() && video.formats.as_ref().unwrap().len() > 0 { + let last_url = video + .formats + .as_ref() + .and_then(|formats| formats.last().map(|f| f.url.clone())); + if let Some(url) = last_url { let mut v = video; - v.url = v.formats.as_ref().unwrap().last().unwrap().url.clone(); + v.url = url; return Some(v); } Some(video) @@ -1096,13 +294,16 @@ async fn videos_post( let query_clone = query.clone(); let per_page_clone = perPage.to_string(); let options_clone = options.clone(); + let channel_clone = channel.clone(); task::spawn_local(async move { // if let AnyProvider::Spankbang(_) = provider_clone { // // Spankbang has a delay for the next page // ntex::time::sleep(ntex::time::Seconds(80)).await; // } - let _ = provider_clone - .get_videos( + let _ = run_provider_guarded( + &channel_clone, + "videos_post.prefetch_next_page", + provider_clone.get_videos( cache_clone, pool_clone, sort_clone, @@ -1110,8 +311,9 @@ async fn videos_post( next_page.to_string(), per_page_clone, options_clone, - ) - .await; + ), + ) + .await; }); //### @@ -1119,8 +321,9 @@ async fn videos_post( if video.duration <= 120 { let mut preview_url = video.url.clone(); if let Some(x) = &video.formats { - // preview is a String here, so use it directly - preview_url = x[0].url.clone(); + if let Some(first) = x.first() { + preview_url = first.url.clone(); + } } video.preview = Some(preview_url); } @@ -1130,36 +333,10 @@ async fn videos_post( } pub fn get_provider(channel: &str) -> Option { - match channel { - "all" => Some(Arc::new(AllProvider::new())), - "perverzija" => Some(Arc::new(PerverzijaProvider::new())), - "hanime" => Some(Arc::new(HanimeProvider::new())), - "pornhub" => Some(Arc::new(PornhubProvider::new())), - "rule34video" => Some(Arc::new(Rule34videoProvider::new())), - "redtube" => Some(Arc::new(RedtubeProvider::new())), - "okporn" => Some(Arc::new(OkpornProvider::new())), - "pornhat" => Some(Arc::new(crate::providers::pornhat::PornhatProvider::new())), - "perfectgirls" => Some(Arc::new( - crate::providers::perfectgirls::PerfectgirlsProvider::new(), - )), - "okxxx" => Some(Arc::new(crate::providers::okxxx::OkxxxProvider::new())), - "homoxxx" => Some(Arc::new(crate::providers::homoxxx::HomoxxxProvider::new())), - "missav" => Some(Arc::new(crate::providers::missav::MissavProvider::new())), - "xxthots" => Some(Arc::new(crate::providers::xxthots::XxthotsProvider::new())), - "sxyprn" => Some(Arc::new(crate::providers::sxyprn::SxyprnProvider::new())), - "porn00" => Some(Arc::new(crate::providers::porn00::Porn00Provider::new())), - "youjizz" => Some(Arc::new(crate::providers::youjizz::YoujizzProvider::new())), - "paradisehill" => Some(Arc::new( - crate::providers::paradisehill::ParadisehillProvider::new(), - )), - "pornzog" => Some(Arc::new(crate::providers::pornzog::PornzogProvider::new())), - // fallback to the dynamic registry - x => ALL_PROVIDERS.get(x).cloned(), - } + ALL_PROVIDERS.get(channel).cloned() } pub async fn test() -> Result { - // Simply await the function instead of blocking the thread let e = io::Error::new(io::ErrorKind::Other, "test error"); let _ = send_discord_error_report( e.to_string(), diff --git a/src/providers/all.rs b/src/providers/all.rs index 1a00722..7c52e19 100644 --- a/src/providers/all.rs +++ b/src/providers/all.rs @@ -1,12 +1,14 @@ use std::fs; use std::time::Duration; use async_trait::async_trait; +use capitalize::Capitalize; +use cute::c; use error_chain::error_chain; use futures::StreamExt; use futures::stream::FuturesUnordered; use crate::api::{get_provider, ClientVersion}; -use crate::providers::{DynProvider, Provider}; -use crate::status::Channel; +use crate::providers::{DynProvider, Provider, report_provider_error, run_provider_guarded}; +use crate::status::{Channel, ChannelOption, FilterOption}; use crate::util::cache::VideoCache; use crate::util::interleave; use crate::videos::{ServerOptions, VideoItem}; @@ -45,24 +47,50 @@ impl Provider for AllProvider { ) -> Vec { let mut sites_str = options.clone().sites.unwrap_or_default(); if sites_str.is_empty() { - let files = fs::read_dir("./src/providers").unwrap(); - let providers = files.map(|entry| entry.unwrap().file_name()) - .filter(|name| name.to_str().unwrap().ends_with(".rs")) - .filter(|name| !name.to_str().unwrap().contains("mod.rs") && !name.to_str().unwrap().contains("all.rs")) - .map(|name| name.to_str().unwrap().replace(".rs", "")) + let files = match fs::read_dir("./src/providers") { + Ok(files) => files, + Err(e) => { + report_provider_error("all", "all.get_videos.read_dir", &e.to_string()).await; + return vec![]; + } + }; + let providers = files + .filter_map(|entry| entry.ok()) + .filter_map(|entry| entry.file_name().into_string().ok()) + .filter(|name| name.ends_with(".rs")) + .filter(|name| !name.contains("mod.rs") && !name.contains("all.rs")) + .map(|name| name.replace(".rs", "")) .collect::>(); sites_str = providers.join(","); } - let providers: Vec = sites_str + let providers: Vec<(String, DynProvider)> = sites_str .split(',') + .map(str::trim) .filter(|s| !s.is_empty()) - .filter_map(|s| get_provider(s)) + .filter_map(|s| { + let provider = get_provider(s); + if provider.is_none() { + Some((s.to_string(), None)) + } else { + provider.map(|p| (s.to_string(), Some(p))) + } + }) + .filter_map(|(name, provider)| match provider { + Some(provider) => Some((name, provider)), + None => { + // fire-and-forget reporting of missing provider keys + tokio::spawn(async move { + report_provider_error("all", "all.get_videos.unknown_provider", &name).await; + }); + None + } + }) .collect(); let mut futures = FuturesUnordered::new(); - for provider in providers { + for (provider_name, provider) in providers { let cache = cache.clone(); let pool = pool.clone(); let sort = sort.clone(); @@ -70,10 +98,16 @@ impl Provider for AllProvider { let page = page.clone(); let per_page = per_page.clone(); let options = options.clone(); + let provider_name_cloned = provider_name.clone(); // Spawn the task so it lives independently of this function futures.push(tokio::spawn(async move { - provider.get_videos(cache, pool, sort, query, page, per_page, options).await + run_provider_guarded( + &provider_name_cloned, + "all.get_videos.provider_task", + provider.get_videos(cache, pool, sort, query, page, per_page, options), + ) + .await })); } @@ -85,9 +119,11 @@ impl Provider for AllProvider { loop { tokio::select! { Some(result) = futures.next() => { - // Ignore errors (panics or task cancellations) - if let Ok(videos) = result { - all_results.push(videos); + match result { + Ok(videos) => all_results.push(videos), + Err(e) => { + report_provider_error("all", "all.get_videos.join_error", &e.to_string()).await; + } } }, _ = &mut timeout_timer => { @@ -105,17 +141,41 @@ impl Provider for AllProvider { fn get_channel(&self, clientversion: ClientVersion) -> Option { let _ = clientversion; + let files = fs::read_dir("./src/providers").ok()?; + let providers = files + .filter_map(|entry| entry.ok()) + .filter_map(|entry| entry.file_name().into_string().ok()) + .filter(|name| name.ends_with(".rs")) + .filter(|name| { + !name.contains("mod.rs") + && !name.contains("all.rs") + }) + .map(|name| name.replace(".rs", "")) + .collect::>(); + let sites = c![FilterOption { + id: x.to_string(), + title: x.capitalize().to_string(), + }, for x in providers.iter()]; + Some(Channel { - id: "placeholder".to_string(), - name: "PLACEHOLDER".to_string(), - description: "PLACEHOLDER FOR PARENT CLASS".to_string(), + id: "all".to_string(), + name: "All".to_string(), + description: "Query from all sites of this Server".to_string(), premium: false, favicon: "https://hottub.spacemoehre.de/favicon.ico".to_string(), status: "active".to_string(), categories: vec![], - options: vec![], + options: vec![ChannelOption { + id: "sites".to_string(), + title: "Sites".to_string(), + description: "What Sites to use".to_string(), + systemImage: "list.number".to_string(), + colorName: "green".to_string(), + options: sites, + multiSelect: true, + }], nsfw: true, - cacheDuration: None, + cacheDuration: Some(1800), }) } } diff --git a/src/providers/beeg.rs b/src/providers/beeg.rs index 2450b8f..95da1c1 100644 --- a/src/providers/beeg.rs +++ b/src/providers/beeg.rs @@ -1,6 +1,6 @@ use crate::DbPool; use crate::api::ClientVersion; -use crate::providers::Provider; +use crate::providers::{Provider, report_provider_error_background}; use crate::util::cache::VideoCache; use crate::util::parse_abbreviated_number; use crate::videos::{ServerOptions, VideoItem}; @@ -203,23 +203,20 @@ impl BeegProvider { options: ServerOptions, ) -> Result> { let mut slug = ""; - if options.categories.is_some() - && !options.categories.as_ref().unwrap().is_empty() - && options.categories.as_ref().unwrap() != "all" - { - slug = options.categories.as_ref().unwrap(); + if let Some(categories) = options.categories.as_ref() { + if !categories.is_empty() && categories != "all" { + slug = categories; + } } - if options.sites.is_some() - && !options.sites.as_ref().unwrap().is_empty() - && options.sites.as_ref().unwrap() != "all" - { - slug = options.sites.as_ref().unwrap(); + if let Some(sites) = options.sites.as_ref() { + if !sites.is_empty() && sites != "all" { + slug = sites; + } } - if options.stars.is_some() - && !options.stars.as_ref().unwrap().is_empty() - && options.stars.as_ref().unwrap() != "all" - { - slug = options.stars.as_ref().unwrap(); + if let Some(stars) = options.stars.as_ref() { + if !stars.is_empty() && stars != "all" { + slug = stars; + } } let video_url = format!( "https://store.externulls.com/facts/tag?limit=100&offset={}{}", @@ -240,9 +237,21 @@ impl BeegProvider { vec![] } }; - let mut requester = options.requester.clone().unwrap(); - let text = requester.get(&video_url, None).await.unwrap(); - let json: serde_json::Value = serde_json::from_str::(&text).unwrap(); + let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); + let text = match requester.get(&video_url, None).await { + Ok(text) => text, + Err(e) => { + report_provider_error_background("beeg", "get.request", &e.to_string()); + return Ok(old_items); + } + }; + let json: serde_json::Value = match serde_json::from_str::(&text) { + Ok(json) => json, + Err(e) => { + report_provider_error_background("beeg", "get.parse_json", &e.to_string()); + return Ok(old_items); + } + }; let video_items: Vec = self.get_video_items_from_html(json.clone()); if !video_items.is_empty() { cache.remove(&video_url); @@ -280,10 +289,22 @@ impl BeegProvider { } }; - let mut requester = options.requester.clone().unwrap(); + let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); - let text = requester.get(&video_url, None).await.unwrap(); - let json: serde_json::Value = serde_json::from_str::(&text).unwrap(); + let text = match requester.get(&video_url, None).await { + Ok(text) => text, + Err(e) => { + report_provider_error_background("beeg", "query.request", &e.to_string()); + return Ok(old_items); + } + }; + let json: serde_json::Value = match serde_json::from_str::(&text) { + Ok(json) => json, + Err(e) => { + report_provider_error_background("beeg", "query.parse_json", &e.to_string()); + return Ok(old_items); + } + }; let video_items: Vec = self.get_video_items_from_html(json.clone()); if !video_items.is_empty() { cache.remove(&video_url); diff --git a/src/providers/chaturbate.rs b/src/providers/chaturbate.rs index d0023eb..040fd66 100644 --- a/src/providers/chaturbate.rs +++ b/src/providers/chaturbate.rs @@ -1,6 +1,6 @@ use crate::DbPool; use crate::api::ClientVersion; -use crate::providers::Provider; +use crate::providers::{Provider, report_provider_error}; use crate::status::*; use crate::util::cache::VideoCache; use crate::videos::{ServerOptions, VideoItem}; @@ -89,17 +89,38 @@ impl ChaturbateProvider { } }; - let mut requester = options.requester.clone().unwrap(); - let text = requester + let mut requester = + crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); + let response = match requester .get_raw_with_headers( &video_url, vec![("X-Requested-With".to_string(), "XMLHttpRequest".to_string())], ) .await - .unwrap() - .text() - .await - .unwrap(); + { + Ok(response) => response, + Err(e) => { + report_provider_error( + "chaturbate", + "get.request", + &format!("url={video_url}; error={e}"), + ) + .await; + return Ok(old_items); + } + }; + let text = match response.text().await { + Ok(text) => text, + Err(e) => { + report_provider_error( + "chaturbate", + "get.response_text", + &format!("url={video_url}; error={e}"), + ) + .await; + return Ok(old_items); + } + }; let video_items: Vec = self.get_video_items_from_html(text.clone()); if !video_items.is_empty() { cache.remove(&video_url); @@ -139,18 +160,38 @@ impl ChaturbateProvider { } }; - let mut requester = options.requester.clone().unwrap(); - - let text = requester + let mut requester = + crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); + let response = match requester .get_raw_with_headers( &video_url, vec![("X-Requested-With".to_string(), "XMLHttpRequest".to_string())], ) .await - .unwrap() - .text() - .await - .unwrap(); + { + Ok(response) => response, + Err(e) => { + report_provider_error( + "chaturbate", + "query.request", + &format!("url={video_url}; error={e}"), + ) + .await; + return Ok(old_items); + } + }; + let text = match response.text().await { + Ok(text) => text, + Err(e) => { + report_provider_error( + "chaturbate", + "query.response_text", + &format!("url={video_url}; error={e}"), + ) + .await; + return Ok(old_items); + } + }; let video_items: Vec = self.get_video_items_from_html(text.clone()); if !video_items.is_empty() { cache.remove(&video_url); @@ -171,7 +212,18 @@ impl ChaturbateProvider { println!("Failed to parse JSON: {}", e); serde_json::Value::Null }); - for video_segment in json.get("rooms").unwrap().as_array().unwrap_or(&vec![]) { + let rooms = match json.get("rooms").and_then(|v| v.as_array()) { + Some(rooms) => rooms, + None => { + crate::providers::report_provider_error_background( + "chaturbate", + "get_video_items_from_html.rooms_missing", + "missing rooms array", + ); + return items; + } + }; + for video_segment in rooms { if video_segment .get("has_password") .unwrap_or(&serde_json::Value::Bool(false)) @@ -184,10 +236,18 @@ impl ChaturbateProvider { // for (index, line) in vid.iter().enumerate() { // println!("Line {}: {}", index, line); // } - let username = video_segment + let Some(username) = video_segment .get("username") .and_then(|v| v.as_str()) - .map(String::from).unwrap(); + .map(String::from) + else { + crate::providers::report_provider_error_background( + "chaturbate", + "get_video_items_from_html.username_missing", + "missing username field", + ); + continue; + }; let video_url: String = format!("{}/{}/", self.url, username); let mut title = video_segment .get("room_subject") @@ -202,7 +262,7 @@ impl ChaturbateProvider { .unwrap_or(&serde_json::Value::String("".to_string())) .as_str() .unwrap_or("") - .split("?").collect::>()[0] + .split("?").collect::>().get(0).copied().unwrap_or_default() .to_string(); let views = video_segment .get("viewers") diff --git a/src/providers/freepornvideosxxx.rs b/src/providers/freepornvideosxxx.rs index 423f527..58b4ef5 100644 --- a/src/providers/freepornvideosxxx.rs +++ b/src/providers/freepornvideosxxx.rs @@ -1,6 +1,6 @@ use crate::DbPool; use crate::api::ClientVersion; -use crate::providers::Provider; +use crate::providers::{Provider, report_provider_error, report_provider_error_background}; use crate::util::cache::VideoCache; use crate::util::parse_abbreviated_number; use crate::util::time::parse_time_to_seconds; @@ -58,10 +58,20 @@ impl FreepornvideosxxxProvider { thread::spawn(move || { // Create a tiny runtime just for these async tasks - let rt = tokio::runtime::Builder::new_current_thread() + let rt = match tokio::runtime::Builder::new_current_thread() .enable_all() .build() - .expect("build tokio runtime"); + { + Ok(rt) => rt, + Err(e) => { + report_provider_error_background( + "freepornvideosxxx", + "spawn_initial_load.runtime_build", + &e.to_string(), + ); + return; + } + }; rt.block_on(async move { // If you have a streaming sites loader, call it here too @@ -83,13 +93,23 @@ impl FreepornvideosxxxProvider { async fn load_stars(base_url: &str, stars: Arc>>) -> Result<()> { let mut requester = util::requester::Requester::new(); for page in [1..10].into_iter().flatten() { - let text = requester + let text = match requester .get( format!("{}/models/total-videos/{}/?gender_id=0", &base_url, page).as_str(), None, ) .await - .unwrap(); + { + Ok(text) => text, + Err(e) => { + report_provider_error_background( + "freepornvideosxxx", + "load_stars.request", + &format!("url={base_url}; page={page}; error={e}"), + ); + break; + } + }; if text.contains("404 Not Found") || text.is_empty() { break; } @@ -97,19 +117,20 @@ impl FreepornvideosxxxProvider { .split("
") .collect::>() .last() - .unwrap() + .copied() + .unwrap_or_default() .split("custom_list_models_models_list_pagination") - .collect::>()[0]; + .collect::>().get(0).copied().unwrap_or_default(); for stars_element in stars_div.split(">()[1..].to_vec() { - let star_url = stars_element.split("href=\"").collect::>()[1] + let star_url = stars_element.split("href=\"").collect::>().get(1).copied().unwrap_or_default() .split("\"") - .collect::>()[0]; - let star_id = star_url.split("/").collect::>()[4].to_string(); + .collect::>().get(0).copied().unwrap_or_default(); + let star_id = star_url.split("/").collect::>().get(4).copied().unwrap_or_default().to_string(); let star_name = stars_element .split("") - .collect::>()[1] + .collect::>().get(1).copied().unwrap_or_default() .split("<") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .to_string(); Self::push_unique( &stars, @@ -130,26 +151,36 @@ impl FreepornvideosxxxProvider { page += 1; let text = requester .get(format!("{}/sites/{}/", &base_url, page).as_str(), None) - .await - .unwrap(); + .await; + let text = match text { + Ok(text) => text, + Err(e) => { + report_provider_error_background( + "freepornvideosxxx", + "load_sites.request", + &format!("url={base_url}; page={page}; error={e}"), + ); + break; + } + }; if text.contains("404 Not Found") || text.is_empty() { break; } let sites_div = text .split("id=\"list_content_sources_sponsors_list_items\"") - .collect::>()[1] + .collect::>().get(1).copied().unwrap_or_default() .split("class=\"pagination\"") - .collect::>()[0]; + .collect::>().get(0).copied().unwrap_or_default(); for sites_element in sites_div.split("class=\"headline\"").collect::>()[1..].to_vec() { - let site_url = sites_element.split("href=\"").collect::>()[1] + let site_url = sites_element.split("href=\"").collect::>().get(1).copied().unwrap_or_default() .split("\"") - .collect::>()[0]; - let site_id = site_url.split("/").collect::>()[4].to_string(); - let site_name = sites_element.split("

").collect::>()[1] + .collect::>().get(0).copied().unwrap_or_default(); + let site_id = site_url.split("/").collect::>().get(4).copied().unwrap_or_default().to_string(); + let site_name = sites_element.split("

").collect::>().get(1).copied().unwrap_or_default() .split("<") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .to_string(); Self::push_unique( &sites, @@ -165,23 +196,33 @@ impl FreepornvideosxxxProvider { async fn load_networks(base_url: &str, networks: Arc>>) -> Result<()> { let mut requester = util::requester::Requester::new(); - let text = requester.get(&base_url, None).await.unwrap(); - let networks_div = text.split("class=\"sites__list\"").collect::>()[1] + let text = match requester.get(&base_url, None).await { + Ok(text) => text, + Err(e) => { + report_provider_error_background( + "freepornvideosxxx", + "load_networks.request", + &format!("url={base_url}; error={e}"), + ); + return Ok(()); + } + }; + let networks_div = text.split("class=\"sites__list\"").collect::>().get(1).copied().unwrap_or_default() .split("

") - .collect::>()[0]; + .collect::>().get(0).copied().unwrap_or_default(); for network_element in networks_div.split("sites__item").collect::>()[1..].to_vec() { if network_element.contains("sites__all") { continue; } - let network_url = network_element.split("href=\"").collect::>()[1] + let network_url = network_element.split("href=\"").collect::>().get(1).copied().unwrap_or_default() .split("\"") - .collect::>()[0]; - let network_id = network_url.split("/").collect::>()[4].to_string(); - let network_name = network_element.split(">").collect::>()[1] + .collect::>().get(0).copied().unwrap_or_default(); + let network_id = network_url.split("/").collect::>().get(4).copied().unwrap_or_default().to_string(); + let network_name = network_element.split(">").collect::>().get(1).copied().unwrap_or_default() .split("<") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .to_string(); Self::push_unique( &networks, @@ -306,35 +347,20 @@ impl FreepornvideosxxxProvider { "most-popular" => "/most-popular".to_string(), _ => "".to_string(), }; - if options.network.is_some() - && !options.network.as_ref().unwrap().is_empty() - && options.network.as_ref().unwrap() != "all" - { - sort_string = format!( - "networks/{}{}", - options.network.as_ref().unwrap(), - alt_sort_string - ); + if let Some(network) = options.network.as_deref() { + if !network.is_empty() && network != "all" { + sort_string = format!("networks/{}{}", network, alt_sort_string); + } } - if options.sites.is_some() - && !options.sites.as_ref().unwrap().is_empty() - && options.sites.as_ref().unwrap() != "all" - { - sort_string = format!( - "sites/{}{}", - options.sites.as_ref().unwrap(), - alt_sort_string - ); + if let Some(site) = options.sites.as_deref() { + if !site.is_empty() && site != "all" { + sort_string = format!("sites/{}{}", site, alt_sort_string); + } } - if options.stars.is_some() - && !options.stars.as_ref().unwrap().is_empty() - && options.stars.as_ref().unwrap() != "all" - { - sort_string = format!( - "models/{}{}", - options.stars.as_ref().unwrap(), - alt_sort_string - ); + if let Some(star) = options.stars.as_deref() { + if !star.is_empty() && star != "all" { + sort_string = format!("models/{}{}", star, alt_sort_string); + } } let video_url = format!("{}/{}/{}/", self.url, sort_string, page); let old_items = match cache.get(&video_url) { @@ -350,8 +376,20 @@ impl FreepornvideosxxxProvider { } }; - let mut requester = options.requester.clone().unwrap(); - let text = requester.get(&video_url, None).await.unwrap(); + let mut requester = + crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); + let text = match requester.get(&video_url, None).await { + Ok(text) => text, + Err(e) => { + report_provider_error( + "freepornvideosxxx", + "get.request", + &format!("url={video_url}; error={e}"), + ) + .await; + return Ok(old_items); + } + }; let video_items: Vec = self.get_video_items_from_html(text.clone()); if !video_items.is_empty() { cache.remove(&video_url); @@ -371,31 +409,41 @@ impl FreepornvideosxxxProvider { ) -> Result> { let mut search_type = "search"; let mut search_string = query.to_string().to_ascii_lowercase().trim().to_string(); - match self - .stars - .read() - .unwrap() - .iter() - .find(|s| s.title.to_ascii_lowercase() == search_string) - { - Some(star) => { - search_type = "models"; - search_string = star.id.clone(); + match self.stars.read() { + Ok(stars) => { + if let Some(star) = stars + .iter() + .find(|s| s.title.to_ascii_lowercase() == search_string) + { + search_type = "models"; + search_string = star.id.clone(); + } + } + Err(e) => { + report_provider_error_background( + "freepornvideosxxx", + "query.stars_read", + &e.to_string(), + ); } - _ => {} } - match self - .sites - .read() - .unwrap() - .iter() - .find(|s| s.title.to_ascii_lowercase() == search_string) - { - Some(site) => { - search_type = "sites"; - search_string = site.id.clone(); + match self.sites.read() { + Ok(sites) => { + if let Some(site) = sites + .iter() + .find(|s| s.title.to_ascii_lowercase() == search_string) + { + search_type = "sites"; + search_string = site.id.clone(); + } + } + Err(e) => { + report_provider_error_background( + "freepornvideosxxx", + "query.sites_read", + &e.to_string(), + ); } - _ => {} } let mut video_url = format!("{}/{}/{}/{}/", self.url, search_type, search_string, page); video_url = video_url.replace(" ", "+"); @@ -414,9 +462,20 @@ impl FreepornvideosxxxProvider { } }; - let mut requester = options.requester.clone().unwrap(); - - let text = requester.get(&video_url, None).await.unwrap(); + let mut requester = + crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); + let text = match requester.get(&video_url, None).await { + Ok(text) => text, + Err(e) => { + report_provider_error( + "freepornvideosxxx", + "query.request", + &format!("url={video_url}; error={e}"), + ) + .await; + return Ok(old_items); + } + }; let video_items: Vec = self.get_video_items_from_html(text.clone()); if !video_items.is_empty() { cache.remove(&video_url); @@ -429,7 +488,18 @@ impl FreepornvideosxxxProvider { fn get_site_id_from_name(&self, site_name: &str) -> Option { // site_name.to_lowercase().replace(" ", "") - for site in self.sites.read().unwrap().iter() { + let sites_guard = match self.sites.read() { + Ok(guard) => guard, + Err(e) => { + report_provider_error_background( + "freepornvideosxxx", + "get_site_id_from_name.sites_read", + &e.to_string(), + ); + return None; + } + }; + for site in sites_guard.iter() { if site .title .to_lowercase() @@ -452,11 +522,11 @@ impl FreepornvideosxxxProvider { if !html.contains("class=\"item\"") { return items; } - let raw_videos = html.split("videos_list_pagination").collect::>()[0] + let raw_videos = html.split("videos_list_pagination").collect::>().get(0).copied().unwrap_or_default() .split(" class=\"pagination\" ") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .split("class=\"list-videos\"") - .collect::>()[1] + .collect::>().get(1).copied().unwrap_or_default() .split("class=\"item\"") .collect::>()[1..] .to_vec(); @@ -465,39 +535,39 @@ impl FreepornvideosxxxProvider { // for (index, line) in vid.iter().enumerate() { // println!("Line {}: {}", index, line); // } - let video_url: String = video_segment.split(">()[1] + let video_url: String = video_segment.split(">().get(1).copied().unwrap_or_default() .split("\"") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .to_string(); - let mut title = video_segment.split(" title=\"").collect::>()[1] + let mut title = video_segment.split(" title=\"").collect::>().get(1).copied().unwrap_or_default() .split("\"") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .to_string(); // html decode title = decode(title.as_bytes()).to_string().unwrap_or(title); - let id = video_url.split("/").collect::>()[4].to_string(); + let id = video_url.split("/").collect::>().get(4).copied().unwrap_or_default().to_string(); - let thumb = match video_segment.split(">()[1] + let thumb = match video_segment.split(">().get(1).copied().unwrap_or_default() .contains("data-src=\"") { - true => video_segment.split(">()[1] + true => video_segment.split(">().get(1).copied().unwrap_or_default() .split("data-src=\"") - .collect::>()[1] + .collect::>().get(1).copied().unwrap_or_default() .split("\"") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .to_string(), - false => video_segment.split(">()[1] + false => video_segment.split(">().get(1).copied().unwrap_or_default() .split("src=\"") - .collect::>()[1] + .collect::>().get(1).copied().unwrap_or_default() .split("\"") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .to_string(), }; let raw_duration = video_segment .split("") - .collect::>()[1] + .collect::>().get(1).copied().unwrap_or_default() .split("<") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .split(" ") .collect::>() .last() @@ -507,9 +577,9 @@ impl FreepornvideosxxxProvider { let views = parse_abbreviated_number( video_segment .split("
") - .collect::>()[1] + .collect::>().get(1).copied().unwrap_or_default() .split("<") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .to_string() .as_str(), ) @@ -517,9 +587,9 @@ impl FreepornvideosxxxProvider { let preview = video_segment .split("data-preview=\"") - .collect::>()[1] + .collect::>().get(1).copied().unwrap_or_default() .split("\"") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .to_string(); let site_name = title .split("]") @@ -533,9 +603,9 @@ impl FreepornvideosxxxProvider { let mut tags = match video_segment.contains("class=\"models\">") { true => video_segment .split("class=\"models\">") - .collect::>()[1] + .collect::>().get(1).copied().unwrap_or_default() .split("
") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .split("href=\"") .collect::>()[1..] .into_iter() @@ -543,17 +613,17 @@ impl FreepornvideosxxxProvider { Self::push_unique( &self.stars, FilterOption { - id: s.split("/").collect::>()[4].to_string(), - title: s.split(">").collect::>()[1] + id: s.split("/").collect::>().get(4).copied().unwrap_or_default().to_string(), + title: s.split(">").collect::>().get(1).copied().unwrap_or_default() .split("<") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .trim() .to_string(), }, ); - s.split(">").collect::>()[1] + s.split(">").collect::>().get(1).copied().unwrap_or_default() .split("<") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .trim() .to_string() }) diff --git a/src/providers/hanime.rs b/src/providers/hanime.rs index 5b4d7b2..c830dc7 100644 --- a/src/providers/hanime.rs +++ b/src/providers/hanime.rs @@ -4,9 +4,11 @@ use futures::future::join_all; use serde_json::json; use std::vec; +use crate::api::ClientVersion; use crate::DbPool; use crate::db; -use crate::providers::Provider; +use crate::providers::{Provider, report_provider_error, report_provider_error_background}; +use crate::status::*; use crate::util::cache::VideoCache; use crate::videos::{self, ServerOptions, VideoItem}; @@ -124,13 +126,83 @@ impl HanimeProvider { } } + fn build_channel(&self, _clientversion: ClientVersion) -> Channel { + Channel { + id: "hanime".to_string(), + name: "Hanime".to_string(), + description: "Free Hentai from Hanime".to_string(), + premium: false, + favicon: "https://www.google.com/s2/favicons?sz=64&domain=hanime.tv".to_string(), + status: "active".to_string(), + categories: vec![], + options: vec![ChannelOption { + id: "sort".to_string(), + title: "Sort".to_string(), + description: "Sort the Videos".to_string(), + systemImage: "list.number".to_string(), + colorName: "blue".to_string(), + options: vec![ + FilterOption { + id: "created_at_unix.desc".to_string(), + title: "Recent Upload".to_string(), + }, + FilterOption { + id: "created_at_unix.asc".to_string(), + title: "Old Upload".to_string(), + }, + FilterOption { + id: "views.desc".to_string(), + title: "Most Views".to_string(), + }, + FilterOption { + id: "views.asc".to_string(), + title: "Least Views".to_string(), + }, + FilterOption { + id: "likes.desc".to_string(), + title: "Most Likes".to_string(), + }, + FilterOption { + id: "likes.asc".to_string(), + title: "Least Likes".to_string(), + }, + FilterOption { + id: "released_at_unix.desc".to_string(), + title: "New".to_string(), + }, + FilterOption { + id: "released_at_unix.asc".to_string(), + title: "Old".to_string(), + }, + FilterOption { + id: "title_sortable.asc".to_string(), + title: "A - Z".to_string(), + }, + FilterOption { + id: "title_sortable.desc".to_string(), + title: "Z - A".to_string(), + }, + ], + multiSelect: false, + }], + nsfw: true, + cacheDuration: None, + } + } + async fn get_video_item( &self, hit: HanimeSearchResult, pool: DbPool, options: ServerOptions, ) -> Result { - let mut conn = pool.get().expect("couldn't get db connection from pool"); + let mut conn = match pool.get() { + Ok(conn) => conn, + Err(e) => { + report_provider_error("hanime", "get_video_item.pool_get", &e.to_string()).await; + return Err(Error::from("Failed to get DB connection")); + } + }; let db_result = db::get_video( &mut conn, format!( @@ -169,13 +241,24 @@ impl HanimeProvider { "m3u8".to_string(), )])); } else { - let _ = db::delete_video( - &mut pool.get().expect("couldn't get db connection from pool"), - format!( - "https://h.freeanimehentai.net/api/v8/video?id={}&", - hit.slug.clone() - ), - ); + match pool.get() { + Ok(mut conn) => { + let _ = db::delete_video( + &mut conn, + format!( + "https://h.freeanimehentai.net/api/v8/video?id={}&", + hit.slug.clone() + ), + ); + } + Err(e) => { + report_provider_error_background( + "hanime", + "get_video_item.delete_video.pool_get", + &e.to_string(), + ); + } + } } } Ok(None) => (), @@ -189,7 +272,7 @@ impl HanimeProvider { id ); - let mut requester = options.requester.clone().unwrap(); + let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); let payload = json!({ "width": 571, "height": 703, "ab": "kh" } ); @@ -216,41 +299,71 @@ impl HanimeProvider { ], ) .await - .unwrap() + .map_err(|e| { + report_provider_error_background( + "hanime", + "get_video_item.get_raw_with_headers", + &e.to_string(), + ); + Error::from(format!("Failed to fetch manifest response: {e}")) + })? .text() .await - .unwrap(); + .map_err(|e| { + report_provider_error_background( + "hanime", + "get_video_item.response_text", + &e.to_string(), + ); + Error::from(format!("Failed to decode manifest response body: {e}")) + })?; if text.contains("Unautho") { println!("Fetched video details for {}: {}", title, text); return Err(Error::from("Unauthorized")); } - let urls = text.split("streams").collect::>()[1]; + let urls = text + .split("streams") + .nth(1) + .ok_or_else(|| Error::from("Missing streams section in manifest"))?; let mut url_vec = vec![]; for el in urls.split("\"url\":\"").collect::>() { - let url = el.split("\"").collect::>()[0]; + let url = el.split("\"").collect::>().get(0).copied().unwrap_or_default(); if !url.is_empty() && url.contains("m3u8") { url_vec.push(url.to_string()); } } - let mut conn = pool.get().expect("couldn't get db connection from pool"); - let _ = db::insert_video( - &mut conn, - &format!( - "https://h.freeanimehentai.net/api/v8/video?id={}&", - hit.slug.clone() - ), - &url_vec[0].clone(), - ); - drop(conn); + let first_url = url_vec + .first() + .cloned() + .ok_or_else(|| Error::from("No stream URL found in manifest"))?; + match pool.get() { + Ok(mut conn) => { + let _ = db::insert_video( + &mut conn, + &format!( + "https://h.freeanimehentai.net/api/v8/video?id={}&", + hit.slug.clone() + ), + &first_url, + ); + } + Err(e) => { + report_provider_error_background( + "hanime", + "get_video_item.insert_video.pool_get", + &e.to_string(), + ); + } + } Ok( - VideoItem::new(id, title, url_vec[0].clone(), channel, thumb, duration) + VideoItem::new(id, title, first_url.clone(), channel, thumb, duration) .tags(hit.tags) .uploader(hit.brand) .views(hit.views as u32) .rating((hit.likes as f32 / (hit.likes + hit.dislikes) as f32) * 100 as f32) .formats(vec![videos::VideoFormat::new( - url_vec[0].clone(), + first_url, "1080".to_string(), "m3u8".to_string(), )]), @@ -268,11 +381,11 @@ impl HanimeProvider { ) -> Result> { let index = format!("hanime:{}:{}:{}", query, page, sort); let order_by = match sort.contains(".") { - true => sort.split(".").collect::>()[0].to_string(), + true => sort.split(".").collect::>().get(0).copied().unwrap_or_default().to_string(), false => "created_at_unix".to_string(), }; let ordering = match sort.contains(".") { - true => sort.split(".").collect::>()[1].to_string(), + true => sort.split(".").collect::>().get(1).copied().unwrap_or_default().to_string(), false => "desc".to_string(), }; let old_items = match cache.get(&index) { @@ -295,11 +408,22 @@ impl HanimeProvider { .order_by(order_by) .ordering(ordering); - let mut requester = options.requester.clone().unwrap(); - let response = requester + let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); + let response = match requester .post_json("https://search.htv-services.com/search", &search, vec![]) .await - .unwrap(); + { + Ok(response) => response, + Err(e) => { + report_provider_error( + "hanime", + "get.search_request", + &format!("query={query}; page={page}; error={e}"), + ) + .await; + return Ok(old_items); + } + }; let hits = match response.json::().await { Ok(resp) => resp.hits, @@ -374,4 +498,8 @@ impl Provider for HanimeProvider { } } } + + fn get_channel(&self, clientversion: ClientVersion) -> Option { + Some(self.build_channel(clientversion)) + } } diff --git a/src/providers/hentaihaven.rs b/src/providers/hentaihaven.rs index 3ddbde8..edf85a9 100644 --- a/src/providers/hentaihaven.rs +++ b/src/providers/hentaihaven.rs @@ -57,10 +57,15 @@ impl HentaihavenProvider { categories: self .categories .read() - .unwrap() - .iter() - .map(|c| c.title.clone()) - .collect(), + .map(|categories| categories.iter().map(|c| c.title.clone()).collect()) + .unwrap_or_else(|e| { + crate::providers::report_provider_error_background( + "hentaihaven", + "build_channel.categories_read", + &e.to_string(), + ); + vec![] + }), options: vec![], nsfw: true, cacheDuration: None, @@ -98,11 +103,20 @@ impl HentaihavenProvider { } }; - let mut requester = options.requester.clone().unwrap(); - let text = requester - .get(&video_url, Some(Version::HTTP_2)) - .await - .unwrap(); + let mut requester = + crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); + let text = match requester.get(&video_url, Some(Version::HTTP_2)).await { + Ok(text) => text, + Err(e) => { + crate::providers::report_provider_error( + "hentaihaven", + "get.request", + &format!("url={video_url}; error={e}"), + ) + .await; + return Ok(old_items); + } + }; let video_items: Vec = self .get_video_items_from_html(text.clone(), &mut requester, pool.clone()) .await; @@ -143,11 +157,20 @@ impl HentaihavenProvider { } }; - let mut requester = options.requester.clone().unwrap(); - let text = requester - .get(&video_url, Some(Version::HTTP_2)) - .await - .unwrap(); + let mut requester = + crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); + let text = match requester.get(&video_url, Some(Version::HTTP_2)).await { + Ok(text) => text, + Err(e) => { + crate::providers::report_provider_error( + "hentaihaven", + "query.request", + &format!("url={video_url}; error={e}"), + ) + .await; + return Ok(old_items); + } + }; if page > 1 { return Ok(vec![]); @@ -308,7 +331,23 @@ impl HentaihavenProvider { .and_then(|s| s.split('"').next()) .ok_or_else(|| ErrorKind::Parse("video url\n\n{seg}".into()))? .to_string(); - let mut conn = pool.get().expect("couldn't get db connection from pool"); + let mut conn = match pool.get() { + Ok(conn) => conn, + Err(e) => { + let msg = format!("DB pool error: {}", e); + send_discord_error_report( + msg.clone(), + None, + Some("Hentai Haven Provider"), + Some("get_video_item.pool_get"), + file!(), + line!(), + module_path!(), + ) + .await; + return Err(msg.into()); + } + }; let db_result = db::get_video(&mut conn, video_url.clone()); drop(conn); match db_result { @@ -456,13 +495,27 @@ impl HentaihavenProvider { .views(views) .aspect_ratio(0.715); - let mut conn = pool.get().expect("couldn't get db connection from pool"); - let _ = db::insert_video( - &mut conn, - &video_url, - &serde_json::to_string(&video_item).unwrap_or_default(), - ); - drop(conn); + match pool.get() { + Ok(mut conn) => { + let _ = db::insert_video( + &mut conn, + &video_url, + &serde_json::to_string(&video_item).unwrap_or_default(), + ); + } + Err(e) => { + send_discord_error_report( + format!("DB pool error: {}", e), + None, + Some("Hentai Haven Provider"), + Some("get_video_item.insert_video.pool_get"), + file!(), + line!(), + module_path!(), + ) + .await; + } + } Ok(video_item) } diff --git a/src/providers/homoxxx.rs b/src/providers/homoxxx.rs index 8143d80..fe676f4 100644 --- a/src/providers/homoxxx.rs +++ b/src/providers/homoxxx.rs @@ -1,5 +1,7 @@ +use crate::api::ClientVersion; use crate::DbPool; -use crate::providers::Provider; +use crate::providers::{Provider, report_provider_error}; +use crate::status::*; use crate::util::cache::VideoCache; use crate::util::flaresolverr::{FlareSolverrRequest, Flaresolverr}; use crate::util::time::parse_time_to_seconds; @@ -29,6 +31,42 @@ impl HomoxxxProvider { url: "https://homo.xxx".to_string(), } } + + fn build_channel(&self, _clientversion: ClientVersion) -> Channel { + Channel { + id: "homoxxx".to_string(), + name: "Homo.xxx".to_string(), + description: "Best Gay Porn".to_string(), + premium: false, + favicon: "https://www.google.com/s2/favicons?sz=64&domain=homo.xxx".to_string(), + status: "active".to_string(), + categories: vec![], + options: vec![ChannelOption { + id: "sort".to_string(), + title: "Sort".to_string(), + description: "Sort the Videos".to_string(), + systemImage: "list.number".to_string(), + colorName: "blue".to_string(), + options: vec![ + FilterOption { + id: "new".to_string(), + title: "New".to_string(), + }, + FilterOption { + id: "popular".to_string(), + title: "Popular".to_string(), + }, + FilterOption { + id: "trending".to_string(), + title: "Trending".to_string(), + }, + ], + multiSelect: false, + }], + nsfw: true, + cacheDuration: Some(1800), + } + } async fn get( &self, cache: VideoCache, @@ -60,11 +98,25 @@ impl HomoxxxProvider { let mut response = client.get(video_url.clone()) // .proxy(proxy.clone()) .send().await?; - if response.status().is_redirection(){ - println!("Redirection detected, following to: {}", response.headers()["Location"].to_str().unwrap()); - response = client.get(response.headers()["Location"].to_str().unwrap()) - // .proxy(proxy.clone()) - .send().await?; + if response.status().is_redirection() { + let location = match response.headers().get("Location").and_then(|h| h.to_str().ok()) { + Some(location) => location, + None => { + report_provider_error( + "homoxxx", + "get.redirect_location", + &format!("url={video_url}; missing/invalid Location header"), + ) + .await; + return Ok(old_items); + } + }; + println!("Redirection detected, following to: {}", location); + response = client + .get(location) + // .proxy(proxy.clone()) + .send() + .await?; } if response.status().is_success() { let text = response.text().await?; @@ -77,7 +129,18 @@ impl HomoxxxProvider { } Ok(video_items) } else { - let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set"); + let flare_url = match env::var("FLARE_URL") { + Ok(url) => url, + Err(e) => { + report_provider_error( + "homoxxx", + "get.flare_url", + &e.to_string(), + ) + .await; + return Ok(old_items); + } + }; let flare = Flaresolverr::new(flare_url); let result = flare .solve(FlareSolverrRequest { @@ -115,7 +178,7 @@ impl HomoxxxProvider { let mut video_url = format!("{}/search/{}/{}/", self.url, search_string, page); if search_string.starts_with("@"){ - let url_part = search_string.split("@").collect::>()[1].replace(":", "/"); + let url_part = search_string.split("@").collect::>().get(1).copied().unwrap_or_default().replace(":", "/"); video_url = format!("{}/{}/", self.url, url_part); } // Check our Video Cache. If the result is younger than 1 hour, we return it. @@ -140,11 +203,24 @@ impl HomoxxxProvider { // .proxy(proxy.clone()) .send().await?; - if response.status().is_redirection(){ - - response = client.get(self.url.clone() + response.headers()["Location"].to_str().unwrap()) - // .proxy(proxy.clone()) - .send().await?; + if response.status().is_redirection() { + let location = match response.headers().get("Location").and_then(|h| h.to_str().ok()) { + Some(location) => location, + None => { + report_provider_error( + "homoxxx", + "query.redirect_location", + &format!("url={video_url}; missing/invalid Location header"), + ) + .await; + return Ok(old_items); + } + }; + response = client + .get(self.url.clone() + location) + // .proxy(proxy.clone()) + .send() + .await?; } if response.status().is_success() { @@ -158,7 +234,18 @@ impl HomoxxxProvider { } Ok(video_items) } else { - let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set"); + let flare_url = match env::var("FLARE_URL") { + Ok(url) => url, + Err(e) => { + report_provider_error( + "homoxxx", + "query.flare_url", + &e.to_string(), + ) + .await; + return Ok(old_items); + } + }; let flare = Flaresolverr::new(flare_url); let result = flare .solve(FlareSolverrRequest { @@ -190,7 +277,7 @@ impl HomoxxxProvider { return vec![]; } let mut items: Vec = Vec::new(); - let raw_videos = html.split("pagination").collect::>()[0] + let raw_videos = html.split("pagination").collect::>().get(0).copied().unwrap_or_default() .split("
") .collect::>()[1..] .to_vec(); @@ -199,31 +286,31 @@ impl HomoxxxProvider { // for (index, line) in vid.iter().enumerate() { // println!("Line {}: {}", index, line); // } - let video_url: String = video_segment.split(">()[1] + let video_url: String = video_segment.split(">().get(1).copied().unwrap_or_default() .split("\"") - .collect::>()[0].to_string(); - let preview_url = video_segment.split("data-preview-custom=\"").collect::>()[1] + .collect::>().get(0).copied().unwrap_or_default().to_string(); + let preview_url = video_segment.split("data-preview-custom=\"").collect::>().get(1).copied().unwrap_or_default() .split("\"") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .to_string(); - let mut title = video_segment.split("\" title=\"").collect::>()[1] + let mut title = video_segment.split("\" title=\"").collect::>().get(1).copied().unwrap_or_default() .split("\"") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .to_string(); // html decode title = decode(title.as_bytes()).to_string().unwrap_or(title); - let id = video_url.split("/").collect::>()[4].to_string(); + let id = video_url.split("/").collect::>().get(4).copied().unwrap_or_default().to_string(); let raw_duration = video_segment - .split("

").collect::>()[1] + .split("

").collect::>().get(1).copied().unwrap_or_default() .split("<") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .to_string(); let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32; - let thumb = video_segment.split("thumb lazyload").collect::>()[1] - .split("data-src=\"").collect::>()[1] + let thumb = video_segment.split("thumb lazyload").collect::>().get(1).copied().unwrap_or_default() + .split("data-src=\"").collect::>().get(1).copied().unwrap_or_default() .split("\"") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .to_string(); let video_item = VideoItem::new( id, @@ -276,4 +363,8 @@ impl Provider for HomoxxxProvider { } } } + + fn get_channel(&self, clientversion: ClientVersion) -> Option { + Some(self.build_channel(clientversion)) + } } diff --git a/src/providers/hqporner.rs b/src/providers/hqporner.rs index eb062a9..4980f89 100644 --- a/src/providers/hqporner.rs +++ b/src/providers/hqporner.rs @@ -289,7 +289,7 @@ impl HqpornerProvider { } }) .filter_map(Result::ok) - .filter(|item| !item.formats.clone().unwrap().is_empty()) + .filter(|item| item.formats.as_ref().map(|formats| !formats.is_empty()).unwrap_or(false)) .collect() } @@ -409,8 +409,24 @@ impl HqpornerProvider { vec![("Referer".to_string(), "https://hqporner.com/".into())], ).await; } - let text2 = r - .unwrap() + let response = match r { + Ok(response) => response, + Err(e) => { + let err = format!("altplayer request failed: {e}"); + send_discord_error_report( + err.clone(), + None, + Some("Hqporner Provider"), + Some(&player_url), + file!(), + line!(), + module_path!(), + ) + .await; + return Ok((tags, formats)); + } + }; + let text2 = response .text() .await .map_err(|e| Error::from(format!("Text conversion failed: {}", e)))?; diff --git a/src/providers/hypnotube.rs b/src/providers/hypnotube.rs index 07ada0f..667ffa2 100644 --- a/src/providers/hypnotube.rs +++ b/src/providers/hypnotube.rs @@ -137,10 +137,16 @@ impl HypnotubeProvider { categories: self .categories .read() - .unwrap() - .iter() - .map(|c| c.title.clone()) - .collect(), + .map(|categories| categories.iter().map(|c| c.title.clone()).collect()) + .unwrap_or_else(|e| { + eprint!("Hypnotube categories lock error: {e}"); + crate::providers::report_provider_error_background( + "hypnotube", + "build_channel.categories_read", + &e.to_string(), + ); + vec![] + }), options: vec![ChannelOption { id: "sort".to_string(), title: "Sort".to_string(), @@ -206,11 +212,19 @@ impl HypnotubeProvider { vec![] } }; - let mut requester = options.requester.clone().unwrap(); - let text = requester - .get(&video_url, Some(Version::HTTP_11)) - .await - .unwrap(); + let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); + let text = match requester.get(&video_url, Some(Version::HTTP_11)).await { + Ok(text) => text, + Err(e) => { + crate::providers::report_provider_error( + "hypnotube", + "get.request", + &format!("url={video_url}; error={e}"), + ) + .await; + return old_items; + } + }; if text.contains("Sorry, no results were found.") { return vec![]; } @@ -259,21 +273,35 @@ impl HypnotubeProvider { } }; - let mut requester = options.requester.clone().unwrap(); - let text = match requester + let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); + let post_response = match requester .post( format!("{}/searchgate.php", self.url).as_str(), format!("q={}&type=videos", query.replace(" ", "+")).as_str(), vec![("Content-Type", "application/x-www-form-urlencoded")], ) .await - .unwrap() - .text() - .await { + Ok(response) => response, + Err(e) => { + crate::providers::report_provider_error( + "hypnotube", + "query.search_post", + &format!("url={video_url}; error={e}"), + ) + .await; + return old_items; + } + }; + let text = match post_response.text().await { Ok(t) => t, Err(e) => { eprint!("Hypnotube search POST request failed: {}", e); + crate::providers::report_provider_error_background( + "hypnotube", + "query.search_post.text", + &e.to_string(), + ); return vec![]; } }; diff --git a/src/providers/javtiful.rs b/src/providers/javtiful.rs index caf9e81..88fed81 100644 --- a/src/providers/javtiful.rs +++ b/src/providers/javtiful.rs @@ -58,10 +58,15 @@ impl JavtifulProvider { categories: self .categories .read() - .unwrap() - .iter() - .map(|c| c.title.clone()) - .collect(), + .map(|categories| categories.iter().map(|c| c.title.clone()).collect()) + .unwrap_or_else(|e| { + crate::providers::report_provider_error_background( + "javtiful", + "build_channel.categories_read", + &e.to_string(), + ); + vec![] + }), options: vec![ChannelOption { id: "sort".to_string(), title: "Sort".to_string(), @@ -130,8 +135,20 @@ impl JavtifulProvider { } }; - let mut requester = options.requester.clone().unwrap(); - let text = requester.get(&video_url, Some(Version::HTTP_2)).await.unwrap(); + let mut requester = + crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); + let text = match requester.get(&video_url, Some(Version::HTTP_2)).await { + Ok(text) => text, + Err(e) => { + crate::providers::report_provider_error( + "javtiful", + "get.request", + &format!("url={video_url}; error={e}"), + ) + .await; + return Ok(old_items); + } + }; if page > 1 && !text.contains(&format!("

  • {}", page)) { return Ok(vec![]); } @@ -178,8 +195,20 @@ impl JavtifulProvider { } }; - let mut requester = options.requester.clone().unwrap(); - let text = requester.get(&video_url, Some(Version::HTTP_2)).await.unwrap(); + let mut requester = + crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); + let text = match requester.get(&video_url, Some(Version::HTTP_2)).await { + Ok(text) => text, + Err(e) => { + crate::providers::report_provider_error( + "javtiful", + "query.request", + &format!("url={video_url}; error={e}"), + ) + .await; + return Ok(old_items); + } + }; if page > 1 && !text.contains(&format!("
  • {}", page)) { return Ok(vec![]); } diff --git a/src/providers/missav.rs b/src/providers/missav.rs index b8410ce..a0d56bc 100644 --- a/src/providers/missav.rs +++ b/src/providers/missav.rs @@ -5,8 +5,10 @@ use error_chain::error_chain; use htmlentity::entity::{decode, ICodedDataTrait}; use futures::future::join_all; use wreq::Version; +use crate::api::ClientVersion; use crate::db; use crate::providers::Provider; +use crate::status::*; use crate::util::cache::VideoCache; use crate::util::discord::{format_error_chain, send_discord_error_report}; use crate::videos::ServerOptions; @@ -41,6 +43,140 @@ impl MissavProvider { } } + fn build_channel(&self, _clientversion: ClientVersion) -> Channel { + Channel { + id: "missav".to_string(), + name: "MissAV".to_string(), + description: "Watch HD JAV Online".to_string(), + premium: false, + favicon: "https://www.google.com/s2/favicons?sz=64&domain=missav.ws".to_string(), + status: "active".to_string(), + categories: vec![], + options: vec![ + ChannelOption { + id: "sort".to_string(), + title: "Sort".to_string(), + description: "Sort the Videos".to_string(), + systemImage: "list.number".to_string(), + colorName: "blue".to_string(), + options: vec![ + FilterOption { + id: "released_at".to_string(), + title: "Release Date".to_string(), + }, + FilterOption { + id: "published_at".to_string(), + title: "Recent Update".to_string(), + }, + FilterOption { + id: "today_views".to_string(), + title: "Today Views".to_string(), + }, + FilterOption { + id: "weekly_views".to_string(), + title: "Weekly Views".to_string(), + }, + FilterOption { + id: "monthly_views".to_string(), + title: "Monthly Views".to_string(), + }, + FilterOption { + id: "views".to_string(), + title: "Total Views".to_string(), + }, + ], + multiSelect: false, + }, + ChannelOption { + id: "filter".to_string(), + title: "Filter".to_string(), + description: "Filter the Videos".to_string(), + systemImage: "line.horizontal.3.decrease.circle".to_string(), + colorName: "green".to_string(), + options: vec![ + FilterOption { + id: "new".to_string(), + title: "Recent update".to_string(), + }, + FilterOption { + id: "release".to_string(), + title: "New Releases".to_string(), + }, + FilterOption { + id: "uncensored-leak".to_string(), + title: "Uncensored".to_string(), + }, + FilterOption { + id: "english-subtitle".to_string(), + title: "English subtitle".to_string(), + }, + ], + multiSelect: false, + }, + ChannelOption { + id: "language".to_string(), + title: "Language".to_string(), + description: "What Language to fetch".to_string(), + systemImage: "flag.fill".to_string(), + colorName: "gray".to_string(), + options: vec![ + FilterOption { + id: "en".to_string(), + title: "English".to_string(), + }, + FilterOption { + id: "cn".to_string(), + title: "简体中文".to_string(), + }, + FilterOption { + id: "ja".to_string(), + title: "日本語".to_string(), + }, + FilterOption { + id: "ko".to_string(), + title: "한국의".to_string(), + }, + FilterOption { + id: "ms".to_string(), + title: "Melayu".to_string(), + }, + FilterOption { + id: "th".to_string(), + title: "ไทย".to_string(), + }, + FilterOption { + id: "de".to_string(), + title: "Deutsch".to_string(), + }, + FilterOption { + id: "fr".to_string(), + title: "Français".to_string(), + }, + FilterOption { + id: "vi".to_string(), + title: "Tiếng Việt".to_string(), + }, + FilterOption { + id: "id".to_string(), + title: "Bahasa Indonesia".to_string(), + }, + FilterOption { + id: "fil".to_string(), + title: "Filipino".to_string(), + }, + FilterOption { + id: "pt".to_string(), + title: "Português".to_string(), + }, + ], + multiSelect: false, + }, + ], + nsfw: true, + cacheDuration: None, + } + } + async fn get(&self, cache: VideoCache, pool: DbPool, page: u8, mut sort: String, options: ServerOptions) -> Result> { // Use ok_or to avoid unwrapping options let language = options.language.as_ref().ok_or("Missing language")?; @@ -185,9 +321,16 @@ impl MissavProvider { let parts_str = vid.split("m3u8").nth(1)?.split("https").next()?; let mut parts: Vec<&str> = parts_str.split('|').collect(); parts.reverse(); - if parts.len() < 8 { return None; } - Some(format!("https://{}.{}/{}-{}-{}-{}-{}/playlist.m3u8", - parts[1], parts[2], parts[3], parts[4], parts[5], parts[6], parts[7])) + Some(format!( + "https://{}.{}/{}-{}-{}-{}-{}/playlist.m3u8", + parts.get(1)?, + parts.get(2)?, + parts.get(3)?, + parts.get(4)?, + parts.get(5)?, + parts.get(6)?, + parts.get(7)? + )) })().ok_or_else(|| ErrorKind::ParsingError(format!("video_url\n{:?}", vid).to_string()))?; let video_item = VideoItem::new(id, title, video_url, "missav".to_string(), thumb, duration) @@ -218,4 +361,8 @@ impl Provider for MissavProvider { vec![] }) } -} \ No newline at end of file + + fn get_channel(&self, clientversion: ClientVersion) -> Option { + Some(self.build_channel(clientversion)) + } +} diff --git a/src/providers/mod.rs b/src/providers/mod.rs index b1f81d7..f1d3097 100644 --- a/src/providers/mod.rs +++ b/src/providers/mod.rs @@ -1,10 +1,13 @@ use async_trait::async_trait; +use futures::FutureExt; use once_cell::sync::Lazy; use rustc_hash::FxHashMap as HashMap; +use std::future::Future; +use std::panic::AssertUnwindSafe; use std::sync::Arc; use crate::{ - DbPool, api::ClientVersion, status::Channel, util::cache::VideoCache, videos::{ServerOptions, VideoItem} + DbPool, api::ClientVersion, status::Channel, util::{cache::VideoCache, discord::send_discord_error_report, requester::Requester}, videos::{ServerOptions, VideoItem} }; pub mod all; @@ -49,6 +52,33 @@ pub type DynProvider = Arc; pub static ALL_PROVIDERS: Lazy> = Lazy::new(|| { let mut m = HashMap::default(); + m.insert("all", Arc::new(all::AllProvider::new()) as DynProvider); + m.insert("perverzija", Arc::new(perverzija::PerverzijaProvider::new()) as DynProvider); + m.insert("hanime", Arc::new(hanime::HanimeProvider::new()) as DynProvider); + m.insert("pornhub", Arc::new(pornhub::PornhubProvider::new()) as DynProvider); + m.insert( + "rule34video", + Arc::new(rule34video::Rule34videoProvider::new()) as DynProvider, + ); + m.insert("redtube", Arc::new(redtube::RedtubeProvider::new()) as DynProvider); + m.insert("okporn", Arc::new(okporn::OkpornProvider::new()) as DynProvider); + m.insert("pornhat", Arc::new(pornhat::PornhatProvider::new()) as DynProvider); + m.insert( + "perfectgirls", + Arc::new(perfectgirls::PerfectgirlsProvider::new()) as DynProvider, + ); + m.insert("okxxx", Arc::new(okxxx::OkxxxProvider::new()) as DynProvider); + m.insert("homoxxx", Arc::new(homoxxx::HomoxxxProvider::new()) as DynProvider); + m.insert("missav", Arc::new(missav::MissavProvider::new()) as DynProvider); + m.insert("xxthots", Arc::new(xxthots::XxthotsProvider::new()) as DynProvider); + m.insert("sxyprn", Arc::new(sxyprn::SxyprnProvider::new()) as DynProvider); + m.insert("porn00", Arc::new(porn00::Porn00Provider::new()) as DynProvider); + m.insert("youjizz", Arc::new(youjizz::YoujizzProvider::new()) as DynProvider); + m.insert( + "paradisehill", + Arc::new(paradisehill::ParadisehillProvider::new()) as DynProvider, + ); + m.insert("pornzog", Arc::new(pornzog::PornzogProvider::new()) as DynProvider); m.insert("omgxxx", Arc::new(omgxxx::OmgxxxProvider::new()) as DynProvider); m.insert("beeg", Arc::new(beeg::BeegProvider::new()) as DynProvider); m.insert("tnaflix", Arc::new(tnaflix::TnaflixProvider::new()) as DynProvider); @@ -74,6 +104,75 @@ pub fn init_providers_now() { Lazy::force(&ALL_PROVIDERS); } +pub fn panic_payload_to_string(payload: Box) -> String { + if let Some(s) = payload.downcast_ref::<&str>() { + return (*s).to_string(); + } + if let Some(s) = payload.downcast_ref::() { + return s.clone(); + } + "unknown panic payload".to_string() +} + +pub async fn run_provider_guarded(provider_name: &str, context: &str, fut: F) -> Vec +where + F: Future>, +{ + match AssertUnwindSafe(fut).catch_unwind().await { + Ok(videos) => videos, + Err(payload) => { + let panic_msg = panic_payload_to_string(payload); + let _ = send_discord_error_report( + format!("Provider panic: {}", provider_name), + None, + Some("Provider Guard"), + Some(&format!("context={}; panic={}", context, panic_msg)), + file!(), + line!(), + module_path!(), + ) + .await; + vec![] + } + } +} + +pub async fn report_provider_error(provider_name: &str, context: &str, msg: &str) { + let _ = send_discord_error_report( + format!("Provider error: {}", provider_name), + None, + Some("Provider Guard"), + Some(&format!("context={}; error={}", context, msg)), + file!(), + line!(), + module_path!(), + ) + .await; +} + +pub fn report_provider_error_background(provider_name: &str, context: &str, msg: &str) { + let provider_name = provider_name.to_string(); + let context = context.to_string(); + let msg = msg.to_string(); + tokio::spawn(async move { + report_provider_error(&provider_name, &context, &msg).await; + }); +} + +pub fn requester_or_default(options: &ServerOptions, provider_name: &str, context: &str) -> Requester { + match options.requester.clone() { + Some(requester) => requester, + None => { + report_provider_error_background( + provider_name, + context, + "ServerOptions.requester missing; using default Requester", + ); + Requester::new() + } + } +} + #[async_trait] pub trait Provider: Send + Sync { diff --git a/src/providers/okporn.rs b/src/providers/okporn.rs index 8f2ee95..c050eb4 100644 --- a/src/providers/okporn.rs +++ b/src/providers/okporn.rs @@ -1,5 +1,7 @@ +use crate::api::ClientVersion; use crate::DbPool; -use crate::providers::Provider; +use crate::providers::{Provider, report_provider_error}; +use crate::status::*; use crate::util::cache::VideoCache; use crate::util::flaresolverr::{FlareSolverrRequest, Flaresolverr}; use crate::util::time::parse_time_to_seconds; @@ -29,6 +31,42 @@ impl OkpornProvider { url: "https://ok.porn".to_string(), } } + + fn build_channel(&self, _clientversion: ClientVersion) -> Channel { + Channel { + id: "okporn".to_string(), + name: "Ok.porn".to_string(), + description: "Tons of HD porno movies".to_string(), + premium: false, + favicon: "https://www.google.com/s2/favicons?sz=64&domain=ok.porn".to_string(), + status: "active".to_string(), + categories: vec![], + options: vec![ChannelOption { + id: "sort".to_string(), + title: "Sort".to_string(), + description: "Sort the Videos".to_string(), + systemImage: "list.number".to_string(), + colorName: "blue".to_string(), + options: vec![ + FilterOption { + id: "new".to_string(), + title: "New".to_string(), + }, + FilterOption { + id: "popular".to_string(), + title: "Popular".to_string(), + }, + FilterOption { + id: "trending".to_string(), + title: "Trending".to_string(), + }, + ], + multiSelect: false, + }], + nsfw: true, + cacheDuration: Some(1800), + } + } async fn get( &self, cache: VideoCache, @@ -60,11 +98,24 @@ impl OkpornProvider { let mut response = client.get(video_url.clone()) // .proxy(proxy.clone()) .send().await?; - if response.status().is_redirection(){ - - response = client.get(self.url.clone() + response.headers()["Location"].to_str().unwrap()) - // .proxy(proxy.clone()) - .send().await?; + if response.status().is_redirection() { + let location = match response.headers().get("Location").and_then(|h| h.to_str().ok()) { + Some(location) => location, + None => { + report_provider_error( + "okporn", + "get.redirect_location", + &format!("url={video_url}; missing/invalid Location header"), + ) + .await; + return Ok(old_items); + } + }; + response = client + .get(self.url.clone() + location) + // .proxy(proxy.clone()) + .send() + .await?; } if response.status().is_success() { let text = response.text().await?; @@ -77,7 +128,18 @@ impl OkpornProvider { } Ok(video_items) } else { - let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set"); + let flare_url = match env::var("FLARE_URL") { + Ok(url) => url, + Err(e) => { + report_provider_error( + "okporn", + "get.flare_url", + &e.to_string(), + ) + .await; + return Ok(old_items); + } + }; let flare = Flaresolverr::new(flare_url); let result = flare .solve(FlareSolverrRequest { @@ -136,11 +198,24 @@ impl OkpornProvider { // .proxy(proxy.clone()) .send().await?; - if response.status().is_redirection(){ - - response = client.get(self.url.clone() + response.headers()["Location"].to_str().unwrap()) - // .proxy(proxy.clone()) - .send().await?; + if response.status().is_redirection() { + let location = match response.headers().get("Location").and_then(|h| h.to_str().ok()) { + Some(location) => location, + None => { + report_provider_error( + "okporn", + "query.redirect_location", + &format!("url={video_url}; missing/invalid Location header"), + ) + .await; + return Ok(old_items); + } + }; + response = client + .get(self.url.clone() + location) + // .proxy(proxy.clone()) + .send() + .await?; } if response.status().is_success() { @@ -154,7 +229,18 @@ impl OkpornProvider { } Ok(video_items) } else { - let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set"); + let flare_url = match env::var("FLARE_URL") { + Ok(url) => url, + Err(e) => { + report_provider_error( + "okporn", + "query.flare_url", + &e.to_string(), + ) + .await; + return Ok(old_items); + } + }; let flare = Flaresolverr::new(flare_url); let result = flare .solve(FlareSolverrRequest { @@ -195,25 +281,25 @@ impl OkpornProvider { // for (index, line) in vid.iter().enumerate() { // println!("Line {}: {}", index, line); // } - let video_url: String = format!("{}{}", self.url, video_segment.split(">()[1] + let video_url: String = format!("{}{}", self.url, video_segment.split(">().get(1).copied().unwrap_or_default() .split("\"") - .collect::>()[0]); - let mut title = video_segment.split("\" title=\"").collect::>()[1] + .collect::>().get(0).copied().unwrap_or_default()); + let mut title = video_segment.split("\" title=\"").collect::>().get(1).copied().unwrap_or_default() .split("\"") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .to_string(); // html decode title = decode(title.as_bytes()).to_string().unwrap_or(title); - let id = video_url.split("/").collect::>()[4].to_string(); - let raw_duration = video_segment.split("").collect::>()[1] + let id = video_url.split("/").collect::>().get(4).copied().unwrap_or_default().to_string(); + let raw_duration = video_segment.split("").collect::>().get(1).copied().unwrap_or_default() .split("<") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .to_string(); let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32; - let thumb = video_segment.split(">()[1].split("data-original=\"").collect::>()[1] + let thumb = video_segment.split(">().get(1).copied().unwrap_or_default().split("data-original=\"").collect::>().get(1).copied().unwrap_or_default() .split("\"") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .to_string(); let video_item = VideoItem::new( @@ -266,4 +352,8 @@ impl Provider for OkpornProvider { } } } + + fn get_channel(&self, clientversion: ClientVersion) -> Option { + Some(self.build_channel(clientversion)) + } } diff --git a/src/providers/okxxx.rs b/src/providers/okxxx.rs index 1dc09c1..fe8d4c3 100644 --- a/src/providers/okxxx.rs +++ b/src/providers/okxxx.rs @@ -1,6 +1,8 @@ +use crate::api::ClientVersion; use crate::util::parse_abbreviated_number; use crate::DbPool; -use crate::providers::Provider; +use crate::providers::{Provider, report_provider_error}; +use crate::status::*; use crate::util::cache::VideoCache; use crate::util::flaresolverr::{FlareSolverrRequest, Flaresolverr}; use crate::util::time::parse_time_to_seconds; @@ -30,6 +32,42 @@ impl OkxxxProvider { url: "https://ok.xxx".to_string(), } } + + fn build_channel(&self, _clientversion: ClientVersion) -> Channel { + Channel { + id: "okxxx".to_string(), + name: "Ok.xxx".to_string(), + description: "free porn tube!".to_string(), + premium: false, + favicon: "https://www.google.com/s2/favicons?sz=64&domain=ok.xxx".to_string(), + status: "active".to_string(), + categories: vec![], + options: vec![ChannelOption { + id: "sort".to_string(), + title: "Sort".to_string(), + description: "Sort the Videos".to_string(), + systemImage: "list.number".to_string(), + colorName: "blue".to_string(), + options: vec![ + FilterOption { + id: "new".to_string(), + title: "New".to_string(), + }, + FilterOption { + id: "popular".to_string(), + title: "Popular".to_string(), + }, + FilterOption { + id: "trending".to_string(), + title: "Trending".to_string(), + }, + ], + multiSelect: false, + }], + nsfw: true, + cacheDuration: Some(1800), + } + } async fn get( &self, cache: VideoCache, @@ -61,11 +99,25 @@ impl OkxxxProvider { let mut response = client.get(video_url.clone()) // .proxy(proxy.clone()) .send().await?; - if response.status().is_redirection(){ - println!("Redirection detected, following to: {}", response.headers()["Location"].to_str().unwrap()); - response = client.get(response.headers()["Location"].to_str().unwrap()) - // .proxy(proxy.clone()) - .send().await?; + if response.status().is_redirection() { + let location = match response.headers().get("Location").and_then(|h| h.to_str().ok()) { + Some(location) => location, + None => { + report_provider_error( + "okxxx", + "get.redirect_location", + &format!("url={video_url}; missing/invalid Location header"), + ) + .await; + return Ok(old_items); + } + }; + println!("Redirection detected, following to: {}", location); + response = client + .get(location) + // .proxy(proxy.clone()) + .send() + .await?; } if response.status().is_success() { let text = response.text().await?; @@ -78,7 +130,18 @@ impl OkxxxProvider { } Ok(video_items) } else { - let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set"); + let flare_url = match env::var("FLARE_URL") { + Ok(url) => url, + Err(e) => { + report_provider_error( + "okxxx", + "get.flare_url", + &e.to_string(), + ) + .await; + return Ok(old_items); + } + }; let flare = Flaresolverr::new(flare_url); let result = flare .solve(FlareSolverrRequest { @@ -116,7 +179,7 @@ impl OkxxxProvider { let mut video_url = format!("{}/search/{}/{}/", self.url, search_string, page); if search_string.starts_with("@"){ - let url_part = search_string.split("@").collect::>()[1].replace(":", "/"); + let url_part = search_string.split("@").collect::>().get(1).copied().unwrap_or_default().replace(":", "/"); video_url = format!("{}/{}/", self.url, url_part); } // Check our Video Cache. If the result is younger than 1 hour, we return it. @@ -141,11 +204,24 @@ impl OkxxxProvider { // .proxy(proxy.clone()) .send().await?; - if response.status().is_redirection(){ - - response = client.get(self.url.clone() + response.headers()["Location"].to_str().unwrap()) - // .proxy(proxy.clone()) - .send().await?; + if response.status().is_redirection() { + let location = match response.headers().get("Location").and_then(|h| h.to_str().ok()) { + Some(location) => location, + None => { + report_provider_error( + "okxxx", + "query.redirect_location", + &format!("url={video_url}; missing/invalid Location header"), + ) + .await; + return Ok(old_items); + } + }; + response = client + .get(self.url.clone() + location) + // .proxy(proxy.clone()) + .send() + .await?; } if response.status().is_success() { @@ -159,7 +235,18 @@ impl OkxxxProvider { } Ok(video_items) } else { - let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set"); + let flare_url = match env::var("FLARE_URL") { + Ok(url) => url, + Err(e) => { + report_provider_error( + "okxxx", + "query.flare_url", + &e.to_string(), + ) + .await; + return Ok(old_items); + } + }; let flare = Flaresolverr::new(flare_url); let result = flare .solve(FlareSolverrRequest { @@ -191,7 +278,7 @@ impl OkxxxProvider { return vec![]; } let mut items: Vec = Vec::new(); - let raw_videos = html.split("
    >()[0] + let raw_videos = html.split("
    >().get(0).copied().unwrap_or_default() .split("item thumb-bl thumb-bl-video video_") .collect::>()[1..] .to_vec(); @@ -200,38 +287,38 @@ impl OkxxxProvider { // for (index, line) in vid.iter().enumerate() { // println!("Line {}: {}", index, line); // } - let video_url: String = format!("{}{}", self.url, video_segment.split(">()[1] + let video_url: String = format!("{}{}", self.url, video_segment.split(">().get(1).copied().unwrap_or_default() .split("\"") - .collect::>()[0]); - let preview_url = video_segment.split("data-preview-custom=\"").collect::>()[1] + .collect::>().get(0).copied().unwrap_or_default()); + let preview_url = video_segment.split("data-preview-custom=\"").collect::>().get(1).copied().unwrap_or_default() .split("\"") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .to_string(); - let mut title = video_segment.split("\" title=\"").collect::>()[1] + let mut title = video_segment.split("\" title=\"").collect::>().get(1).copied().unwrap_or_default() .split("\"") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .to_string(); // html decode title = decode(title.as_bytes()).to_string().unwrap_or(title); - let id = video_url.split("/").collect::>()[4].to_string(); - let raw_duration = video_segment.split("fa fa-clock-o").collect::>()[1] - .split("").collect::>()[1] + let id = video_url.split("/").collect::>().get(4).copied().unwrap_or_default().to_string(); + let raw_duration = video_segment.split("fa fa-clock-o").collect::>().get(1).copied().unwrap_or_default() + .split("").collect::>().get(1).copied().unwrap_or_default() .split("<") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .to_string(); let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32; - let thumb = format!("https:{}", video_segment.split(" class=\"thumb lazy-load\"").collect::>()[1] - .split("data-original=\"").collect::>()[1] + let thumb = format!("https:{}", video_segment.split(" class=\"thumb lazy-load\"").collect::>().get(1).copied().unwrap_or_default() + .split("data-original=\"").collect::>().get(1).copied().unwrap_or_default() .split("\"") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .to_string()); let mut tags = vec![]; if video_segment.contains("href=\"/sites/"){ let raw_tags = video_segment.split("href=\"/sites/").collect::>()[1..] .iter() - .map(|s| s.split("/\"").collect::>()[0].to_string()) + .map(|s| s.split("/\"").collect::>().get(0).copied().unwrap_or_default().to_string()) .collect::>(); for tag in raw_tags { if !tag.is_empty() { @@ -242,7 +329,7 @@ impl OkxxxProvider { if video_segment.contains("href=\"/models/"){ let raw_tags = video_segment.split("href=\"/models/").collect::>()[1..] .iter() - .map(|s| s.split("/\"").collect::>()[0].to_string()) + .map(|s| s.split("/\"").collect::>().get(0).copied().unwrap_or_default().to_string()) .collect::>(); for tag in raw_tags { if !tag.is_empty() { @@ -251,10 +338,10 @@ impl OkxxxProvider { } } - let views_part = video_segment.split("fa fa-eye").collect::>()[1] - .split("").collect::>()[1] + let views_part = video_segment.split("fa fa-eye").collect::>().get(1).copied().unwrap_or_default() + .split("").collect::>().get(1).copied().unwrap_or_default() .split("<") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .to_string(); let views = parse_abbreviated_number(&views_part).unwrap_or(0) as u32; @@ -311,4 +398,8 @@ impl Provider for OkxxxProvider { } } } + + fn get_channel(&self, clientversion: ClientVersion) -> Option { + Some(self.build_channel(clientversion)) + } } diff --git a/src/providers/omgxxx.rs b/src/providers/omgxxx.rs index 30a54f0..09b2de6 100644 --- a/src/providers/omgxxx.rs +++ b/src/providers/omgxxx.rs @@ -1,6 +1,6 @@ use crate::DbPool; use crate::api::ClientVersion; -use crate::providers::Provider; +use crate::providers::{Provider, report_provider_error, report_provider_error_background}; use crate::util::cache::VideoCache; use crate::util::parse_abbreviated_number; use crate::util::time::parse_time_to_seconds; @@ -58,10 +58,20 @@ impl OmgxxxProvider { thread::spawn(move || { // Create a tiny runtime just for these async tasks - let rt = tokio::runtime::Builder::new_current_thread() + let rt = match tokio::runtime::Builder::new_current_thread() .enable_all() .build() - .expect("build tokio runtime"); + { + Ok(rt) => rt, + Err(e) => { + report_provider_error_background( + "omgxxx", + "spawn_initial_load.runtime_build", + &e.to_string(), + ); + return; + } + }; rt.block_on(async move { // If you have a streaming sites loader, call it here too @@ -83,13 +93,23 @@ impl OmgxxxProvider { async fn load_stars(base_url: &str, stars: Arc>>) -> Result<()> { let mut requester = util::requester::Requester::new(); for page in [1..10].into_iter().flatten() { - let text = requester + let text = match requester .get( format!("{}/models/total-videos/{}/?gender_id=0", &base_url, page).as_str(), None, ) .await - .unwrap(); + { + Ok(text) => text, + Err(e) => { + report_provider_error_background( + "omgxxx", + "load_stars.request", + &format!("url={base_url}; page={page}; error={e}"), + ); + break; + } + }; if text.contains("404 Not Found") || text.is_empty() { break; } @@ -97,19 +117,20 @@ impl OmgxxxProvider { .split("
    ") .collect::>() .last() - .unwrap() + .copied() + .unwrap_or_default() .split("custom_list_models_models_list_pagination") - .collect::>()[0]; + .collect::>().get(0).copied().unwrap_or_default(); for stars_element in stars_div.split(">()[1..].to_vec() { - let star_url = stars_element.split("href=\"").collect::>()[1] + let star_url = stars_element.split("href=\"").collect::>().get(1).copied().unwrap_or_default() .split("\"") - .collect::>()[0]; - let star_id = star_url.split("/").collect::>()[4].to_string(); + .collect::>().get(0).copied().unwrap_or_default(); + let star_id = star_url.split("/").collect::>().get(4).copied().unwrap_or_default().to_string(); let star_name = stars_element .split("") - .collect::>()[1] + .collect::>().get(1).copied().unwrap_or_default() .split("<") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .to_string(); Self::push_unique( &stars, @@ -130,26 +151,36 @@ impl OmgxxxProvider { page += 1; let text = requester .get(format!("{}/sites/{}/", &base_url, page).as_str(), None) - .await - .unwrap(); + .await; + let text = match text { + Ok(text) => text, + Err(e) => { + report_provider_error_background( + "omgxxx", + "load_sites.request", + &format!("url={base_url}; page={page}; error={e}"), + ); + break; + } + }; if text.contains("404 Not Found") || text.is_empty() { break; } let sites_div = text .split("id=\"list_content_sources_sponsors_list_items\"") - .collect::>()[1] + .collect::>().get(1).copied().unwrap_or_default() .split("class=\"pagination\"") - .collect::>()[0]; + .collect::>().get(0).copied().unwrap_or_default(); for sites_element in sites_div.split("class=\"headline\"").collect::>()[1..].to_vec() { - let site_url = sites_element.split("href=\"").collect::>()[1] + let site_url = sites_element.split("href=\"").collect::>().get(1).copied().unwrap_or_default() .split("\"") - .collect::>()[0]; - let site_id = site_url.split("/").collect::>()[4].to_string(); - let site_name = sites_element.split("

    ").collect::>()[1] + .collect::>().get(0).copied().unwrap_or_default(); + let site_id = site_url.split("/").collect::>().get(4).copied().unwrap_or_default().to_string(); + let site_name = sites_element.split("

    ").collect::>().get(1).copied().unwrap_or_default() .split("<") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .to_string(); Self::push_unique( &sites, @@ -165,23 +196,33 @@ impl OmgxxxProvider { async fn load_networks(base_url: &str, networks: Arc>>) -> Result<()> { let mut requester = util::requester::Requester::new(); - let text = requester.get(&base_url, None).await.unwrap(); - let networks_div = text.split("class=\"sites__list\"").collect::>()[1] + let text = match requester.get(&base_url, None).await { + Ok(text) => text, + Err(e) => { + report_provider_error_background( + "omgxxx", + "load_networks.request", + &format!("url={base_url}; error={e}"), + ); + return Ok(()); + } + }; + let networks_div = text.split("class=\"sites__list\"").collect::>().get(1).copied().unwrap_or_default() .split("

    ") - .collect::>()[0]; + .collect::>().get(0).copied().unwrap_or_default(); for network_element in networks_div.split("sites__item").collect::>()[1..].to_vec() { if network_element.contains("sites__all") { continue; } - let network_url = network_element.split("href=\"").collect::>()[1] + let network_url = network_element.split("href=\"").collect::>().get(1).copied().unwrap_or_default() .split("\"") - .collect::>()[0]; - let network_id = network_url.split("/").collect::>()[4].to_string(); - let network_name = network_element.split(">").collect::>()[1] + .collect::>().get(0).copied().unwrap_or_default(); + let network_id = network_url.split("/").collect::>().get(4).copied().unwrap_or_default().to_string(); + let network_name = network_element.split(">").collect::>().get(1).copied().unwrap_or_default() .split("<") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .to_string(); Self::push_unique( &networks, @@ -306,35 +347,20 @@ impl OmgxxxProvider { "most-popular" => "/most-popular".to_string(), _ => "".to_string(), }; - if options.network.is_some() - && !options.network.as_ref().unwrap().is_empty() - && options.network.as_ref().unwrap() != "all" - { - sort_string = format!( - "networks/{}{}", - options.network.as_ref().unwrap(), - alt_sort_string - ); + if let Some(network) = options.network.as_deref() { + if !network.is_empty() && network != "all" { + sort_string = format!("networks/{}{}", network, alt_sort_string); + } } - if options.sites.is_some() - && !options.sites.as_ref().unwrap().is_empty() - && options.sites.as_ref().unwrap() != "all" - { - sort_string = format!( - "sites/{}{}", - options.sites.as_ref().unwrap(), - alt_sort_string - ); + if let Some(site) = options.sites.as_deref() { + if !site.is_empty() && site != "all" { + sort_string = format!("sites/{}{}", site, alt_sort_string); + } } - if options.stars.is_some() - && !options.stars.as_ref().unwrap().is_empty() - && options.stars.as_ref().unwrap() != "all" - { - sort_string = format!( - "models/{}{}", - options.stars.as_ref().unwrap(), - alt_sort_string - ); + if let Some(star) = options.stars.as_deref() { + if !star.is_empty() && star != "all" { + sort_string = format!("models/{}{}", star, alt_sort_string); + } } let video_url = format!("{}/{}/{}/", self.url, sort_string, page); let old_items = match cache.get(&video_url) { @@ -350,8 +376,20 @@ impl OmgxxxProvider { } }; - let mut requester = options.requester.clone().unwrap(); - let text = requester.get(&video_url, None).await.unwrap(); + let mut requester = + crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); + let text = match requester.get(&video_url, None).await { + Ok(text) => text, + Err(e) => { + report_provider_error( + "omgxxx", + "get.request", + &format!("url={video_url}; error={e}"), + ) + .await; + return Ok(old_items); + } + }; let video_items: Vec = self.get_video_items_from_html(text.clone()); if !video_items.is_empty() { cache.remove(&video_url); @@ -371,31 +409,41 @@ impl OmgxxxProvider { ) -> Result> { let mut search_type = "search"; let mut search_string = query.to_string().to_ascii_lowercase().trim().to_string(); - match self - .stars - .read() - .unwrap() - .iter() - .find(|s| s.title.to_ascii_lowercase() == search_string) - { - Some(star) => { - search_type = "models"; - search_string = star.id.clone(); + match self.stars.read() { + Ok(stars) => { + if let Some(star) = stars + .iter() + .find(|s| s.title.to_ascii_lowercase() == search_string) + { + search_type = "models"; + search_string = star.id.clone(); + } + } + Err(e) => { + report_provider_error_background( + "omgxxx", + "query.stars_read", + &e.to_string(), + ); } - _ => {} } - match self - .sites - .read() - .unwrap() - .iter() - .find(|s| s.title.to_ascii_lowercase() == search_string) - { - Some(site) => { - search_type = "sites"; - search_string = site.id.clone(); + match self.sites.read() { + Ok(sites) => { + if let Some(site) = sites + .iter() + .find(|s| s.title.to_ascii_lowercase() == search_string) + { + search_type = "sites"; + search_string = site.id.clone(); + } + } + Err(e) => { + report_provider_error_background( + "omgxxx", + "query.sites_read", + &e.to_string(), + ); } - _ => {} } let mut video_url = format!("{}/{}/{}/{}/", self.url, search_type, search_string, page); video_url = video_url.replace(" ", "+"); @@ -414,9 +462,20 @@ impl OmgxxxProvider { } }; - let mut requester = options.requester.clone().unwrap(); - - let text = requester.get(&video_url, None).await.unwrap(); + let mut requester = + crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); + let text = match requester.get(&video_url, None).await { + Ok(text) => text, + Err(e) => { + report_provider_error( + "omgxxx", + "query.request", + &format!("url={video_url}; error={e}"), + ) + .await; + return Ok(old_items); + } + }; let video_items: Vec = self.get_video_items_from_html(text.clone()); if !video_items.is_empty() { cache.remove(&video_url); @@ -429,7 +488,18 @@ impl OmgxxxProvider { fn get_site_id_from_name(&self, site_name: &str) -> Option { // site_name.to_lowercase().replace(" ", "") - for site in self.sites.read().unwrap().iter() { + let sites_guard = match self.sites.read() { + Ok(guard) => guard, + Err(e) => { + report_provider_error_background( + "omgxxx", + "get_site_id_from_name.sites_read", + &e.to_string(), + ); + return None; + } + }; + for site in sites_guard.iter() { if site .title .to_lowercase() @@ -452,11 +522,11 @@ impl OmgxxxProvider { if !html.contains("class=\"item\"") { return items; } - let raw_videos = html.split("videos_list_pagination").collect::>()[0] + let raw_videos = html.split("videos_list_pagination").collect::>().get(0).copied().unwrap_or_default() .split(" class=\"pagination\" ") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .split("class=\"list-videos\"") - .collect::>()[1] + .collect::>().get(1).copied().unwrap_or_default() .split("class=\"item\"") .collect::>()[1..] .to_vec(); @@ -465,39 +535,39 @@ impl OmgxxxProvider { // for (index, line) in vid.iter().enumerate() { // println!("Line {}: {}", index, line); // } - let video_url: String = video_segment.split(">()[1] + let video_url: String = video_segment.split(">().get(1).copied().unwrap_or_default() .split("\"") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .to_string(); - let mut title = video_segment.split(" title=\"").collect::>()[1] + let mut title = video_segment.split(" title=\"").collect::>().get(1).copied().unwrap_or_default() .split("\"") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .to_string(); // html decode title = decode(title.as_bytes()).to_string().unwrap_or(title); - let id = video_url.split("/").collect::>()[4].to_string(); + let id = video_url.split("/").collect::>().get(4).copied().unwrap_or_default().to_string(); - let thumb = match video_segment.split("img loading").collect::>()[1] + let thumb = match video_segment.split("img loading").collect::>().get(1).copied().unwrap_or_default() .contains("data-src=\"") { - true => video_segment.split("img loading").collect::>()[1] + true => video_segment.split("img loading").collect::>().get(1).copied().unwrap_or_default() .split("data-src=\"") - .collect::>()[1] + .collect::>().get(1).copied().unwrap_or_default() .split("\"") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .to_string(), - false => video_segment.split("img loading").collect::>()[1] + false => video_segment.split("img loading").collect::>().get(1).copied().unwrap_or_default() .split("data-original=\"") - .collect::>()[1] + .collect::>().get(1).copied().unwrap_or_default() .split("\"") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .to_string(), }; let raw_duration = video_segment .split("") - .collect::>()[1] + .collect::>().get(1).copied().unwrap_or_default() .split("<") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .split(" ") .collect::>() .last() @@ -507,9 +577,9 @@ impl OmgxxxProvider { let views = parse_abbreviated_number( video_segment .split("
    ") - .collect::>()[1] + .collect::>().get(1).copied().unwrap_or_default() .split("<") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .to_string() .as_str(), ) @@ -517,9 +587,9 @@ impl OmgxxxProvider { let preview = video_segment .split("data-preview=\"") - .collect::>()[1] + .collect::>().get(1).copied().unwrap_or_default() .split("\"") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .to_string(); let site_name = title .split("]") @@ -533,9 +603,9 @@ impl OmgxxxProvider { let mut tags = match video_segment.contains("class=\"models\">") { true => video_segment .split("class=\"models\">") - .collect::>()[1] + .collect::>().get(1).copied().unwrap_or_default() .split("
    ") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .split("href=\"") .collect::>()[1..] .into_iter() @@ -543,17 +613,17 @@ impl OmgxxxProvider { Self::push_unique( &self.stars, FilterOption { - id: s.split("/").collect::>()[4].to_string(), - title: s.split(">").collect::>()[1] + id: s.split("/").collect::>().get(4).copied().unwrap_or_default().to_string(), + title: s.split(">").collect::>().get(1).copied().unwrap_or_default() .split("<") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .trim() .to_string(), }, ); - s.split(">").collect::>()[1] + s.split(">").collect::>().get(1).copied().unwrap_or_default() .split("<") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .trim() .to_string() }) diff --git a/src/providers/paradisehill.rs b/src/providers/paradisehill.rs index 2651f0d..8d27ac7 100644 --- a/src/providers/paradisehill.rs +++ b/src/providers/paradisehill.rs @@ -1,5 +1,7 @@ +use crate::api::ClientVersion; use crate::DbPool; use crate::providers::Provider; +use crate::status::*; use crate::util::cache::VideoCache; use crate::util::requester::Requester; use crate::videos::VideoItem; @@ -28,13 +30,28 @@ impl ParadisehillProvider { url: "https://en.paradisehill.cc".to_string(), } } + + fn build_channel(&self, _clientversion: ClientVersion) -> Channel { + Channel { + id: "paradisehill".to_string(), + name: "Paradisehill".to_string(), + description: "Porn Movies on Paradise Hill".to_string(), + premium: false, + favicon: "https://www.google.com/s2/favicons?sz=64&domain=en.paradisehill.cc".to_string(), + status: "active".to_string(), + categories: vec![], + options: vec![], + nsfw: true, + cacheDuration: None, + } + } async fn get( &self, cache: VideoCache, page: u8, options: ServerOptions, ) -> Result> { - let mut requester = options.requester.clone().unwrap(); + let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); let url_str = format!("{}/all/?sort=created_at&page={}", self.url, page); @@ -51,7 +68,18 @@ impl ParadisehillProvider { } }; - let text = requester.get(&url_str, None).await.unwrap(); + let text = match requester.get(&url_str, None).await { + Ok(text) => text, + Err(e) => { + crate::providers::report_provider_error( + "paradisehill", + "get.request", + &format!("url={url_str}; error={e}"), + ) + .await; + return Ok(old_items); + } + }; // Pass a reference to options if needed, or reconstruct as needed let video_items: Vec = self .get_video_items_from_html(text.clone(), requester) @@ -73,7 +101,7 @@ impl ParadisehillProvider { options: ServerOptions, ) -> Result> { // Extract needed fields from options at the start - let mut requester = options.requester.clone().unwrap(); + let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); let search_string = query.replace(" ", "+"); let url_str = format!( "{}/search/?pattern={}&page={}", @@ -93,7 +121,18 @@ impl ParadisehillProvider { vec![] } }; - let text = requester.get(&url_str, None).await.unwrap(); + let text = match requester.get(&url_str, None).await { + Ok(text) => text, + Err(e) => { + crate::providers::report_provider_error( + "paradisehill", + "query.request", + &format!("url={url_str}; error={e}"), + ) + .await; + return Ok(old_items); + } + }; let video_items: Vec = self .get_video_items_from_html(text.clone(), requester) .await; @@ -109,46 +148,90 @@ impl ParadisehillProvider { async fn get_video_items_from_html( &self, html: String, - requester: Requester, + _requester: Requester, ) -> Vec { if html.is_empty() { println!("HTML is empty"); return vec![]; } - let raw_videos = html.split("item list-film-item").collect::>()[1..].to_vec(); - let mut urls: Vec = vec![]; - for video_segment in &raw_videos { - // let vid = video_segment.split("\n").collect::>(); - // for (index, line) in vid.iter().enumerate() { - // println!("Line {}: {}", index, line.to_string().trim()); - // } + let mut items: Vec = Vec::new(); + for video_segment in html.split("item list-film-item").skip(1) { + let href = video_segment + .split("    >()[1] - .split("\"") - .collect::>()[0] - .to_string() + let video_url = format!("{}{}", self.url, href); + let id = href + .trim_matches('/') + .split('/') + .next() + .unwrap_or_default() + .to_string(); + if id.is_empty() { + continue; + } + + let mut title = video_segment + .split("itemprop=\"name\">") + .nth(1) + .and_then(|s| s.split('<').next()) + .unwrap_or_default() + .trim() + .to_string(); + title = decode(title.as_bytes()).to_string().unwrap_or(title); + + let mut thumb = video_segment + .split("itemprop=\"image\" src=\"") + .nth(1) + .and_then(|s| s.split('"').next()) + .unwrap_or_default() + .to_string(); + if thumb.starts_with('/') { + thumb = format!("{}{}", self.url, thumb); + } + + let genre = video_segment + .split("itemprop=\"genre\">") + .nth(1) + .and_then(|s| s.split('<').next()) + .unwrap_or_default() + .trim() + .to_string(); + let tags = if genre.is_empty() { vec![] } else { vec![genre] }; + + items.push( + VideoItem::new(id, title, video_url, "paradisehill".to_string(), thumb, 0) + .aspect_ratio(0.697674419 as f32) + .tags(tags), ); - urls.push(url_str.clone()); } - let futures = urls - .into_iter() - .map(|el| self.get_video_item(el.clone(), requester.clone())); - let results: Vec> = join_all(futures).await; - let video_items: Vec = results.into_iter().filter_map(Result::ok).collect(); - return video_items; + items } async fn get_video_item(&self, url_str: String, mut requester: Requester) -> Result { - let vid = requester.get(&url_str, None).await.unwrap(); + let vid = match requester.get(&url_str, None).await { + Ok(vid) => vid, + Err(e) => { + crate::providers::report_provider_error( + "paradisehill", + "get_video_item.request", + &format!("url={url_str}; error={e}"), + ) + .await; + return Err(Error::from(e.to_string())); + } + }; let mut title = vid .split(">()[1] + .collect::>().get(1).copied().unwrap_or_default() .split("\"") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .trim() .to_string(); title = decode(title.as_bytes()).to_string().unwrap_or(title); @@ -156,27 +239,27 @@ impl ParadisehillProvider { "{}{}", self.url, vid.split(">()[1] + .collect::>().get(1).copied().unwrap_or_default() .split("\"") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .to_string() ); - let video_urls = vid.split("var videoList = ").collect::>()[1] + let video_urls = vid.split("var videoList = ").collect::>().get(1).copied().unwrap_or_default() .split("\"src\":\"") .collect::>()[1..].to_vec(); let mut formats = vec![]; for url in video_urls { let video_url = url .split("\"") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .replace("\\", "") .to_string(); let format = videos::VideoFormat::new(video_url.clone(), "1080".to_string(), "mp4".to_string()) // .protocol("https".to_string()) - .format_id(video_url.split("/").last().unwrap().to_string()) - .format_note(format!("{}", video_url.split("_").last().unwrap().replace(".mp4", "").to_string())) + .format_id(video_url.split("/").last().unwrap_or_default().to_string()) + .format_note(video_url.split("_").last().unwrap_or_default().replace(".mp4", "")) ; formats.push(format); } @@ -184,9 +267,9 @@ impl ParadisehillProvider { formats.reverse(); let id = url_str .split("/") - .collect::>()[3] + .collect::>().get(3).copied().unwrap_or_default() .split("_") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .to_string(); let video_item = VideoItem::new( @@ -237,4 +320,8 @@ impl Provider for ParadisehillProvider { } } } + + fn get_channel(&self, clientversion: ClientVersion) -> Option { + Some(self.build_channel(clientversion)) + } } diff --git a/src/providers/perfectgirls.rs b/src/providers/perfectgirls.rs index 8153bff..3da033d 100644 --- a/src/providers/perfectgirls.rs +++ b/src/providers/perfectgirls.rs @@ -1,6 +1,8 @@ +use crate::api::ClientVersion; use crate::util::parse_abbreviated_number; use crate::DbPool; -use crate::providers::Provider; +use crate::providers::{Provider, report_provider_error}; +use crate::status::*; use crate::util::cache::VideoCache; use crate::util::flaresolverr::{FlareSolverrRequest, Flaresolverr}; use crate::util::time::parse_time_to_seconds; @@ -30,6 +32,42 @@ impl PerfectgirlsProvider { url: "https://www.perfectgirls.xxx".to_string(), } } + + fn build_channel(&self, _clientversion: ClientVersion) -> Channel { + Channel { + id: "perfectgirls".to_string(), + name: "Perfectgirls".to_string(), + description: "Perfect Girls Tube".to_string(), + premium: false, + favicon: "https://www.google.com/s2/favicons?sz=64&domain=perfectgirls.xxx".to_string(), + status: "active".to_string(), + categories: vec![], + options: vec![ChannelOption { + id: "sort".to_string(), + title: "Sort".to_string(), + description: "Sort the Videos".to_string(), + systemImage: "list.number".to_string(), + colorName: "blue".to_string(), + options: vec![ + FilterOption { + id: "new".to_string(), + title: "New".to_string(), + }, + FilterOption { + id: "popular".to_string(), + title: "Popular".to_string(), + }, + FilterOption { + id: "trending".to_string(), + title: "Trending".to_string(), + }, + ], + multiSelect: false, + }], + nsfw: true, + cacheDuration: Some(1800), + } + } async fn get( &self, cache: VideoCache, @@ -61,11 +99,25 @@ impl PerfectgirlsProvider { let mut response = client.get(video_url.clone()) // .proxy(proxy.clone()) .send().await?; - if response.status().is_redirection(){ - println!("Redirection detected, following to: {}", response.headers()["Location"].to_str().unwrap()); - response = client.get(response.headers()["Location"].to_str().unwrap()) - // .proxy(proxy.clone()) - .send().await?; + if response.status().is_redirection() { + let location = match response.headers().get("Location").and_then(|h| h.to_str().ok()) { + Some(location) => location, + None => { + report_provider_error( + "perfectgirls", + "get.redirect_location", + &format!("url={video_url}; missing/invalid Location header"), + ) + .await; + return Ok(old_items); + } + }; + println!("Redirection detected, following to: {}", location); + response = client + .get(location) + // .proxy(proxy.clone()) + .send() + .await?; } if response.status().is_success() { let text = response.text().await?; @@ -78,7 +130,18 @@ impl PerfectgirlsProvider { } Ok(video_items) } else { - let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set"); + let flare_url = match env::var("FLARE_URL") { + Ok(url) => url, + Err(e) => { + report_provider_error( + "perfectgirls", + "get.flare_url", + &e.to_string(), + ) + .await; + return Ok(old_items); + } + }; let flare = Flaresolverr::new(flare_url); let result = flare .solve(FlareSolverrRequest { @@ -116,7 +179,7 @@ impl PerfectgirlsProvider { let mut video_url = format!("{}/search/{}/{}/", self.url, search_string, page); if search_string.starts_with("@"){ - let url_part = search_string.split("@").collect::>()[1].replace(":", "/"); + let url_part = search_string.split("@").collect::>().get(1).copied().unwrap_or_default().replace(":", "/"); video_url = format!("{}/{}/", self.url, url_part); } // Check our Video Cache. If the result is younger than 1 hour, we return it. @@ -141,11 +204,24 @@ impl PerfectgirlsProvider { // .proxy(proxy.clone()) .send().await?; - if response.status().is_redirection(){ - - response = client.get(self.url.clone() + response.headers()["Location"].to_str().unwrap()) - // .proxy(proxy.clone()) - .send().await?; + if response.status().is_redirection() { + let location = match response.headers().get("Location").and_then(|h| h.to_str().ok()) { + Some(location) => location, + None => { + report_provider_error( + "perfectgirls", + "query.redirect_location", + &format!("url={video_url}; missing/invalid Location header"), + ) + .await; + return Ok(old_items); + } + }; + response = client + .get(self.url.clone() + location) + // .proxy(proxy.clone()) + .send() + .await?; } if response.status().is_success() { @@ -159,7 +235,18 @@ impl PerfectgirlsProvider { } Ok(video_items) } else { - let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set"); + let flare_url = match env::var("FLARE_URL") { + Ok(url) => url, + Err(e) => { + report_provider_error( + "perfectgirls", + "query.flare_url", + &e.to_string(), + ) + .await; + return Ok(old_items); + } + }; let flare = Flaresolverr::new(flare_url); let result = flare .solve(FlareSolverrRequest { @@ -191,7 +278,7 @@ impl PerfectgirlsProvider { return vec![]; } let mut items: Vec = Vec::new(); - let raw_videos = html.split("
    >()[0] + let raw_videos = html.split("
    >().get(0).copied().unwrap_or_default() .split("item thumb-bl thumb-bl-video video_") .collect::>()[1..] .to_vec(); @@ -200,31 +287,31 @@ impl PerfectgirlsProvider { // for (index, line) in vid.iter().enumerate() { // println!("Line {}: {}", index, line); // } - let video_url: String = format!("{}{}", self.url, video_segment.split(">()[1] + let video_url: String = format!("{}{}", self.url, video_segment.split(">().get(1).copied().unwrap_or_default() .split("\"") - .collect::>()[0]); - let preview_url = video_segment.split("data-preview-custom=\"").collect::>()[1] + .collect::>().get(0).copied().unwrap_or_default()); + let preview_url = video_segment.split("data-preview-custom=\"").collect::>().get(1).copied().unwrap_or_default() .split("\"") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .to_string(); - let mut title = video_segment.split("\" title=\"").collect::>()[1] + let mut title = video_segment.split("\" title=\"").collect::>().get(1).copied().unwrap_or_default() .split("\"") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .to_string(); // html decode title = decode(title.as_bytes()).to_string().unwrap_or(title); - let id = video_url.split("/").collect::>()[4].to_string(); - let raw_duration = video_segment.split("fa fa-clock-o").collect::>()[1] - .split("").collect::>()[1] + let id = video_url.split("/").collect::>().get(4).copied().unwrap_or_default().to_string(); + let raw_duration = video_segment.split("fa fa-clock-o").collect::>().get(1).copied().unwrap_or_default() + .split("").collect::>().get(1).copied().unwrap_or_default() .split("<") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .to_string(); let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32; - let mut thumb = video_segment.split(" class=\"thumb lazy-load\"").collect::>()[1] - .split("data-original=\"").collect::>()[1] + let mut thumb = video_segment.split(" class=\"thumb lazy-load\"").collect::>().get(1).copied().unwrap_or_default() + .split("data-original=\"").collect::>().get(1).copied().unwrap_or_default() .split("\"") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .to_string(); if thumb.starts_with("//"){ thumb = format!("https:{}",thumb); @@ -234,7 +321,7 @@ impl PerfectgirlsProvider { if video_segment.contains("href=\"/channels/"){ let raw_tags = video_segment.split("href=\"/channels/").collect::>()[1..] .iter() - .map(|s| s.split("/\"").collect::>()[0].to_string()) + .map(|s| s.split("/\"").collect::>().get(0).copied().unwrap_or_default().to_string()) .collect::>(); for tag in raw_tags { if !tag.is_empty() { @@ -245,7 +332,7 @@ impl PerfectgirlsProvider { if video_segment.contains("href=\"/pornstars/"){ let raw_tags = video_segment.split("href=\"/pornstars/").collect::>()[1..] .iter() - .map(|s| s.split("/\"").collect::>()[0].to_string()) + .map(|s| s.split("/\"").collect::>().get(0).copied().unwrap_or_default().to_string()) .collect::>(); for tag in raw_tags { if !tag.is_empty() { @@ -254,10 +341,10 @@ impl PerfectgirlsProvider { } } - let views_part = video_segment.split("fa fa-eye").collect::>()[1] - .split("").collect::>()[1] + let views_part = video_segment.split("fa fa-eye").collect::>().get(1).copied().unwrap_or_default() + .split("").collect::>().get(1).copied().unwrap_or_default() .split("<") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .to_string(); let views = parse_abbreviated_number(&views_part).unwrap_or(0) as u32; @@ -314,4 +401,8 @@ impl Provider for PerfectgirlsProvider { } } } + + fn get_channel(&self, clientversion: ClientVersion) -> Option { + Some(self.build_channel(clientversion)) + } } diff --git a/src/providers/perverzija.rs b/src/providers/perverzija.rs index be4b2b1..f6ca001 100644 --- a/src/providers/perverzija.rs +++ b/src/providers/perverzija.rs @@ -1,6 +1,8 @@ use crate::DbPool; +use crate::api::ClientVersion; use crate::db; -use crate::providers::Provider; +use crate::providers::{Provider, report_provider_error, report_provider_error_background}; +use crate::status::*; use crate::util::cache::VideoCache; use crate::util::time::parse_time_to_seconds; use crate::videos::ServerOptions; @@ -40,6 +42,42 @@ impl PerverzijaProvider { url: "https://tube.perverzija.com/".to_string(), } } + + fn build_channel(&self, clientversion: ClientVersion) -> Channel { + let _ = clientversion; + + Channel { + id: "perverzija".to_string(), + name: "Perverzija".to_string(), + description: "Free videos from Perverzija".to_string(), + premium: false, + favicon: "https://www.google.com/s2/favicons?sz=64&domain=tube.perverzija.com" + .to_string(), + status: "active".to_string(), + categories: vec![], + options: vec![ChannelOption { + id: "featured".to_string(), + title: "Featured".to_string(), + description: "Filter Featured Videos.".to_string(), + systemImage: "star".to_string(), + colorName: "red".to_string(), + options: vec![ + FilterOption { + id: "all".to_string(), + title: "No".to_string(), + }, + FilterOption { + id: "featured".to_string(), + title: "Yes".to_string(), + }, + ], + multiSelect: false, + }], + nsfw: true, + cacheDuration: None, + } + } + async fn get( &self, cache: VideoCache, @@ -47,7 +85,7 @@ impl PerverzijaProvider { page: u8, options: ServerOptions, ) -> Result> { - let featured = options.featured.unwrap_or("".to_string()); + let featured = options.featured.clone().unwrap_or("".to_string()); let mut prefix_uri = "".to_string(); if featured == "featured" { prefix_uri = "featured-scenes/".to_string(); @@ -71,8 +109,20 @@ impl PerverzijaProvider { } }; - let mut requester = options.requester.clone().unwrap(); - let text = requester.get(&url_str, Some(Version::HTTP_2)).await.unwrap(); + let mut requester = + crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); + let text = match requester.get(&url_str, Some(Version::HTTP_2)).await { + Ok(text) => text, + Err(e) => { + report_provider_error( + "perverzija", + "get.request", + &format!("url={url_str}; error={e}"), + ) + .await; + return Ok(old_items); + } + }; let video_items: Vec = self.get_video_items_from_html(text.clone(), pool); if !video_items.is_empty() { cache.remove(&url_str); @@ -122,8 +172,20 @@ impl PerverzijaProvider { } }; - let mut requester = options.requester.clone().unwrap(); - let text = requester.get(&url_str, Some(Version::HTTP_2)).await.unwrap(); + let mut requester = + crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); + let text = match requester.get(&url_str, Some(Version::HTTP_2)).await { + Ok(text) => text, + Err(e) => { + report_provider_error( + "perverzija", + "query.request", + &format!("url={url_str}; error={e}"), + ) + .await; + return Ok(old_items); + } + }; let video_items: Vec = match query_parse { true => { self.get_video_items_from_html_query(text.clone(), pool) @@ -146,51 +208,61 @@ impl PerverzijaProvider { return vec![]; } let mut items: Vec = Vec::new(); - let video_listing_content = html.split("video-listing-content").collect::>()[1]; + let video_listing_content = html.split("video-listing-content").collect::>().get(1).copied().unwrap_or_default(); let raw_videos = video_listing_content .split("video-item post") .collect::>()[1..] .to_vec(); for video_segment in &raw_videos { let vid = video_segment.split("\n").collect::>(); - if vid.len() > 20 { + if vid.len() > 20 || vid.len() < 8 { + report_provider_error_background( + "perverzija", + "get_video_items_from_html.snippet_shape", + &format!("unexpected snippet length={}", vid.len()), + ); continue; } + let line0 = vid.get(0).copied().unwrap_or_default(); + let line1 = vid.get(1).copied().unwrap_or_default(); + let line4 = vid.get(4).copied().unwrap_or_default(); + let line6 = vid.get(6).copied().unwrap_or_default(); + let line7 = vid.get(7).copied().unwrap_or_default(); // for (index, line) in vid.iter().enumerate() { // println!("Line {}: {}", index, line.to_string().trim()); // } - let mut title = vid[1].split(">").collect::>()[1] + let mut title = line1.split(">").collect::>().get(1).copied().unwrap_or_default() .split("<") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .to_string(); // html decode title = decode(title.as_bytes()).to_string().unwrap_or(title); - if !vid[1].contains("iframe src="") { + if !line1.contains("iframe src="") { continue; } - let url_str = vid[1].split("iframe src="").collect::>()[1] + let url_str = line1.split("iframe src="").collect::>().get(1).copied().unwrap_or_default() .split(""") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .to_string() .replace("index.php", "xs1.php"); if url_str.starts_with("https://streamtape.com/") { continue; // Skip Streamtape links } - let id = url_str.split("data=").collect::>()[1] + let id = url_str.split("data=").collect::>().get(1).copied().unwrap_or_default() .split("&") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .to_string(); let raw_duration = match vid.len() { - 10 => vid[6].split("time_dur\">").collect::>()[1] + 10 => line6.split("time_dur\">").collect::>().get(1).copied().unwrap_or_default() .split("<") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .to_string(), _ => "00:00".to_string(), }; let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32; - if !vid[4].contains("srcset=") - && vid[4].split("src=\"").collect::>().len() == 1 + if !line4.contains("srcset=") + && line4.split("src=\"").collect::>().len() == 1 { for (index, line) in vid.iter().enumerate() { println!("Line {}: {}\n\n", index, line); @@ -201,45 +273,54 @@ impl PerverzijaProvider { for v in vid.clone() { let line = v.trim(); if line.starts_with(">()[1] + thumb = line.split(" src=\"").collect::>().get(1).copied().unwrap_or_default() .split("\"") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .to_string(); } } - let embed_html = vid[1].split("data-embed='").collect::>()[1] + let embed_html = line1.split("data-embed='").collect::>().get(1).copied().unwrap_or_default() .split("'") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .to_string(); - let id_url = vid[1].split("data-url='").collect::>()[1] + let id_url = line1.split("data-url='").collect::>().get(1).copied().unwrap_or_default() .split("'") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .to_string(); - let mut conn = pool.get().expect("couldn't get db connection from pool"); - let _ = db::insert_video(&mut conn, &id_url, &url_str); - drop(conn); + match pool.get() { + Ok(mut conn) => { + let _ = db::insert_video(&mut conn, &id_url, &url_str); + } + Err(e) => { + report_provider_error_background( + "perverzija", + "get_video_items_from_html.insert_video.pool_get", + &e.to_string(), + ); + } + } let referer_url = "https://xtremestream.xyz/".to_string(); let embed = VideoEmbed::new(embed_html, url_str.clone()); let mut tags: Vec = Vec::new(); // Placeholder for tags, adjust as needed - let studios_parts = vid[7].split("a href=\"").collect::>(); + let studios_parts = line7.split("a href=\"").collect::>(); for studio in studios_parts.iter().skip(1) { if studio.starts_with("https://tube.perverzija.com/studio/") { tags.push( - studio.split("/\"").collect::>()[0] + studio.split("/\"").collect::>().get(0).copied().unwrap_or_default() .replace("https://tube.perverzija.com/studio/", "@studio:") .to_string(), ); } } - for tag in vid[0].split(" ").collect::>() { + for tag in line0.split(" ").collect::>() { if tag.starts_with("stars-") { - let tag_name = tag.split("stars-").collect::>()[1] + let tag_name = tag.split("stars-").collect::>().get(1).copied().unwrap_or_default() .split("\"") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .to_string(); if !tag_name.is_empty() { tags.push(format!("@stars:{}", tag_name)); @@ -247,9 +328,9 @@ impl PerverzijaProvider { } } - for tag in vid[0].split(" ").collect::>() { + for tag in line0.split(" ").collect::>() { if tag.starts_with("tag-") { - let tag_name = tag.split("tag-").collect::>()[1].to_string(); + let tag_name = tag.split("tag-").collect::>().get(1).copied().unwrap_or_default().to_string(); if !tag_name.is_empty() { tags.push(tag_name.replace("-", " ").to_string()); } @@ -292,37 +373,55 @@ impl PerverzijaProvider { async fn get_video_item(&self, snippet: &str, pool: DbPool) -> Result { let vid = snippet.split("\n").collect::>(); - if vid.len() > 30 { + if vid.len() > 30 || vid.len() < 7 { + report_provider_error_background( + "perverzija", + "get_video_item.snippet_shape", + &format!("unexpected snippet length={}", vid.len()), + ); return Err("Unexpected video snippet length".into()); } + let line5 = vid.get(5).copied().unwrap_or_default(); + let line6 = vid.get(6).copied().unwrap_or_default(); - let mut title = vid[5].split(" title=\"").collect::>()[1] + let mut title = line5.split(" title=\"").collect::>().get(1).copied().unwrap_or_default() .split("\"") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .to_string(); title = decode(title.as_bytes()).to_string().unwrap_or(title); - let thumb = match vid[6].split(" src=\"").collect::>().len() { + let thumb = match line6.split(" src=\"").collect::>().len() { 1 => { for (index, line) in vid.iter().enumerate() { println!("Line {}: {}", index, line.to_string().trim()); } return Err("Failed to parse thumbnail URL".into()); } - _ => vid[6].split(" src=\"").collect::>()[1] + _ => line6.split(" src=\"").collect::>().get(1).copied().unwrap_or_default() .split("\"") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .to_string(), }; let duration = 0; - let lookup_url = vid[5].split(" href=\"").collect::>()[1] + let lookup_url = line5.split(" href=\"").collect::>().get(1).copied().unwrap_or_default() .split("\"") - .collect::>()[0] + .collect::>().get(0).copied().unwrap_or_default() .to_string(); let referer_url = "https://xtremestream.xyz/".to_string(); - let mut conn = pool.get().expect("couldn't get db connection from pool"); + let mut conn = match pool.get() { + Ok(conn) => conn, + Err(e) => { + report_provider_error( + "perverzija", + "get_video_item.pool_get", + &e.to_string(), + ) + .await; + return Err("couldn't get db connection from pool".into()); + } + }; let db_result = db::get_video(&mut conn, lookup_url.clone()); match db_result { Ok(Some(entry)) => { @@ -334,9 +433,9 @@ impl PerverzijaProvider { if url_str.starts_with("!") { return Err("Video was removed".into()); } - let mut id = url_str.split("data=").collect::>()[1].to_string(); + let mut id = url_str.split("data=").collect::>().get(1).copied().unwrap_or_default().to_string(); if id.contains("&") { - id = id.split("&").collect::>()[0].to_string() + id = id.split("&").collect::>().get(0).copied().unwrap_or_default().to_string() } let mut video_item = VideoItem::new( id, @@ -382,9 +481,9 @@ impl PerverzijaProvider { } }; - let mut url_str = text.split("