From 2627505adeeb5324d4e49b9ca7bf5b729ebbd552 Mon Sep 17 00:00:00 2001 From: Simon Date: Thu, 5 Mar 2026 18:18:48 +0000 Subject: [PATCH] fixes and cleanup --- src/api.rs | 64 +++-- src/main.rs | 62 ++--- src/models.rs | 2 +- src/providers/all.rs | 34 ++- src/providers/beeg.rs | 92 +++++-- src/providers/chaturbate.rs | 9 +- src/providers/freepornvideosxxx.rs | 285 +++++++++++++++++---- src/providers/hanime.rs | 33 ++- src/providers/hentaihaven.rs | 33 +-- src/providers/homoxxx.rs | 173 ++++++++----- src/providers/hqporner.rs | 181 +++++++++---- src/providers/hypnotube.rs | 6 +- src/providers/javtiful.rs | 111 ++++---- src/providers/missav.rs | 216 ++++++++++++---- src/providers/mod.rs | 174 +++++++++---- src/providers/noodlemagazine.rs | 34 +-- src/providers/okporn.rs | 156 ++++++----- src/providers/okxxx.rs | 242 ++++++++++++------ src/providers/omgxxx.rs | 294 ++++++++++++++++----- src/providers/paradisehill.rs | 22 +- src/providers/perfectgirls.rs | 243 ++++++++++++------ src/providers/perverzija.rs | 398 +++++++++++++++++++---------- src/providers/pimpbunny.rs | 49 ++-- src/providers/pmvhaven.rs | 257 ++++++++++++++----- src/providers/porn00.rs | 102 ++++++-- src/providers/pornhat.rs | 166 +++++++++--- src/providers/pornhub.rs | 35 +-- src/providers/pornzog.rs | 42 +-- src/providers/redtube.rs | 107 ++++++-- src/providers/rule34gen.rs | 247 ++++++++++++------ src/providers/rule34video.rs | 22 +- src/providers/sxyprn.rs | 16 +- src/providers/xxdbx.rs | 205 +++++++++++---- src/providers/xxthots.rs | 90 +++++-- src/providers/youjizz.rs | 127 +++++++-- src/proxies/hanimecdn.rs | 2 +- src/proxies/hqpornerthumb.rs | 51 ++++ src/proxies/javtiful.rs | 28 +- src/proxies/mod.rs | 18 +- src/proxies/sxyprn.rs | 41 ++- src/proxy.rs | 23 +- src/status.rs | 18 +- src/util/cache.rs | 2 +- src/util/discord.rs | 16 +- src/util/flaresolverr.rs | 9 +- src/util/mod.rs | 11 +- src/util/proxy.rs | 15 +- src/util/requester.rs | 24 +- src/videos.rs | 34 ++- 49 files changed, 3245 insertions(+), 1376 deletions(-) create mode 100644 src/proxies/hqpornerthumb.rs diff --git a/src/api.rs b/src/api.rs index 6db2e44..08b0420 100644 --- a/src/api.rs +++ b/src/api.rs @@ -1,15 +1,18 @@ +use crate::providers::{ + ALL_PROVIDERS, DynProvider, panic_payload_to_string, report_provider_error, + run_provider_guarded, +}; +use crate::util::cache::VideoCache; +use crate::util::discord::send_discord_error_report; +use crate::util::proxy::{Proxy, all_proxies_snapshot}; +use crate::util::requester::Requester; +use crate::{DbPool, db, status::*, videos::*}; use ntex::http::header; use ntex::web; use ntex::web::HttpRequest; use std::cmp::Ordering; use std::io; use tokio::task; -use crate::providers::{ALL_PROVIDERS, DynProvider, panic_payload_to_string, report_provider_error, run_provider_guarded}; -use crate::util::cache::VideoCache; -use crate::util::discord::send_discord_error_report; -use crate::util::proxy::{Proxy, all_proxies_snapshot}; -use crate::util::requester::Requester; -use crate::{DbPool, db, status::*, videos::*}; #[derive(Debug, Clone)] pub struct ClientVersion { @@ -119,10 +122,9 @@ async fn status(req: HttpRequest) -> Result { let mut status = Status::new(); for (provider_name, provider) in ALL_PROVIDERS.iter() { - let channel_result = - std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - provider.get_channel(clientversion.clone()) - })); + let channel_result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + provider.get_channel(clientversion.clone()) + })); match channel_result { Ok(Some(channel)) => status.add_channel(channel), Ok(None) => {} @@ -218,7 +220,16 @@ async fn videos_post( .as_deref() .unwrap_or("all") .to_string(); - let sites = video_request.sites.as_deref().unwrap_or("").to_string(); + let sites = if channel == "all" { + video_request + .all_provider_sites + .as_deref() + .or(video_request.sites.as_deref()) + .unwrap_or("") + .to_string() + } else { + video_request.sites.as_deref().unwrap_or("").to_string() + }; let filter = video_request.filter.as_deref().unwrap_or("new").to_string(); let language = video_request .language @@ -261,22 +272,25 @@ async fn videos_post( ) .await; - // There is a bug in Hottub38 that makes the client error for a 403-url even though formats work fine + // There is a bug in Hottub38 that makes the client error for a 403-url even though formats work fine if clientversion == ClientVersion::new(38, 0, "Hot%20Tub".to_string()) { // filter out videos without preview for old clients - video_items = video_items.into_iter().filter_map(|video| { - let last_url = video - .formats - .as_ref() - .and_then(|formats| formats.last().map(|f| f.url.clone())); - if let Some(url) = last_url { - let mut v = video; - v.url = url; - return Some(v); - } - Some(video) - }).collect(); - } + video_items = video_items + .into_iter() + .filter_map(|video| { + let last_url = video + .formats + .as_ref() + .and_then(|formats| formats.last().map(|f| f.url.clone())); + if let Some(url) = last_url { + let mut v = video; + v.url = url; + return Some(v); + } + Some(video) + }) + .collect(); + } videos.items = video_items.clone(); if video_items.len() == 0 { diff --git a/src/main.rs b/src/main.rs index 5f8de6d..2c98c42 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,31 +1,32 @@ #![warn(unused_extern_crates)] #![allow(non_snake_case)] - use std::{env, thread}; -use diesel::{r2d2::{self, ConnectionManager}, SqliteConnection}; +use diesel::{ + SqliteConnection, + r2d2::{self, ConnectionManager}, +}; use dotenvy::dotenv; -use ntex_files as fs; use ntex::web; +use ntex_files as fs; mod api; -mod proxy; mod db; mod models; mod providers; +mod proxies; +mod proxy; mod schema; mod status; mod util; mod videos; -mod proxies; type DbPool = r2d2::Pool>; // #[macro_use(c)] // extern crate cute; - #[ntex::main] async fn main() -> std::io::Result<()> { // std::env::set_var("RUST_BACKTRACE", "1"); @@ -33,7 +34,7 @@ async fn main() -> std::io::Result<()> { // Enable request logging if std::env::var("RUST_LOG").is_err() { - unsafe{ + unsafe { std::env::set_var("RUST_LOG", "warn"); } } @@ -49,20 +50,22 @@ async fn main() -> std::io::Result<()> { let mut requester = util::requester::Requester::new(); requester.set_proxy(env::var("PROXY").unwrap_or("0".to_string()) != "0".to_string()); - let cache: util::cache::VideoCache = crate::util::cache::VideoCache::new().max_size(100_000).to_owned(); - - thread::spawn(move || { - // Create a tiny runtime just for these async tasks - let rt = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .expect("build tokio runtime"); + let cache: util::cache::VideoCache = crate::util::cache::VideoCache::new() + .max_size(100_000) + .to_owned(); - rt.block_on(async move { - providers::init_providers_now(); - }); + thread::spawn(move || { + // Create a tiny runtime just for these async tasks + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("build tokio runtime"); + + rt.block_on(async move { + providers::init_providers_now(); }); - + }); + web::HttpServer::new(move || { web::App::new() .state(pool.clone()) @@ -72,17 +75,16 @@ async fn main() -> std::io::Result<()> { .service(web::scope("/api").configure(api::config)) .service(web::scope("/proxy").configure(proxy::config)) .service( - web::resource("/") - .route(web::get().to(|req: web::HttpRequest| async move{ - let host = match std::env::var("DOMAIN"){ - Ok(d) => d, - Err(_) => req.connection_info().host().to_string() - }; - let source_forward_header = format!("hottub://source?url={}", host); - web::HttpResponse::Found() - .header("Location", source_forward_header) - .finish() - })) + web::resource("/").route(web::get().to(|req: web::HttpRequest| async move { + let host = match std::env::var("DOMAIN") { + Ok(d) => d, + Err(_) => req.connection_info().host().to_string(), + }; + let source_forward_header = format!("hottub://source?url={}", host); + web::HttpResponse::Found() + .header("Location", source_forward_header) + .finish() + })), ) .service(fs::Files::new("/", "static").index_file("index.html")) }) diff --git a/src/models.rs b/src/models.rs index fda1bae..34e007c 100644 --- a/src/models.rs +++ b/src/models.rs @@ -1,5 +1,5 @@ use diesel::prelude::*; -use serde::{Serialize}; +use serde::Serialize; #[derive(Debug, Clone, Serialize, Queryable, Insertable)] #[diesel(table_name = crate::schema::videos)] diff --git a/src/providers/all.rs b/src/providers/all.rs index 7c52e19..92c261e 100644 --- a/src/providers/all.rs +++ b/src/providers/all.rs @@ -1,18 +1,18 @@ -use std::fs; -use std::time::Duration; +use crate::DbPool; +use crate::api::{ClientVersion, get_provider}; +use crate::providers::{DynProvider, Provider, report_provider_error, run_provider_guarded}; +use crate::status::{Channel, ChannelOption, FilterOption}; +use crate::util::cache::VideoCache; +use crate::util::interleave; +use crate::videos::{ServerOptions, VideoItem}; use async_trait::async_trait; use capitalize::Capitalize; use cute::c; use error_chain::error_chain; use futures::StreamExt; use futures::stream::FuturesUnordered; -use crate::api::{get_provider, ClientVersion}; -use crate::providers::{DynProvider, Provider, report_provider_error, run_provider_guarded}; -use crate::status::{Channel, ChannelOption, FilterOption}; -use crate::util::cache::VideoCache; -use crate::util::interleave; -use crate::videos::{ServerOptions, VideoItem}; -use crate::DbPool; +use std::fs; +use std::time::Duration; error_chain! { foreign_links { @@ -21,11 +21,9 @@ error_chain! { } } - #[derive(Debug, Clone)] #[allow(dead_code)] -pub struct AllProvider { -} +pub struct AllProvider {} impl AllProvider { pub fn new() -> Self { @@ -81,7 +79,8 @@ impl Provider for AllProvider { None => { // fire-and-forget reporting of missing provider keys tokio::spawn(async move { - report_provider_error("all", "all.get_videos.unknown_provider", &name).await; + report_provider_error("all", "all.get_videos.unknown_provider", &name) + .await; }); None } @@ -128,7 +127,7 @@ impl Provider for AllProvider { }, _ = &mut timeout_timer => { // 55 seconds passed. Stop waiting and return what we have. - // The tasks remaining in 'futures' will continue running in the + // The tasks remaining in 'futures' will continue running in the // background because they were 'tokio::spawn'ed. break; }, @@ -146,10 +145,7 @@ impl Provider for AllProvider { .filter_map(|entry| entry.ok()) .filter_map(|entry| entry.file_name().into_string().ok()) .filter(|name| name.ends_with(".rs")) - .filter(|name| { - !name.contains("mod.rs") - && !name.contains("all.rs") - }) + .filter(|name| !name.contains("mod.rs") && !name.contains("all.rs")) .map(|name| name.replace(".rs", "")) .collect::>(); let sites = c![FilterOption { @@ -166,7 +162,7 @@ impl Provider for AllProvider { status: "active".to_string(), categories: vec![], options: vec![ChannelOption { - id: "sites".to_string(), + id: "all_provider_sites".to_string(), title: "Sites".to_string(), description: "What Sites to use".to_string(), systemImage: "list.number".to_string(), diff --git a/src/providers/beeg.rs b/src/providers/beeg.rs index 95da1c1..69231c7 100644 --- a/src/providers/beeg.rs +++ b/src/providers/beeg.rs @@ -37,9 +37,18 @@ pub struct BeegProvider { impl BeegProvider { pub fn new() -> Self { let provider = BeegProvider { - sites: Arc::new(RwLock::new(vec![FilterOption { id: "all".into(), title: "All".into() }])), - stars: Arc::new(RwLock::new(vec![FilterOption { id: "all".into(), title: "All".into() }])), - categories: Arc::new(RwLock::new(vec![FilterOption { id: "all".into(), title: "All".into() }])), + sites: Arc::new(RwLock::new(vec![FilterOption { + id: "all".into(), + title: "All".into(), + }])), + stars: Arc::new(RwLock::new(vec![FilterOption { + id: "all".into(), + title: "All".into(), + }])), + categories: Arc::new(RwLock::new(vec![FilterOption { + id: "all".into(), + title: "All".into(), + }])), }; provider.spawn_initial_load(); @@ -52,7 +61,10 @@ impl BeegProvider { let stars = Arc::clone(&self.stars); thread::spawn(move || { - let rt = match tokio::runtime::Builder::new_current_thread().enable_all().build() { + let rt = match tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + { Ok(rt) => rt, Err(e) => { eprintln!("beeg runtime init failed: {}", e); @@ -77,14 +89,18 @@ impl BeegProvider { async fn fetch_tags() -> Result { let mut requester = util::requester::Requester::new(); let text = match requester - .get("https://store.externulls.com/tag/facts/tags?get_original=true&slug=index", None) - .await { - Ok(text) => text, - Err(e) => { - eprintln!("beeg fetch_tags failed: {}", e); - return Err(ErrorKind::Parse("failed to fetch tags".into()).into()); - } - }; + .get( + "https://store.externulls.com/tag/facts/tags?get_original=true&slug=index", + None, + ) + .await + { + Ok(text) => text, + Err(e) => { + eprintln!("beeg fetch_tags failed: {}", e); + return Err(ErrorKind::Parse("failed to fetch tags".into()).into()); + } + }; Ok(serde_json::from_str(&text)?) } @@ -99,7 +115,13 @@ impl BeegProvider { s.get("tg_name").and_then(|v| v.as_str()), s.get("tg_slug").and_then(|v| v.as_str()), ) { - Self::push_unique(&stars, FilterOption { id: id.into(), title: name.into() }); + Self::push_unique( + &stars, + FilterOption { + id: id.into(), + title: name.into(), + }, + ); } } Ok(()) @@ -139,7 +161,13 @@ impl BeegProvider { s.get("tg_name").and_then(|v| v.as_str()), s.get("tg_slug").and_then(|v| v.as_str()), ) { - Self::push_unique(&sites, FilterOption { id: id.into(), title: name.into() }); + Self::push_unique( + &sites, + FilterOption { + id: id.into(), + title: name.into(), + }, + ); } } Ok(()) @@ -178,7 +206,11 @@ impl BeegProvider { description: "Filter for different Networks".into(), systemImage: "list.dash".into(), colorName: "purple".into(), - options: self.categories.read().map(|v| v.clone()).unwrap_or_default(), + options: self + .categories + .read() + .map(|v| v.clone()) + .unwrap_or_default(), multiSelect: false, }, ChannelOption { @@ -220,7 +252,8 @@ impl BeegProvider { } let video_url = format!( "https://store.externulls.com/facts/tag?limit=100&offset={}{}", - page - 1, match slug { + page - 1, + match slug { "" => "&id=27173".to_string(), _ => format!("&slug={}", slug.replace(" ", "")), } @@ -237,7 +270,8 @@ impl BeegProvider { vec![] } }; - let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); + let mut requester = + crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); let text = match requester.get(&video_url, None).await { Ok(text) => text, Err(e) => { @@ -289,7 +323,8 @@ impl BeegProvider { } }; - let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); + let mut requester = + crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); let text = match requester.get(&video_url, None).await { Ok(text) => text, @@ -323,14 +358,24 @@ impl BeegProvider { }; for video in array { - let file = match video.get("file") { Some(v) => v, None => continue }; - let hls = match file.get("hls_resources") { Some(v) => v, None => continue }; + let file = match video.get("file") { + Some(v) => v, + None => continue, + }; + let hls = match file.get("hls_resources") { + Some(v) => v, + None => continue, + }; let key = match hls.get("fl_cdn_multi").and_then(|v| v.as_str()) { Some(v) => v, None => continue, }; - let id = file.get("id").and_then(|v| v.as_i64()).unwrap_or(0).to_string(); + let id = file + .get("id") + .and_then(|v| v.as_i64()) + .unwrap_or(0) + .to_string(); let title = file .get("data") .and_then(|v| v.get(0)) @@ -352,7 +397,10 @@ impl BeegProvider { .and_then(|s| parse_abbreviated_number(s)) .unwrap_or(0); - let thumb = format!("https://thumbs.externulls.com/videos/{}/0.webp?size=480x270", id); + let thumb = format!( + "https://thumbs.externulls.com/videos/{}/0.webp?size=480x270", + id + ); let mut item = VideoItem::new( id, diff --git a/src/providers/chaturbate.rs b/src/providers/chaturbate.rs index 040fd66..9e75e21 100644 --- a/src/providers/chaturbate.rs +++ b/src/providers/chaturbate.rs @@ -252,7 +252,8 @@ impl ChaturbateProvider { let mut title = video_segment .get("room_subject") .and_then(|v| v.as_str()) - .map(String::from).unwrap_or("".to_string()); + .map(String::from) + .unwrap_or("".to_string()); // html decode title = decode(title.as_bytes()).to_string().unwrap_or(title); let id = username.clone(); @@ -262,7 +263,11 @@ impl ChaturbateProvider { .unwrap_or(&serde_json::Value::String("".to_string())) .as_str() .unwrap_or("") - .split("?").collect::>().get(0).copied().unwrap_or_default() + .split("?") + .collect::>() + .get(0) + .copied() + .unwrap_or_default() .to_string(); let views = video_segment .get("viewers") diff --git a/src/providers/freepornvideosxxx.rs b/src/providers/freepornvideosxxx.rs index 58b4ef5..f2c98e3 100644 --- a/src/providers/freepornvideosxxx.rs +++ b/src/providers/freepornvideosxxx.rs @@ -120,17 +120,40 @@ impl FreepornvideosxxxProvider { .copied() .unwrap_or_default() .split("custom_list_models_models_list_pagination") - .collect::>().get(0).copied().unwrap_or_default(); + .collect::>() + .get(0) + .copied() + .unwrap_or_default(); for stars_element in stars_div.split(">()[1..].to_vec() { - let star_url = stars_element.split("href=\"").collect::>().get(1).copied().unwrap_or_default() + let star_url = stars_element + .split("href=\"") + .collect::>() + .get(1) + .copied() + .unwrap_or_default() .split("\"") - .collect::>().get(0).copied().unwrap_or_default(); - let star_id = star_url.split("/").collect::>().get(4).copied().unwrap_or_default().to_string(); + .collect::>() + .get(0) + .copied() + .unwrap_or_default(); + let star_id = star_url + .split("/") + .collect::>() + .get(4) + .copied() + .unwrap_or_default() + .to_string(); let star_name = stars_element .split("") - .collect::>().get(1).copied().unwrap_or_default() + .collect::>() + .get(1) + .copied() + .unwrap_or_default() .split("<") - .collect::>().get(0).copied().unwrap_or_default() + .collect::>() + .get(0) + .copied() + .unwrap_or_default() .to_string(); Self::push_unique( &stars, @@ -168,19 +191,47 @@ impl FreepornvideosxxxProvider { } let sites_div = text .split("id=\"list_content_sources_sponsors_list_items\"") - .collect::>().get(1).copied().unwrap_or_default() + .collect::>() + .get(1) + .copied() + .unwrap_or_default() .split("class=\"pagination\"") - .collect::>().get(0).copied().unwrap_or_default(); + .collect::>() + .get(0) + .copied() + .unwrap_or_default(); for sites_element in sites_div.split("class=\"headline\"").collect::>()[1..].to_vec() { - let site_url = sites_element.split("href=\"").collect::>().get(1).copied().unwrap_or_default() + let site_url = sites_element + .split("href=\"") + .collect::>() + .get(1) + .copied() + .unwrap_or_default() .split("\"") - .collect::>().get(0).copied().unwrap_or_default(); - let site_id = site_url.split("/").collect::>().get(4).copied().unwrap_or_default().to_string(); - let site_name = sites_element.split("

").collect::>().get(1).copied().unwrap_or_default() + .collect::>() + .get(0) + .copied() + .unwrap_or_default(); + let site_id = site_url + .split("/") + .collect::>() + .get(4) + .copied() + .unwrap_or_default() + .to_string(); + let site_name = sites_element + .split("

") + .collect::>() + .get(1) + .copied() + .unwrap_or_default() .split("<") - .collect::>().get(0).copied().unwrap_or_default() + .collect::>() + .get(0) + .copied() + .unwrap_or_default() .to_string(); Self::push_unique( &sites, @@ -207,22 +258,52 @@ impl FreepornvideosxxxProvider { return Ok(()); } }; - let networks_div = text.split("class=\"sites__list\"").collect::>().get(1).copied().unwrap_or_default() + let networks_div = text + .split("class=\"sites__list\"") + .collect::>() + .get(1) + .copied() + .unwrap_or_default() .split("") - .collect::>().get(0).copied().unwrap_or_default(); + .collect::>() + .get(0) + .copied() + .unwrap_or_default(); for network_element in networks_div.split("sites__item").collect::>()[1..].to_vec() { if network_element.contains("sites__all") { continue; } - let network_url = network_element.split("href=\"").collect::>().get(1).copied().unwrap_or_default() + let network_url = network_element + .split("href=\"") + .collect::>() + .get(1) + .copied() + .unwrap_or_default() .split("\"") - .collect::>().get(0).copied().unwrap_or_default(); - let network_id = network_url.split("/").collect::>().get(4).copied().unwrap_or_default().to_string(); - let network_name = network_element.split(">").collect::>().get(1).copied().unwrap_or_default() + .collect::>() + .get(0) + .copied() + .unwrap_or_default(); + let network_id = network_url + .split("/") + .collect::>() + .get(4) + .copied() + .unwrap_or_default() + .to_string(); + let network_name = network_element + .split(">") + .collect::>() + .get(1) + .copied() + .unwrap_or_default() .split("<") - .collect::>().get(0).copied().unwrap_or_default() + .collect::>() + .get(0) + .copied() + .unwrap_or_default() .to_string(); Self::push_unique( &networks, @@ -271,7 +352,8 @@ impl FreepornvideosxxxProvider { name: "FreePornVideos XXX".to_string(), description: "Free Porn Videos".to_string(), premium: false, - favicon: "https://www.google.com/s2/favicons?sz=64&domain=www.freepornvideos.xxx".to_string(), + favicon: "https://www.google.com/s2/favicons?sz=64&domain=www.freepornvideos.xxx" + .to_string(), status: "active".to_string(), categories: vec![], options: vec![ @@ -522,11 +604,22 @@ impl FreepornvideosxxxProvider { if !html.contains("class=\"item\"") { return items; } - let raw_videos = html.split("videos_list_pagination").collect::>().get(0).copied().unwrap_or_default() + let raw_videos = html + .split("videos_list_pagination") + .collect::>() + .get(0) + .copied() + .unwrap_or_default() .split(" class=\"pagination\" ") - .collect::>().get(0).copied().unwrap_or_default() + .collect::>() + .get(0) + .copied() + .unwrap_or_default() .split("class=\"list-videos\"") - .collect::>().get(1).copied().unwrap_or_default() + .collect::>() + .get(1) + .copied() + .unwrap_or_default() .split("class=\"item\"") .collect::>()[1..] .to_vec(); @@ -535,39 +628,94 @@ impl FreepornvideosxxxProvider { // for (index, line) in vid.iter().enumerate() { // println!("Line {}: {}", index, line); // } - let video_url: String = video_segment.split(">().get(1).copied().unwrap_or_default() + let video_url: String = video_segment + .split(">() + .get(1) + .copied() + .unwrap_or_default() .split("\"") - .collect::>().get(0).copied().unwrap_or_default() + .collect::>() + .get(0) + .copied() + .unwrap_or_default() .to_string(); - let mut title = video_segment.split(" title=\"").collect::>().get(1).copied().unwrap_or_default() + let mut title = video_segment + .split(" title=\"") + .collect::>() + .get(1) + .copied() + .unwrap_or_default() .split("\"") - .collect::>().get(0).copied().unwrap_or_default() + .collect::>() + .get(0) + .copied() + .unwrap_or_default() .to_string(); // html decode title = decode(title.as_bytes()).to_string().unwrap_or(title); - let id = video_url.split("/").collect::>().get(4).copied().unwrap_or_default().to_string(); + let id = video_url + .split("/") + .collect::>() + .get(4) + .copied() + .unwrap_or_default() + .to_string(); - let thumb = match video_segment.split(">().get(1).copied().unwrap_or_default() + let thumb = match video_segment + .split(">() + .get(1) + .copied() + .unwrap_or_default() .contains("data-src=\"") { - true => video_segment.split(">().get(1).copied().unwrap_or_default() + true => video_segment + .split(">() + .get(1) + .copied() + .unwrap_or_default() .split("data-src=\"") - .collect::>().get(1).copied().unwrap_or_default() + .collect::>() + .get(1) + .copied() + .unwrap_or_default() .split("\"") - .collect::>().get(0).copied().unwrap_or_default() + .collect::>() + .get(0) + .copied() + .unwrap_or_default() .to_string(), - false => video_segment.split(">().get(1).copied().unwrap_or_default() + false => video_segment + .split(">() + .get(1) + .copied() + .unwrap_or_default() .split("src=\"") - .collect::>().get(1).copied().unwrap_or_default() + .collect::>() + .get(1) + .copied() + .unwrap_or_default() .split("\"") - .collect::>().get(0).copied().unwrap_or_default() + .collect::>() + .get(0) + .copied() + .unwrap_or_default() .to_string(), }; let raw_duration = video_segment .split("") - .collect::>().get(1).copied().unwrap_or_default() + .collect::>() + .get(1) + .copied() + .unwrap_or_default() .split("<") - .collect::>().get(0).copied().unwrap_or_default() + .collect::>() + .get(0) + .copied() + .unwrap_or_default() .split(" ") .collect::>() .last() @@ -577,9 +725,15 @@ impl FreepornvideosxxxProvider { let views = parse_abbreviated_number( video_segment .split("
") - .collect::>().get(1).copied().unwrap_or_default() + .collect::>() + .get(1) + .copied() + .unwrap_or_default() .split("<") - .collect::>().get(0).copied().unwrap_or_default() + .collect::>() + .get(0) + .copied() + .unwrap_or_default() .to_string() .as_str(), ) @@ -587,9 +741,15 @@ impl FreepornvideosxxxProvider { let preview = video_segment .split("data-preview=\"") - .collect::>().get(1).copied().unwrap_or_default() + .collect::>() + .get(1) + .copied() + .unwrap_or_default() .split("\"") - .collect::>().get(0).copied().unwrap_or_default() + .collect::>() + .get(0) + .copied() + .unwrap_or_default() .to_string(); let site_name = title .split("]") @@ -603,9 +763,15 @@ impl FreepornvideosxxxProvider { let mut tags = match video_segment.contains("class=\"models\">") { true => video_segment .split("class=\"models\">") - .collect::>().get(1).copied().unwrap_or_default() + .collect::>() + .get(1) + .copied() + .unwrap_or_default() .split("
") - .collect::>().get(0).copied().unwrap_or_default() + .collect::>() + .get(0) + .copied() + .unwrap_or_default() .split("href=\"") .collect::>()[1..] .into_iter() @@ -613,17 +779,38 @@ impl FreepornvideosxxxProvider { Self::push_unique( &self.stars, FilterOption { - id: s.split("/").collect::>().get(4).copied().unwrap_or_default().to_string(), - title: s.split(">").collect::>().get(1).copied().unwrap_or_default() + id: s + .split("/") + .collect::>() + .get(4) + .copied() + .unwrap_or_default() + .to_string(), + title: s + .split(">") + .collect::>() + .get(1) + .copied() + .unwrap_or_default() .split("<") - .collect::>().get(0).copied().unwrap_or_default() + .collect::>() + .get(0) + .copied() + .unwrap_or_default() .trim() .to_string(), }, ); - s.split(">").collect::>().get(1).copied().unwrap_or_default() + s.split(">") + .collect::>() + .get(1) + .copied() + .unwrap_or_default() .split("<") - .collect::>().get(0).copied().unwrap_or_default() + .collect::>() + .get(0) + .copied() + .unwrap_or_default() .trim() .to_string() }) diff --git a/src/providers/hanime.rs b/src/providers/hanime.rs index c830dc7..32e6e00 100644 --- a/src/providers/hanime.rs +++ b/src/providers/hanime.rs @@ -4,8 +4,8 @@ use futures::future::join_all; use serde_json::json; use std::vec; -use crate::api::ClientVersion; use crate::DbPool; +use crate::api::ClientVersion; use crate::db; use crate::providers::{Provider, report_provider_error, report_provider_error_background}; use crate::status::*; @@ -272,7 +272,8 @@ impl HanimeProvider { id ); - let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); + let mut requester = + crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); let payload = json!({ "width": 571, "height": 703, "ab": "kh" } ); @@ -289,7 +290,7 @@ impl HanimeProvider { ], ) .await; // Initial request to set cookies - ntex::time::sleep(ntex::time::Seconds(1)).await; + ntex::time::sleep(ntex::time::Seconds(1)).await; let text = requester .get_raw_with_headers( &url, @@ -328,7 +329,12 @@ impl HanimeProvider { let mut url_vec = vec![]; for el in urls.split("\"url\":\"").collect::>() { - let url = el.split("\"").collect::>().get(0).copied().unwrap_or_default(); + let url = el + .split("\"") + .collect::>() + .get(0) + .copied() + .unwrap_or_default(); if !url.is_empty() && url.contains("m3u8") { url_vec.push(url.to_string()); } @@ -381,11 +387,23 @@ impl HanimeProvider { ) -> Result> { let index = format!("hanime:{}:{}:{}", query, page, sort); let order_by = match sort.contains(".") { - true => sort.split(".").collect::>().get(0).copied().unwrap_or_default().to_string(), + true => sort + .split(".") + .collect::>() + .get(0) + .copied() + .unwrap_or_default() + .to_string(), false => "created_at_unix".to_string(), }; let ordering = match sort.contains(".") { - true => sort.split(".").collect::>().get(1).copied().unwrap_or_default().to_string(), + true => sort + .split(".") + .collect::>() + .get(1) + .copied() + .unwrap_or_default() + .to_string(), false => "desc".to_string(), }; let old_items = match cache.get(&index) { @@ -408,7 +426,8 @@ impl HanimeProvider { .order_by(order_by) .ordering(ordering); - let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); + let mut requester = + crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); let response = match requester .post_json("https://search.htv-services.com/search", &search, vec![]) .await diff --git a/src/providers/hentaihaven.rs b/src/providers/hentaihaven.rs index edf85a9..890eb9a 100644 --- a/src/providers/hentaihaven.rs +++ b/src/providers/hentaihaven.rs @@ -137,11 +137,7 @@ impl HentaihavenProvider { options: ServerOptions, pool: DbPool, ) -> Result> { - let video_url = format!( - "{}/?s={}", - self.url, - query.replace(" ", "+"), - ); + let video_url = format!("{}/?s={}", self.url, query.replace(" ", "+"),); // Check our Video Cache. If the result is younger than 1 hour, we return it. let old_items = match cache.get(&video_url) { Some((time, items)) => { @@ -171,8 +167,7 @@ impl HentaihavenProvider { return Ok(old_items); } }; - if page > 1 - { + if page > 1 { return Ok(vec![]); } let video_items: Vec = self @@ -438,8 +433,7 @@ impl HentaihavenProvider { .last() .and_then(|s| s.split("summary-content\">").nth(1)) .and_then(|s| s.split(" Total").nth(0)) - .map(|s| - s.trim().parse::().unwrap_or(0)) + .map(|s| s.trim().parse::().unwrap_or(0)) .unwrap_or(0); let mut formats = vec![]; let episode_block = html @@ -470,14 +464,23 @@ impl HentaihavenProvider { let format = VideoFormat::new(episode_url, "1080p".to_string(), "m3u8".to_string()) .format_id(episode_title.clone()) .http_header("Connection".to_string(), "keep-alive".to_string()) - .http_header("User-Agent".to_string(), "Mozilla/5.0 (X11; Linux x86_64; rv:140.0) Gecko/20100101 Firefox/140.0".to_string()) - .http_header("Accept".to_string(), "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8".to_string()) + .http_header( + "User-Agent".to_string(), + "Mozilla/5.0 (X11; Linux x86_64; rv:140.0) Gecko/20100101 Firefox/140.0" + .to_string(), + ) + .http_header( + "Accept".to_string(), + "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8".to_string(), + ) .http_header("Accept-Language".to_string(), "en-US,en;q=0.5".to_string()) - .http_header("Accept-Encoding".to_string(), "gzip, deflate, br".to_string()) + .http_header( + "Accept-Encoding".to_string(), + "gzip, deflate, br".to_string(), + ) .http_header("Sec-Fetch-Mode".to_string(), "navigate".to_string()) - .http_header("Origin".to_string(), self.url.clone()) - .format_note(episode_title.clone()) - ; + .http_header("Origin".to_string(), self.url.clone()) + .format_note(episode_title.clone()); formats.push(format); } if formats.is_empty() { diff --git a/src/providers/homoxxx.rs b/src/providers/homoxxx.rs index fe676f4..d4146a8 100644 --- a/src/providers/homoxxx.rs +++ b/src/providers/homoxxx.rs @@ -1,5 +1,5 @@ -use crate::api::ClientVersion; use crate::DbPool; +use crate::api::ClientVersion; use crate::providers::{Provider, report_provider_error}; use crate::status::*; use crate::util::cache::VideoCache; @@ -11,7 +11,7 @@ use error_chain::error_chain; use htmlentity::entity::{ICodedDataTrait, decode}; use std::env; use std::vec; -use wreq::{Client}; +use wreq::Client; use wreq_util::Emulation; error_chain! { @@ -67,12 +67,7 @@ impl HomoxxxProvider { cacheDuration: Some(1800), } } - async fn get( - &self, - cache: VideoCache, - page: u8, - sort: &str, - ) -> Result> { + async fn get(&self, cache: VideoCache, page: u8, sort: &str) -> Result> { let sort_string = match sort { "trending" => "/trending", "popular" => "/popular", @@ -93,13 +88,22 @@ impl HomoxxxProvider { }; // let proxy = Proxy::all("http://192.168.0.103:8081").unwrap(); - let client = Client::builder().cert_verification(false).emulation(Emulation::Firefox136).build()?; + let client = Client::builder() + .cert_verification(false) + .emulation(Emulation::Firefox136) + .build()?; - let mut response = client.get(video_url.clone()) - // .proxy(proxy.clone()) - .send().await?; + let mut response = client + .get(video_url.clone()) + // .proxy(proxy.clone()) + .send() + .await?; if response.status().is_redirection() { - let location = match response.headers().get("Location").and_then(|h| h.to_str().ok()) { + let location = match response + .headers() + .get("Location") + .and_then(|h| h.to_str().ok()) + { Some(location) => location, None => { report_provider_error( @@ -132,12 +136,7 @@ impl HomoxxxProvider { let flare_url = match env::var("FLARE_URL") { Ok(url) => url, Err(e) => { - report_provider_error( - "homoxxx", - "get.flare_url", - &e.to_string(), - ) - .await; + report_provider_error("homoxxx", "get.flare_url", &e.to_string()).await; return Ok(old_items); } }; @@ -168,17 +167,18 @@ impl HomoxxxProvider { Ok(video_items) } } - async fn query( - &self, - cache: VideoCache, - page: u8, - query: &str, - ) -> Result> { + async fn query(&self, cache: VideoCache, page: u8, query: &str) -> Result> { let search_string = query.to_lowercase().trim().replace(" ", "-"); let mut video_url = format!("{}/search/{}/{}/", self.url, search_string, page); - if search_string.starts_with("@"){ - let url_part = search_string.split("@").collect::>().get(1).copied().unwrap_or_default().replace(":", "/"); + if search_string.starts_with("@") { + let url_part = search_string + .split("@") + .collect::>() + .get(1) + .copied() + .unwrap_or_default() + .replace(":", "/"); video_url = format!("{}/{}/", self.url, url_part); } // Check our Video Cache. If the result is younger than 1 hour, we return it. @@ -197,14 +197,23 @@ impl HomoxxxProvider { }; // let proxy = Proxy::all("http://192.168.0.103:8081").unwrap(); - let client = Client::builder().cert_verification(false).emulation(Emulation::Firefox136).build()?; + let client = Client::builder() + .cert_verification(false) + .emulation(Emulation::Firefox136) + .build()?; - let mut response = client.get(video_url.clone()) - // .proxy(proxy.clone()) - .send().await?; + let mut response = client + .get(video_url.clone()) + // .proxy(proxy.clone()) + .send() + .await?; if response.status().is_redirection() { - let location = match response.headers().get("Location").and_then(|h| h.to_str().ok()) { + let location = match response + .headers() + .get("Location") + .and_then(|h| h.to_str().ok()) + { Some(location) => location, None => { report_provider_error( @@ -237,12 +246,7 @@ impl HomoxxxProvider { let flare_url = match env::var("FLARE_URL") { Ok(url) => url, Err(e) => { - report_provider_error( - "homoxxx", - "query.flare_url", - &e.to_string(), - ) - .await; + report_provider_error("homoxxx", "query.flare_url", &e.to_string()).await; return Ok(old_items); } }; @@ -277,7 +281,12 @@ impl HomoxxxProvider { return vec![]; } let mut items: Vec = Vec::new(); - let raw_videos = html.split("pagination").collect::>().get(0).copied().unwrap_or_default() + let raw_videos = html + .split("pagination") + .collect::>() + .get(0) + .copied() + .unwrap_or_default() .split("
") .collect::>()[1..] .to_vec(); @@ -286,31 +295,81 @@ impl HomoxxxProvider { // for (index, line) in vid.iter().enumerate() { // println!("Line {}: {}", index, line); // } - let video_url: String = video_segment.split(">().get(1).copied().unwrap_or_default() - .split("\"") - .collect::>().get(0).copied().unwrap_or_default().to_string(); - let preview_url = video_segment.split("data-preview-custom=\"").collect::>().get(1).copied().unwrap_or_default() + let video_url: String = video_segment + .split(">() + .get(1) + .copied() + .unwrap_or_default() .split("\"") - .collect::>().get(0).copied().unwrap_or_default() + .collect::>() + .get(0) + .copied() + .unwrap_or_default() .to_string(); - let mut title = video_segment.split("\" title=\"").collect::>().get(1).copied().unwrap_or_default() + let preview_url = video_segment + .split("data-preview-custom=\"") + .collect::>() + .get(1) + .copied() + .unwrap_or_default() .split("\"") - .collect::>().get(0).copied().unwrap_or_default() + .collect::>() + .get(0) + .copied() + .unwrap_or_default() + .to_string(); + let mut title = video_segment + .split("\" title=\"") + .collect::>() + .get(1) + .copied() + .unwrap_or_default() + .split("\"") + .collect::>() + .get(0) + .copied() + .unwrap_or_default() .to_string(); // html decode title = decode(title.as_bytes()).to_string().unwrap_or(title); - let id = video_url.split("/").collect::>().get(4).copied().unwrap_or_default().to_string(); + let id = video_url + .split("/") + .collect::>() + .get(4) + .copied() + .unwrap_or_default() + .to_string(); let raw_duration = video_segment - .split("

").collect::>().get(1).copied().unwrap_or_default() + .split("

") + .collect::>() + .get(1) + .copied() + .unwrap_or_default() .split("<") - .collect::>().get(0).copied().unwrap_or_default() + .collect::>() + .get(0) + .copied() + .unwrap_or_default() .to_string(); let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32; - let thumb = video_segment.split("thumb lazyload").collect::>().get(1).copied().unwrap_or_default() - .split("data-src=\"").collect::>().get(1).copied().unwrap_or_default() + let thumb = video_segment + .split("thumb lazyload") + .collect::>() + .get(1) + .copied() + .unwrap_or_default() + .split("data-src=\"") + .collect::>() + .get(1) + .copied() + .unwrap_or_default() .split("\"") - .collect::>().get(0).copied().unwrap_or_default() + .collect::>() + .get(0) + .copied() + .unwrap_or_default() .to_string(); let video_item = VideoItem::new( id, @@ -320,14 +379,11 @@ impl HomoxxxProvider { thumb, duration, ) - .preview(preview_url) - ; + .preview(preview_url); items.push(video_item); } return items; } - - } #[async_trait] @@ -346,10 +402,7 @@ impl Provider for HomoxxxProvider { let _ = per_page; let _ = pool; let videos: std::result::Result, Error> = match query { - Some(q) => { - self.query(cache, page.parse::().unwrap_or(1), &q,) - .await - } + Some(q) => self.query(cache, page.parse::().unwrap_or(1), &q).await, None => { self.get(cache, page.parse::().unwrap_or(1), &sort) .await diff --git a/src/providers/hqporner.rs b/src/providers/hqporner.rs index 4980f89..71891b4 100644 --- a/src/providers/hqporner.rs +++ b/src/providers/hqporner.rs @@ -9,10 +9,9 @@ use crate::util::time::parse_time_to_seconds; use crate::videos::{ServerOptions, VideoFormat, VideoItem}; use async_trait::async_trait; use error_chain::error_chain; -use futures::future::join_all; +use futures::stream::{FuturesUnordered, StreamExt}; use htmlentity::entity::{ICodedDataTrait, decode}; use std::sync::{Arc, RwLock}; -use std::thread::sleep; use std::{thread, vec}; use titlecase::Titlecase; @@ -263,17 +262,37 @@ impl HqpornerProvider { }) .unwrap_or_default(); - let futures = raw_videos - .into_iter() - .map(|el| self.get_video_item(el, requester.clone())); + // Limit concurrent detail-page requests to reduce transient connect errors. + let mut in_flight = FuturesUnordered::new(); + let mut iter = raw_videos.into_iter(); + let mut items = Vec::new(); + const MAX_IN_FLIGHT: usize = 6; - join_all(futures) - .await - .into_iter() - .inspect(|r| { - if let Err(e) = r { + loop { + while in_flight.len() < MAX_IN_FLIGHT { + let Some(seg) = iter.next() else { + break; + }; + in_flight.push(self.get_video_item(seg, requester.clone())); + } + + let Some(result) = in_flight.next().await else { + break; + }; + match result { + Ok(item) + if item + .formats + .as_ref() + .map(|formats| !formats.is_empty()) + .unwrap_or(false) => + { + items.push(item); + } + Ok(_) => {} + Err(e) => { let msg = e.to_string(); - let chain = format_error_chain(e); + let chain = format_error_chain(&e); tokio::spawn(async move { let _ = send_discord_error_report( msg, @@ -287,10 +306,10 @@ impl HqpornerProvider { .await; }); } - }) - .filter_map(Result::ok) - .filter(|item| item.formats.as_ref().map(|formats| !formats.is_empty()).unwrap_or(false)) - .collect() + } + } + + items } async fn get_video_item(&self, seg: String, mut requester: Requester) -> Result { @@ -319,13 +338,22 @@ impl HqpornerProvider { .and_then(|s| s.split('.').next()) .ok_or_else(|| ErrorKind::Parse(format!("id \n{seg}").into()))? .to_string(); - let thumb = format!( - "https:{}", - seg.split("onmouseleave='defaultImage(\"") - .nth(1) - .and_then(|s| s.split('"').next()) - .ok_or_else(|| ErrorKind::Parse(format!("thumb \n{seg}").into()))? - ); + let thumb_raw = seg + .split("onmouseleave='defaultImage(\"") + .nth(1) + .and_then(|s| s.split('"').next()) + .ok_or_else(|| ErrorKind::Parse(format!("thumb \n{seg}").into()))?; + let thumb_abs = if thumb_raw.starts_with("//") { + format!("https:{}", thumb_raw) + } else if thumb_raw.starts_with("http://") || thumb_raw.starts_with("https://") { + thumb_raw.to_string() + } else { + format!("https://{}", thumb_raw.trim_start_matches('/')) + }; + let thumb = match thumb_abs.strip_prefix("https://") { + Some(path) => format!("https://hottub.spacemoehre.de/proxy/hqporner-thumb/{path}"), + None => thumb_abs, + }; let raw_duration = seg .split("") .nth(1) @@ -335,7 +363,7 @@ impl HqpornerProvider { let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32; let (tags, formats) = self.extract_media(&video_url, &mut requester).await?; - + Ok( VideoItem::new(id, title, video_url, "hqporner".into(), thumb, duration) .formats(formats) @@ -350,17 +378,35 @@ impl HqpornerProvider { ) -> Result<(Vec, Vec)> { let mut formats = vec![]; let mut tags = vec![]; - let resp = requester - .get_raw_with_headers( - url, - vec![("Referer".to_string(), "https://hqporner.com/".into())], - ) + let headers = vec![("Referer".to_string(), "https://hqporner.com/".into())]; + let mut text = match self + .fetch_text_with_retries(requester, url, &headers, 3) .await - .map_err(|e| Error::from(format!("Request failed: {}", e)))?; - let text = resp - .text() - .await - .map_err(|e| Error::from(format!("Text conversion failed: {}", e)))?; + { + Ok(text) => text, + Err(primary_err) => { + if url.contains("://hqporner.com/") { + let fallback_url = url.replace("://hqporner.com/", "://www.hqporner.com/"); + self.fetch_text_with_retries(requester, &fallback_url, &headers, 3) + .await + .map_err(|fallback_err| { + Error::from(format!( + "Request failed: primary={primary_err}; fallback={fallback_err}" + )) + })? + } else { + return Err(Error::from(format!("Request failed: {}", primary_err))); + } + } + }; + + if text.is_empty() && url.contains("://hqporner.com/") { + let fallback_url = url.replace("://hqporner.com/", "://www.hqporner.com/"); + text = self + .fetch_text_with_retries(requester, &fallback_url, &headers, 3) + .await + .unwrap_or_default(); + } if text.contains("Why do I see it?") { return Ok((tags, formats)); @@ -395,22 +441,11 @@ impl HqpornerProvider { .and_then(|s| s.split('\'').next()) .ok_or("No player link")? ); - let mut r = requester - .get_raw_with_headers( - &player_url, - vec![("Referer".to_string(), "https://hqporner.com/".into())], - ).await; - - if let Err(_e) = &r { - sleep(std::time::Duration::from_secs(1)); - r = requester - .get_raw_with_headers( - &player_url, - vec![("Referer".to_string(), "https://hqporner.com/".into())], - ).await; - } - let response = match r { - Ok(response) => response, + let response_text = match self + .fetch_text_with_retries(requester, &player_url, &headers, 2) + .await + { + Ok(text) => text, Err(e) => { let err = format!("altplayer request failed: {e}"); send_discord_error_report( @@ -426,13 +461,10 @@ impl HqpornerProvider { return Ok((tags, formats)); } }; - let text2 = response - .text() - .await - .map_err(|e| Error::from(format!("Text conversion failed: {}", e)))?; + let text2 = response_text; // Check for error response - if text2.starts_with("ERR:"){ + if text2.starts_with("ERR:") { return Ok((tags, formats)); } @@ -467,6 +499,37 @@ impl HqpornerProvider { Ok((tags, formats)) } + + async fn fetch_text_with_retries( + &self, + requester: &mut Requester, + url: &str, + headers: &[(String, String)], + max_attempts: u8, + ) -> std::result::Result { + let mut last_err = String::new(); + + for attempt in 1..=max_attempts { + match requester.get_raw_with_headers(url, headers.to_vec()).await { + Ok(resp) => match resp.text().await { + Ok(text) => return Ok(text), + Err(e) => { + last_err = + format!("text read failed (attempt {attempt}/{max_attempts}): {e}"); + } + }, + Err(e) => { + last_err = format!("request failed (attempt {attempt}/{max_attempts}): {e}"); + } + } + + if attempt < max_attempts { + tokio::time::sleep(std::time::Duration::from_millis(250 * attempt as u64)).await; + } + } + + Err(last_err) + } } #[async_trait] @@ -488,7 +551,15 @@ impl Provider for HqpornerProvider { }; res.unwrap_or_else(|e| { eprintln!("Hqporner error: {e}"); - let _ = send_discord_error_report(e.to_string(), Some(format_error_chain(&e)), None, None, file!(), line!(), module_path!()); + let _ = send_discord_error_report( + e.to_string(), + Some(format_error_chain(&e)), + None, + None, + file!(), + line!(), + module_path!(), + ); vec![] }) } diff --git a/src/providers/hypnotube.rs b/src/providers/hypnotube.rs index 667ffa2..7c4f01c 100644 --- a/src/providers/hypnotube.rs +++ b/src/providers/hypnotube.rs @@ -212,7 +212,8 @@ impl HypnotubeProvider { vec![] } }; - let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); + let mut requester = + crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); let text = match requester.get(&video_url, Some(Version::HTTP_11)).await { Ok(text) => text, Err(e) => { @@ -273,7 +274,8 @@ impl HypnotubeProvider { } }; - let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); + let mut requester = + crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); let post_response = match requester .post( format!("{}/searchgate.php", self.url).as_str(), diff --git a/src/providers/javtiful.rs b/src/providers/javtiful.rs index 88fed81..0451d78 100644 --- a/src/providers/javtiful.rs +++ b/src/providers/javtiful.rs @@ -11,9 +11,9 @@ use crate::videos::{ServerOptions, VideoFormat, VideoItem}; use async_trait::async_trait; use error_chain::error_chain; use futures::future::join_all; -use htmlentity::entity::{decode, ICodedDataTrait}; +use htmlentity::entity::{ICodedDataTrait, decode}; use std::sync::{Arc, RwLock}; -use std::{vec}; +use std::vec; use titlecase::Titlecase; use wreq::Version; @@ -118,10 +118,7 @@ impl JavtifulProvider { "most viewed" => "/sort=most_viewed", _ => "", }; - let video_url = format!( - "{}/videos{}?page={}", - self.url, sort_string, page - ); + let video_url = format!("{}/videos{}?page={}", self.url, sort_string, page); let old_items = match cache.get(&video_url) { Some((time, items)) => { if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { @@ -149,7 +146,12 @@ impl JavtifulProvider { return Ok(old_items); } }; - if page > 1 && !text.contains(&format!("

  • {}", page)) { + if page > 1 + && !text.contains(&format!( + "
  • {}", + page + )) + { return Ok(vec![]); } let video_items: Vec = self @@ -178,7 +180,10 @@ impl JavtifulProvider { }; let video_url = format!( "{}/search/videos{}?search_query={}&page={}", - self.url, sort_string, query.replace(" ","+"), page + self.url, + sort_string, + query.replace(" ", "+"), + page ); // Check our Video Cache. If the result is younger than 1 hour, we return it. let old_items = match cache.get(&video_url) { @@ -209,7 +214,12 @@ impl JavtifulProvider { return Ok(old_items); } }; - if page > 1 && !text.contains(&format!("
  • {}", page)) { + if page > 1 + && !text.contains(&format!( + "
  • {}", + page + )) + { return Ok(vec![]); } let video_items: Vec = self @@ -233,11 +243,10 @@ impl JavtifulProvider { return vec![]; } - let block = match html - .split("pagination ") - .next() - .and_then(|s| s.split("row row-cols-1 row-cols-sm-2 row-cols-lg-3 row-cols-xl-4").nth(1)) - { + let block = match html.split("pagination ").next().and_then(|s| { + s.split("row row-cols-1 row-cols-sm-2 row-cols-lg-3 row-cols-xl-4") + .nth(1) + }) { Some(b) => b, None => { eprint!("Javtiful Provider: Failed to get block from html"); @@ -250,9 +259,10 @@ impl JavtifulProvider { file!(), line!(), module_path!(), - ).await; - return vec![] - }, + ) + .await; + return vec![]; + } }; let futures = block @@ -281,7 +291,8 @@ impl JavtifulProvider { file!(), // Note: these might report the utility line line!(), // better to hardcode or pass from outside module_path!(), - ).await; + ) + .await; }); } }) @@ -289,11 +300,7 @@ impl JavtifulProvider { .collect() } - async fn get_video_item( - &self, - seg: String, - mut requester: Requester, - ) -> Result { + async fn get_video_item(&self, seg: String, mut requester: Requester) -> Result { let video_url = seg .split(" href=\"") .nth(1) @@ -309,7 +316,10 @@ impl JavtifulProvider { .trim() .to_string(); - title = decode(title.as_bytes()).to_string().unwrap_or(title).titlecase(); + title = decode(title.as_bytes()) + .to_string() + .unwrap_or(title) + .titlecase(); let id = video_url .split('/') .nth(5) @@ -340,26 +350,17 @@ impl JavtifulProvider { .unwrap_or("") .to_string(); let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32; - let (tags, formats, views) = - self.extract_media(&video_url, &mut requester).await?; + let (tags, formats, views) = self.extract_media(&video_url, &mut requester).await?; if preview.len() == 0 { preview = format!("https://trailers.jav.si/preview/{id}.mp4"); } - let video_item = VideoItem::new( - id, - title, - video_url, - "javtiful".into(), - thumb, - duration, - ) - .formats(formats) - .tags(tags) - .preview(preview) - .views(views); + let video_item = VideoItem::new(id, title, video_url, "javtiful".into(), thumb, duration) + .formats(formats) + .tags(tags) + .preview(preview) + .views(views); Ok(video_item) - } async fn extract_media( @@ -371,7 +372,9 @@ impl JavtifulProvider { .get(url, Some(Version::HTTP_2)) .await .map_err(|e| Error::from(format!("{}", e)))?; - let tags = text.split("related-actress").next() + let tags = text + .split("related-actress") + .next() .and_then(|s| s.split("video-comments").next()) .and_then(|s| s.split(">Tags<").nth(1)) .map(|tag_block| { @@ -383,26 +386,34 @@ impl JavtifulProvider { .split('>') .nth(1) .and_then(|s| s.split('<').next()) - .map(|s| decode(s.as_bytes()).to_string().unwrap_or(s.to_string()).titlecase()) + .map(|s| { + decode(s.as_bytes()) + .to_string() + .unwrap_or(s.to_string()) + .titlecase() + }) }) .collect() }) .unwrap_or_else(|| vec![]); for tag in &tags { - Self::push_unique(&self.categories, FilterOption { - id: tag.to_ascii_lowercase().replace(" ","+"), - title: tag.to_string(), - }); + Self::push_unique( + &self.categories, + FilterOption { + id: tag.to_ascii_lowercase().replace(" ", "+"), + title: tag.to_string(), + }, + ); } - let views = text.split(" Views ") + let views = text + .split(" Views ") .next() .and_then(|s| s.split(" ").last()) - .and_then(|s| s.replace(".","") - .parse::().ok()) + .and_then(|s| s.replace(".", "").parse::().ok()) .unwrap_or(0); - let quality="1080p".to_string(); - let video_url = url.replace("javtiful.com","hottub.spacemoehre.de/proxy/javtiful"); + let quality = "1080p".to_string(); + let video_url = url.replace("javtiful.com", "hottub.spacemoehre.de/proxy/javtiful"); Ok(( tags, vec![VideoFormat::new(video_url, quality, "video/mp4".into())], diff --git a/src/providers/missav.rs b/src/providers/missav.rs index a0d56bc..a057c26 100644 --- a/src/providers/missav.rs +++ b/src/providers/missav.rs @@ -1,20 +1,20 @@ -use std::vec; -use async_trait::async_trait; -use diesel::r2d2; -use error_chain::error_chain; -use htmlentity::entity::{decode, ICodedDataTrait}; -use futures::future::join_all; -use wreq::Version; +use crate::DbPool; use crate::api::ClientVersion; use crate::db; use crate::providers::Provider; use crate::status::*; use crate::util::cache::VideoCache; use crate::util::discord::{format_error_chain, send_discord_error_report}; -use crate::videos::ServerOptions; -use crate::videos::{VideoItem}; -use crate::DbPool; use crate::util::requester::Requester; +use crate::videos::ServerOptions; +use crate::videos::VideoItem; +use async_trait::async_trait; +use diesel::r2d2; +use error_chain::error_chain; +use futures::future::join_all; +use htmlentity::entity::{ICodedDataTrait, decode}; +use std::vec; +use wreq::Version; error_chain! { foreign_links { @@ -39,7 +39,7 @@ pub struct MissavProvider { impl MissavProvider { pub fn new() -> Self { MissavProvider { - url: "https://missav.ws".to_string() + url: "https://missav.ws".to_string(), } } @@ -177,7 +177,14 @@ impl MissavProvider { } } - async fn get(&self, cache: VideoCache, pool: DbPool, page: u8, mut sort: String, options: ServerOptions) -> Result> { + async fn get( + &self, + cache: VideoCache, + pool: DbPool, + page: u8, + mut sort: String, + options: ServerOptions, + ) -> Result> { // Use ok_or to avoid unwrapping options let language = options.language.as_ref().ok_or("Missing language")?; let filter = options.filter.as_ref().ok_or("Missing filter")?; @@ -187,35 +194,57 @@ impl MissavProvider { sort = format!("&sort={}", sort); } let url_str = format!("{}/{}/{}?page={}{}", self.url, language, filter, page, sort); - + if let Some((time, items)) = cache.get(&url_str) { if time.elapsed().unwrap_or_default().as_secs() < 3600 { return Ok(items.clone()); } } - let text = requester.get(&url_str, Some(Version::HTTP_2)).await.unwrap_or_else(|e| { - eprintln!("Error fetching Missav URL {}: {}", url_str, e); - let _ = send_discord_error_report(e.to_string(), None, Some(&url_str), None, file!(), line!(), module_path!()); - "".to_string() - }); + let text = requester + .get(&url_str, Some(Version::HTTP_2)) + .await + .unwrap_or_else(|e| { + eprintln!("Error fetching Missav URL {}: {}", url_str, e); + let _ = send_discord_error_report( + e.to_string(), + None, + Some(&url_str), + None, + file!(), + line!(), + module_path!(), + ); + "".to_string() + }); let video_items = self.get_video_items_from_html(text, pool, requester).await; - + if !video_items.is_empty() { cache.insert(url_str, video_items.clone()); } Ok(video_items) } - async fn query(&self, cache: VideoCache, pool: DbPool, page: u8, query: &str, mut sort: String, options: ServerOptions) -> Result> { + async fn query( + &self, + cache: VideoCache, + pool: DbPool, + page: u8, + query: &str, + mut sort: String, + options: ServerOptions, + ) -> Result> { let language = options.language.as_ref().ok_or("Missing language")?; let mut requester = options.requester.clone().ok_or("Missing requester")?; - + let search_string = query.replace(" ", "%20"); if !sort.is_empty() { sort = format!("&sort={}", sort); } - let url_str = format!("{}/{}/search/{}?page={}{}", self.url, language, search_string, page, sort); + let url_str = format!( + "{}/{}/search/{}?page={}{}", + self.url, language, search_string, page, sort + ); if let Some((time, items)) = cache.get(&url_str) { if time.elapsed().unwrap_or_default().as_secs() < 3600 { @@ -223,24 +252,44 @@ impl MissavProvider { } } - let text = requester.get(&url_str, Some(Version::HTTP_2)).await.unwrap_or_else(|e| { - eprintln!("Error fetching Missav URL {}: {}", url_str, e); - let _ = send_discord_error_report(e.to_string(), None, Some(&url_str), None, file!(), line!(), module_path!()); - "".to_string() - }); + let text = requester + .get(&url_str, Some(Version::HTTP_2)) + .await + .unwrap_or_else(|e| { + eprintln!("Error fetching Missav URL {}: {}", url_str, e); + let _ = send_discord_error_report( + e.to_string(), + None, + Some(&url_str), + None, + file!(), + line!(), + module_path!(), + ); + "".to_string() + }); let video_items = self.get_video_items_from_html(text, pool, requester).await; - + if !video_items.is_empty() { cache.insert(url_str, video_items.clone()); } Ok(video_items) } - async fn get_video_items_from_html(&self, html: String, pool: DbPool, requester: Requester) -> Vec { - if html.is_empty() { return vec![]; } + async fn get_video_items_from_html( + &self, + html: String, + pool: DbPool, + requester: Requester, + ) -> Vec { + if html.is_empty() { + return vec![]; + } let segments: Vec<&str> = html.split("@mouseenter=\"setPreview(\'").collect(); - if segments.len() < 2 { return vec![]; } + if segments.len() < 2 { + return vec![]; + } let mut urls = vec![]; for video_segment in &segments[1..] { @@ -253,14 +302,27 @@ impl MissavProvider { } } - let futures = urls.into_iter().map(|url| self.get_video_item(url, pool.clone(), requester.clone())); - join_all(futures).await.into_iter().filter_map(Result::ok).collect() + let futures = urls + .into_iter() + .map(|url| self.get_video_item(url, pool.clone(), requester.clone())); + join_all(futures) + .await + .into_iter() + .filter_map(Result::ok) + .collect() } - async fn get_video_item(&self, url_str: String, pool: DbPool, mut requester: Requester) -> Result { + async fn get_video_item( + &self, + url_str: String, + pool: DbPool, + mut requester: Requester, + ) -> Result { // 1. Database Check { - let mut conn = pool.get().map_err(|e| Error::from(format!("Pool error: {}", e)))?; + let mut conn = pool + .get() + .map_err(|e| Error::from(format!("Pool error: {}", e)))?; if let Ok(Some(entry)) = db::get_video(&mut conn, url_str.clone()) { if let Ok(video_item) = serde_json::from_str::(entry.as_str()) { return Ok(video_item); @@ -269,11 +331,22 @@ impl MissavProvider { } // 2. Fetch Page - let vid = requester.get(&url_str, Some(Version::HTTP_2)).await.unwrap_or_else(|e| { - eprintln!("Error fetching Missav URL {}: {}", url_str, e); - let _ = send_discord_error_report(e.to_string(), None, Some(&url_str), None, file!(), line!(), module_path!()); - "".to_string() - }); + let vid = requester + .get(&url_str, Some(Version::HTTP_2)) + .await + .unwrap_or_else(|e| { + eprintln!("Error fetching Missav URL {}: {}", url_str, e); + let _ = send_discord_error_report( + e.to_string(), + None, + Some(&url_str), + None, + file!(), + line!(), + module_path!(), + ); + "".to_string() + }); // Helper closure to extract content between two strings let extract = |html: &str, start_tag: &str, end_tag: &str| -> Option { @@ -285,24 +358,33 @@ impl MissavProvider { let mut title = extract(&vid, "().ok()) - .unwrap_or(0); + let duration = extract( + &vid, + "().ok()) + .unwrap_or(0); let id = url_str.split('/').last().ok_or("No ID found")?.to_string(); // 3. Extract Tags (Generic approach to avoid repetitive code) let mut tags = vec![]; - for (label, prefix) in [("Actress:", "@actress"), ("Actor:", "@actor"), ("Maker:", "@maker"), ("Genre:", "@genre")] { + for (label, prefix) in [ + ("Actress:", "@actress"), + ("Actor:", "@actor"), + ("Maker:", "@maker"), + ("Genre:", "@genre"), + ] { let marker = format!("{}", label); if let Some(section) = extract(&vid, &marker, "
    • ") { for part in section.split("class=\"text-nord13 font-medium\">").skip(1) { @@ -331,15 +413,24 @@ impl MissavProvider { parts.get(6)?, parts.get(7)? )) - })().ok_or_else(|| ErrorKind::ParsingError(format!("video_url\n{:?}", vid).to_string()))?; + })() + .ok_or_else(|| ErrorKind::ParsingError(format!("video_url\n{:?}", vid).to_string()))?; - let video_item = VideoItem::new(id, title, video_url, "missav".to_string(), thumb, duration) - .tags(tags) - .preview(format!("https://fourhoi.com/{}/preview.mp4", url_str.split('/').last().unwrap_or_default())); + let video_item = + VideoItem::new(id, title, video_url, "missav".to_string(), thumb, duration) + .tags(tags) + .preview(format!( + "https://fourhoi.com/{}/preview.mp4", + url_str.split('/').last().unwrap_or_default() + )); // 5. Cache to DB if let Ok(mut conn) = pool.get() { - let _ = db::insert_video(&mut conn, &url_str, &serde_json::to_string(&video_item).unwrap_or_default()); + let _ = db::insert_video( + &mut conn, + &url_str, + &serde_json::to_string(&video_item).unwrap_or_default(), + ); } Ok(video_item) @@ -348,7 +439,16 @@ impl MissavProvider { #[async_trait] impl Provider for MissavProvider { - async fn get_videos(&self, cache: VideoCache, pool: DbPool, sort: String, query: Option, page: String, _per_page: String, options: ServerOptions) -> Vec { + async fn get_videos( + &self, + cache: VideoCache, + pool: DbPool, + sort: String, + query: Option, + page: String, + _per_page: String, + options: ServerOptions, + ) -> Vec { let page_num = page.parse::().unwrap_or(1); let result = match query { Some(q) => self.query(cache, pool, page_num, &q, sort, options).await, @@ -357,7 +457,15 @@ impl Provider for MissavProvider { result.unwrap_or_else(|e| { eprintln!("Error fetching videos: {}", e); - let _ = send_discord_error_report(e.to_string(), Some(format_error_chain(&e)), None, None, file!(), line!(), module_path!()); + let _ = send_discord_error_report( + e.to_string(), + Some(format_error_chain(&e)), + None, + None, + file!(), + line!(), + module_path!(), + ); vec![] }) } diff --git a/src/providers/mod.rs b/src/providers/mod.rs index f1d3097..1bde43e 100644 --- a/src/providers/mod.rs +++ b/src/providers/mod.rs @@ -7,7 +7,11 @@ use std::panic::AssertUnwindSafe; use std::sync::Arc; use crate::{ - DbPool, api::ClientVersion, status::Channel, util::{cache::VideoCache, discord::send_discord_error_report, requester::Requester}, videos::{ServerOptions, VideoItem} + DbPool, + api::ClientVersion, + status::Channel, + util::{cache::VideoCache, discord::send_discord_error_report, requester::Requester}, + videos::{ServerOptions, VideoItem}, }; pub mod all; @@ -24,27 +28,27 @@ pub mod pornhat; pub mod redtube; pub mod rule34video; // pub mod hentaimoon; +pub mod beeg; pub mod missav; -pub mod porn00; -pub mod sxyprn; -pub mod xxthots; pub mod omgxxx; pub mod paradisehill; +pub mod porn00; pub mod pornzog; -pub mod youjizz; -pub mod beeg; +pub mod sxyprn; pub mod tnaflix; +pub mod xxthots; +pub mod youjizz; // pub mod pornxp; -pub mod rule34gen; -pub mod xxdbx; -pub mod hqporner; -pub mod noodlemagazine; -pub mod pimpbunny; -pub mod javtiful; -pub mod hypnotube; +pub mod chaturbate; pub mod freepornvideosxxx; pub mod hentaihaven; -pub mod chaturbate; +pub mod hqporner; +pub mod hypnotube; +pub mod javtiful; +pub mod noodlemagazine; +pub mod pimpbunny; +pub mod rule34gen; +pub mod xxdbx; // pub mod tube8; // convenient alias @@ -53,47 +57,128 @@ pub type DynProvider = Arc; pub static ALL_PROVIDERS: Lazy> = Lazy::new(|| { let mut m = HashMap::default(); m.insert("all", Arc::new(all::AllProvider::new()) as DynProvider); - m.insert("perverzija", Arc::new(perverzija::PerverzijaProvider::new()) as DynProvider); - m.insert("hanime", Arc::new(hanime::HanimeProvider::new()) as DynProvider); - m.insert("pornhub", Arc::new(pornhub::PornhubProvider::new()) as DynProvider); + m.insert( + "perverzija", + Arc::new(perverzija::PerverzijaProvider::new()) as DynProvider, + ); + m.insert( + "hanime", + Arc::new(hanime::HanimeProvider::new()) as DynProvider, + ); + m.insert( + "pornhub", + Arc::new(pornhub::PornhubProvider::new()) as DynProvider, + ); m.insert( "rule34video", Arc::new(rule34video::Rule34videoProvider::new()) as DynProvider, ); - m.insert("redtube", Arc::new(redtube::RedtubeProvider::new()) as DynProvider); - m.insert("okporn", Arc::new(okporn::OkpornProvider::new()) as DynProvider); - m.insert("pornhat", Arc::new(pornhat::PornhatProvider::new()) as DynProvider); + m.insert( + "redtube", + Arc::new(redtube::RedtubeProvider::new()) as DynProvider, + ); + m.insert( + "okporn", + Arc::new(okporn::OkpornProvider::new()) as DynProvider, + ); + m.insert( + "pornhat", + Arc::new(pornhat::PornhatProvider::new()) as DynProvider, + ); m.insert( "perfectgirls", Arc::new(perfectgirls::PerfectgirlsProvider::new()) as DynProvider, ); - m.insert("okxxx", Arc::new(okxxx::OkxxxProvider::new()) as DynProvider); - m.insert("homoxxx", Arc::new(homoxxx::HomoxxxProvider::new()) as DynProvider); - m.insert("missav", Arc::new(missav::MissavProvider::new()) as DynProvider); - m.insert("xxthots", Arc::new(xxthots::XxthotsProvider::new()) as DynProvider); - m.insert("sxyprn", Arc::new(sxyprn::SxyprnProvider::new()) as DynProvider); - m.insert("porn00", Arc::new(porn00::Porn00Provider::new()) as DynProvider); - m.insert("youjizz", Arc::new(youjizz::YoujizzProvider::new()) as DynProvider); + m.insert( + "okxxx", + Arc::new(okxxx::OkxxxProvider::new()) as DynProvider, + ); + m.insert( + "homoxxx", + Arc::new(homoxxx::HomoxxxProvider::new()) as DynProvider, + ); + m.insert( + "missav", + Arc::new(missav::MissavProvider::new()) as DynProvider, + ); + m.insert( + "xxthots", + Arc::new(xxthots::XxthotsProvider::new()) as DynProvider, + ); + m.insert( + "sxyprn", + Arc::new(sxyprn::SxyprnProvider::new()) as DynProvider, + ); + m.insert( + "porn00", + Arc::new(porn00::Porn00Provider::new()) as DynProvider, + ); + m.insert( + "youjizz", + Arc::new(youjizz::YoujizzProvider::new()) as DynProvider, + ); m.insert( "paradisehill", Arc::new(paradisehill::ParadisehillProvider::new()) as DynProvider, ); - m.insert("pornzog", Arc::new(pornzog::PornzogProvider::new()) as DynProvider); - m.insert("omgxxx", Arc::new(omgxxx::OmgxxxProvider::new()) as DynProvider); + m.insert( + "pornzog", + Arc::new(pornzog::PornzogProvider::new()) as DynProvider, + ); + m.insert( + "omgxxx", + Arc::new(omgxxx::OmgxxxProvider::new()) as DynProvider, + ); m.insert("beeg", Arc::new(beeg::BeegProvider::new()) as DynProvider); - m.insert("tnaflix", Arc::new(tnaflix::TnaflixProvider::new()) as DynProvider); + m.insert( + "tnaflix", + Arc::new(tnaflix::TnaflixProvider::new()) as DynProvider, + ); // m.insert("pornxp", Arc::new(pornxp::PornxpProvider::new()) as DynProvider); - m.insert("rule34gen", Arc::new(rule34gen::Rule34genProvider::new()) as DynProvider); - m.insert("xxdbx", Arc::new(xxdbx::XxdbxProvider::new()) as DynProvider); - m.insert("hqporner", Arc::new(hqporner::HqpornerProvider::new()) as DynProvider); - m.insert("pmvhaven", Arc::new(pmvhaven::PmvhavenProvider::new()) as DynProvider); - m.insert("noodlemagazine", Arc::new(noodlemagazine::NoodlemagazineProvider::new()) as DynProvider); - m.insert("pimpbunny", Arc::new(pimpbunny::PimpbunnyProvider::new()) as DynProvider); - m.insert("javtiful", Arc::new(javtiful::JavtifulProvider::new()) as DynProvider); - m.insert("hypnotube", Arc::new(hypnotube::HypnotubeProvider::new()) as DynProvider); - m.insert("freepornvideosxxx", Arc::new(freepornvideosxxx::FreepornvideosxxxProvider::new()) as DynProvider); - m.insert("hentaihaven", Arc::new(hentaihaven::HentaihavenProvider::new()) as DynProvider); - m.insert("chaturbate", Arc::new(chaturbate::ChaturbateProvider::new()) as DynProvider); + m.insert( + "rule34gen", + Arc::new(rule34gen::Rule34genProvider::new()) as DynProvider, + ); + m.insert( + "xxdbx", + Arc::new(xxdbx::XxdbxProvider::new()) as DynProvider, + ); + m.insert( + "hqporner", + Arc::new(hqporner::HqpornerProvider::new()) as DynProvider, + ); + m.insert( + "pmvhaven", + Arc::new(pmvhaven::PmvhavenProvider::new()) as DynProvider, + ); + m.insert( + "noodlemagazine", + Arc::new(noodlemagazine::NoodlemagazineProvider::new()) as DynProvider, + ); + m.insert( + "pimpbunny", + Arc::new(pimpbunny::PimpbunnyProvider::new()) as DynProvider, + ); + m.insert( + "javtiful", + Arc::new(javtiful::JavtifulProvider::new()) as DynProvider, + ); + m.insert( + "hypnotube", + Arc::new(hypnotube::HypnotubeProvider::new()) as DynProvider, + ); + m.insert( + "freepornvideosxxx", + Arc::new(freepornvideosxxx::FreepornvideosxxxProvider::new()) as DynProvider, + ); + m.insert( + "hentaihaven", + Arc::new(hentaihaven::HentaihavenProvider::new()) as DynProvider, + ); + m.insert( + "chaturbate", + Arc::new(chaturbate::ChaturbateProvider::new()) as DynProvider, + ); // m.insert("tube8", Arc::new(tube8::Tube8Provider::new()) as DynProvider); // add more here as you migrate them m @@ -159,7 +244,11 @@ pub fn report_provider_error_background(provider_name: &str, context: &str, msg: }); } -pub fn requester_or_default(options: &ServerOptions, provider_name: &str, context: &str) -> Requester { +pub fn requester_or_default( + options: &ServerOptions, + provider_name: &str, + context: &str, +) -> Requester { match options.requester.clone() { Some(requester) => requester, None => { @@ -173,7 +262,6 @@ pub fn requester_or_default(options: &ServerOptions, provider_name: &str, contex } } - #[async_trait] pub trait Provider: Send + Sync { async fn get_videos( diff --git a/src/providers/noodlemagazine.rs b/src/providers/noodlemagazine.rs index efbb1b0..79bd6a8 100644 --- a/src/providers/noodlemagazine.rs +++ b/src/providers/noodlemagazine.rs @@ -3,18 +3,18 @@ use crate::api::ClientVersion; use crate::providers::Provider; use crate::status::*; use crate::util::cache::VideoCache; -use crate::util::requester::Requester; use crate::util::parse_abbreviated_number; +use crate::util::requester::Requester; use crate::util::time::parse_time_to_seconds; use crate::videos::{ServerOptions, VideoFormat, VideoItem}; use async_trait::async_trait; use error_chain::error_chain; use futures::future::join_all; -use htmlentity::entity::{decode, ICodedDataTrait}; -use wreq::Version; -use titlecase::Titlecase; +use htmlentity::entity::{ICodedDataTrait, decode}; use std::vec; +use titlecase::Titlecase; +use wreq::Version; error_chain! { foreign_links { @@ -223,18 +223,16 @@ impl NoodlemagazineProvider { .await .ok_or_else(|| Error::from("media extraction failed"))?; - Ok( - VideoItem::new( - id, - title, - video_url, - "noodlemagazine".into(), - thumb, - duration, - ) - .views(views) - .formats(formats), + Ok(VideoItem::new( + id, + title, + video_url, + "noodlemagazine".into(), + thumb, + duration, ) + .views(views) + .formats(formats)) } async fn extract_media( @@ -247,11 +245,7 @@ impl NoodlemagazineProvider { .await .unwrap_or_default(); - let json_str = text - .split("window.playlist = ") - .nth(1)? - .split(';') - .next()?; + let json_str = text.split("window.playlist = ").nth(1)?.split(';').next()?; let json: serde_json::Value = serde_json::from_str(json_str).ok()?; let sources = json["sources"].as_array()?; diff --git a/src/providers/okporn.rs b/src/providers/okporn.rs index c050eb4..5e45a22 100644 --- a/src/providers/okporn.rs +++ b/src/providers/okporn.rs @@ -1,18 +1,18 @@ -use crate::api::ClientVersion; use crate::DbPool; +use crate::api::ClientVersion; use crate::providers::{Provider, report_provider_error}; use crate::status::*; use crate::util::cache::VideoCache; use crate::util::flaresolverr::{FlareSolverrRequest, Flaresolverr}; use crate::util::time::parse_time_to_seconds; use crate::videos::{ServerOptions, VideoItem}; +use async_trait::async_trait; use error_chain::error_chain; use htmlentity::entity::{ICodedDataTrait, decode}; use std::env; use std::vec; -use wreq::{Client}; +use wreq::Client; use wreq_util::Emulation; -use async_trait::async_trait; error_chain! { foreign_links { @@ -67,12 +67,7 @@ impl OkpornProvider { cacheDuration: Some(1800), } } - async fn get( - &self, - cache: VideoCache, - page: u8, - sort: &str, - ) -> Result> { + async fn get(&self, cache: VideoCache, page: u8, sort: &str) -> Result> { let sort_string = match sort { "trending" => "/trending", "popular" => "/popular", @@ -93,13 +88,22 @@ impl OkpornProvider { }; // let proxy = Proxy::all("http://192.168.0.103:8081").unwrap(); - let client = Client::builder().cert_verification(false).emulation(Emulation::Firefox136).build()?; + let client = Client::builder() + .cert_verification(false) + .emulation(Emulation::Firefox136) + .build()?; - let mut response = client.get(video_url.clone()) - // .proxy(proxy.clone()) - .send().await?; + let mut response = client + .get(video_url.clone()) + // .proxy(proxy.clone()) + .send() + .await?; if response.status().is_redirection() { - let location = match response.headers().get("Location").and_then(|h| h.to_str().ok()) { + let location = match response + .headers() + .get("Location") + .and_then(|h| h.to_str().ok()) + { Some(location) => location, None => { report_provider_error( @@ -131,12 +135,7 @@ impl OkpornProvider { let flare_url = match env::var("FLARE_URL") { Ok(url) => url, Err(e) => { - report_provider_error( - "okporn", - "get.flare_url", - &e.to_string(), - ) - .await; + report_provider_error("okporn", "get.flare_url", &e.to_string()).await; return Ok(old_items); } }; @@ -167,12 +166,7 @@ impl OkpornProvider { Ok(video_items) } } - async fn query( - &self, - cache: VideoCache, - page: u8, - query: &str, - ) -> Result> { + async fn query(&self, cache: VideoCache, page: u8, query: &str) -> Result> { let search_string = query.to_lowercase().trim().replace(" ", "-"); let video_url = format!("{}/search/{}/{}/", self.url, search_string, page); @@ -192,14 +186,23 @@ impl OkpornProvider { }; // let proxy = Proxy::all("http://192.168.0.103:8081").unwrap(); - let client = Client::builder().cert_verification(false).emulation(Emulation::Firefox136).build()?; + let client = Client::builder() + .cert_verification(false) + .emulation(Emulation::Firefox136) + .build()?; - let mut response = client.get(video_url.clone()) - // .proxy(proxy.clone()) - .send().await?; + let mut response = client + .get(video_url.clone()) + // .proxy(proxy.clone()) + .send() + .await?; if response.status().is_redirection() { - let location = match response.headers().get("Location").and_then(|h| h.to_str().ok()) { + let location = match response + .headers() + .get("Location") + .and_then(|h| h.to_str().ok()) + { Some(location) => location, None => { report_provider_error( @@ -232,12 +235,7 @@ impl OkpornProvider { let flare_url = match env::var("FLARE_URL") { Ok(url) => url, Err(e) => { - report_provider_error( - "okporn", - "query.flare_url", - &e.to_string(), - ) - .await; + report_provider_error("okporn", "query.flare_url", &e.to_string()).await; return Ok(old_items); } }; @@ -272,34 +270,78 @@ impl OkpornProvider { return vec![]; } let mut items: Vec = Vec::new(); - let raw_videos = html - .split("
    >()[1..] - .to_vec(); + let raw_videos = html.split("
    >()[1..].to_vec(); for video_segment in &raw_videos { // let vid = video_segment.split("\n").collect::>(); // for (index, line) in vid.iter().enumerate() { // println!("Line {}: {}", index, line); // } - let video_url: String = format!("{}{}", self.url, video_segment.split(">().get(1).copied().unwrap_or_default() + let video_url: String = format!( + "{}{}", + self.url, + video_segment + .split(">() + .get(1) + .copied() + .unwrap_or_default() .split("\"") - .collect::>().get(0).copied().unwrap_or_default()); - let mut title = video_segment.split("\" title=\"").collect::>().get(1).copied().unwrap_or_default() + .collect::>() + .get(0) + .copied() + .unwrap_or_default() + ); + let mut title = video_segment + .split("\" title=\"") + .collect::>() + .get(1) + .copied() + .unwrap_or_default() .split("\"") - .collect::>().get(0).copied().unwrap_or_default() + .collect::>() + .get(0) + .copied() + .unwrap_or_default() .to_string(); // html decode title = decode(title.as_bytes()).to_string().unwrap_or(title); - let id = video_url.split("/").collect::>().get(4).copied().unwrap_or_default().to_string(); - let raw_duration = video_segment.split("").collect::>().get(1).copied().unwrap_or_default() - .split("<") - .collect::>().get(0).copied().unwrap_or_default() - .to_string(); + let id = video_url + .split("/") + .collect::>() + .get(4) + .copied() + .unwrap_or_default() + .to_string(); + let raw_duration = video_segment + .split("") + .collect::>() + .get(1) + .copied() + .unwrap_or_default() + .split("<") + .collect::>() + .get(0) + .copied() + .unwrap_or_default() + .to_string(); let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32; - let thumb = video_segment.split(">().get(1).copied().unwrap_or_default().split("data-original=\"").collect::>().get(1).copied().unwrap_or_default() + let thumb = video_segment + .split(">() + .get(1) + .copied() + .unwrap_or_default() + .split("data-original=\"") + .collect::>() + .get(1) + .copied() + .unwrap_or_default() .split("\"") - .collect::>().get(0).copied().unwrap_or_default() + .collect::>() + .get(0) + .copied() + .unwrap_or_default() .to_string(); let video_item = VideoItem::new( @@ -309,14 +351,11 @@ impl OkpornProvider { "okporn".to_string(), thumb, duration, - ) - ; + ); items.push(video_item); } return items; } - - } #[async_trait] @@ -335,10 +374,7 @@ impl Provider for OkpornProvider { let _ = per_page; let _ = pool; let videos: std::result::Result, Error> = match query { - Some(q) => { - self.query(cache, page.parse::().unwrap_or(1), &q,) - .await - } + Some(q) => self.query(cache, page.parse::().unwrap_or(1), &q).await, None => { self.get(cache, page.parse::().unwrap_or(1), &sort) .await diff --git a/src/providers/okxxx.rs b/src/providers/okxxx.rs index fe8d4c3..ec55ef3 100644 --- a/src/providers/okxxx.rs +++ b/src/providers/okxxx.rs @@ -1,19 +1,19 @@ -use crate::api::ClientVersion; -use crate::util::parse_abbreviated_number; use crate::DbPool; +use crate::api::ClientVersion; use crate::providers::{Provider, report_provider_error}; use crate::status::*; use crate::util::cache::VideoCache; use crate::util::flaresolverr::{FlareSolverrRequest, Flaresolverr}; +use crate::util::parse_abbreviated_number; use crate::util::time::parse_time_to_seconds; use crate::videos::{ServerOptions, VideoItem}; +use async_trait::async_trait; use error_chain::error_chain; use htmlentity::entity::{ICodedDataTrait, decode}; use std::env; use std::vec; -use wreq::{Client}; +use wreq::Client; use wreq_util::Emulation; -use async_trait::async_trait; error_chain! { foreign_links { @@ -68,12 +68,7 @@ impl OkxxxProvider { cacheDuration: Some(1800), } } - async fn get( - &self, - cache: VideoCache, - page: u8, - sort: &str, - ) -> Result> { + async fn get(&self, cache: VideoCache, page: u8, sort: &str) -> Result> { let sort_string = match sort { "trending" => "/trending", "popular" => "/popular", @@ -94,13 +89,22 @@ impl OkxxxProvider { }; // let proxy = Proxy::all("http://192.168.0.103:8081").unwrap(); - let client = Client::builder().cert_verification(false).emulation(Emulation::Firefox136).build()?; + let client = Client::builder() + .cert_verification(false) + .emulation(Emulation::Firefox136) + .build()?; - let mut response = client.get(video_url.clone()) - // .proxy(proxy.clone()) - .send().await?; + let mut response = client + .get(video_url.clone()) + // .proxy(proxy.clone()) + .send() + .await?; if response.status().is_redirection() { - let location = match response.headers().get("Location").and_then(|h| h.to_str().ok()) { + let location = match response + .headers() + .get("Location") + .and_then(|h| h.to_str().ok()) + { Some(location) => location, None => { report_provider_error( @@ -133,12 +137,7 @@ impl OkxxxProvider { let flare_url = match env::var("FLARE_URL") { Ok(url) => url, Err(e) => { - report_provider_error( - "okxxx", - "get.flare_url", - &e.to_string(), - ) - .await; + report_provider_error("okxxx", "get.flare_url", &e.to_string()).await; return Ok(old_items); } }; @@ -169,17 +168,18 @@ impl OkxxxProvider { Ok(video_items) } } - async fn query( - &self, - cache: VideoCache, - page: u8, - query: &str, - ) -> Result> { + async fn query(&self, cache: VideoCache, page: u8, query: &str) -> Result> { let search_string = query.to_lowercase().trim().replace(" ", "-"); let mut video_url = format!("{}/search/{}/{}/", self.url, search_string, page); - if search_string.starts_with("@"){ - let url_part = search_string.split("@").collect::>().get(1).copied().unwrap_or_default().replace(":", "/"); + if search_string.starts_with("@") { + let url_part = search_string + .split("@") + .collect::>() + .get(1) + .copied() + .unwrap_or_default() + .replace(":", "/"); video_url = format!("{}/{}/", self.url, url_part); } // Check our Video Cache. If the result is younger than 1 hour, we return it. @@ -198,14 +198,23 @@ impl OkxxxProvider { }; // let proxy = Proxy::all("http://192.168.0.103:8081").unwrap(); - let client = Client::builder().cert_verification(false).emulation(Emulation::Firefox136).build()?; + let client = Client::builder() + .cert_verification(false) + .emulation(Emulation::Firefox136) + .build()?; - let mut response = client.get(video_url.clone()) - // .proxy(proxy.clone()) - .send().await?; + let mut response = client + .get(video_url.clone()) + // .proxy(proxy.clone()) + .send() + .await?; if response.status().is_redirection() { - let location = match response.headers().get("Location").and_then(|h| h.to_str().ok()) { + let location = match response + .headers() + .get("Location") + .and_then(|h| h.to_str().ok()) + { Some(location) => location, None => { report_provider_error( @@ -238,12 +247,7 @@ impl OkxxxProvider { let flare_url = match env::var("FLARE_URL") { Ok(url) => url, Err(e) => { - report_provider_error( - "okxxx", - "query.flare_url", - &e.to_string(), - ) - .await; + report_provider_error("okxxx", "query.flare_url", &e.to_string()).await; return Ok(old_items); } }; @@ -278,7 +282,12 @@ impl OkxxxProvider { return vec![]; } let mut items: Vec = Vec::new(); - let raw_videos = html.split("
    >().get(0).copied().unwrap_or_default() + let raw_videos = html + .split("
    >() + .get(0) + .copied() + .unwrap_or_default() .split("item thumb-bl thumb-bl-video video_") .collect::>()[1..] .to_vec(); @@ -287,61 +296,150 @@ impl OkxxxProvider { // for (index, line) in vid.iter().enumerate() { // println!("Line {}: {}", index, line); // } - let video_url: String = format!("{}{}", self.url, video_segment.split(">().get(1).copied().unwrap_or_default() + let video_url: String = format!( + "{}{}", + self.url, + video_segment + .split(">() + .get(1) + .copied() + .unwrap_or_default() .split("\"") - .collect::>().get(0).copied().unwrap_or_default()); - let preview_url = video_segment.split("data-preview-custom=\"").collect::>().get(1).copied().unwrap_or_default() + .collect::>() + .get(0) + .copied() + .unwrap_or_default() + ); + let preview_url = video_segment + .split("data-preview-custom=\"") + .collect::>() + .get(1) + .copied() + .unwrap_or_default() .split("\"") - .collect::>().get(0).copied().unwrap_or_default() + .collect::>() + .get(0) + .copied() + .unwrap_or_default() .to_string(); - let mut title = video_segment.split("\" title=\"").collect::>().get(1).copied().unwrap_or_default() + let mut title = video_segment + .split("\" title=\"") + .collect::>() + .get(1) + .copied() + .unwrap_or_default() .split("\"") - .collect::>().get(0).copied().unwrap_or_default() + .collect::>() + .get(0) + .copied() + .unwrap_or_default() .to_string(); // html decode title = decode(title.as_bytes()).to_string().unwrap_or(title); - let id = video_url.split("/").collect::>().get(4).copied().unwrap_or_default().to_string(); - let raw_duration = video_segment.split("fa fa-clock-o").collect::>().get(1).copied().unwrap_or_default() - .split("").collect::>().get(1).copied().unwrap_or_default() + let id = video_url + .split("/") + .collect::>() + .get(4) + .copied() + .unwrap_or_default() + .to_string(); + let raw_duration = video_segment + .split("fa fa-clock-o") + .collect::>() + .get(1) + .copied() + .unwrap_or_default() + .split("") + .collect::>() + .get(1) + .copied() + .unwrap_or_default() .split("<") - .collect::>().get(0).copied().unwrap_or_default() + .collect::>() + .get(0) + .copied() + .unwrap_or_default() .to_string(); let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32; - let thumb = format!("https:{}", video_segment.split(" class=\"thumb lazy-load\"").collect::>().get(1).copied().unwrap_or_default() - .split("data-original=\"").collect::>().get(1).copied().unwrap_or_default() - .split("\"") - .collect::>().get(0).copied().unwrap_or_default() - .to_string()); - + let thumb = format!( + "https:{}", + video_segment + .split(" class=\"thumb lazy-load\"") + .collect::>() + .get(1) + .copied() + .unwrap_or_default() + .split("data-original=\"") + .collect::>() + .get(1) + .copied() + .unwrap_or_default() + .split("\"") + .collect::>() + .get(0) + .copied() + .unwrap_or_default() + .to_string() + ); + let mut tags = vec![]; - if video_segment.contains("href=\"/sites/"){ + if video_segment.contains("href=\"/sites/") { let raw_tags = video_segment.split("href=\"/sites/").collect::>()[1..] .iter() - .map(|s| s.split("/\"").collect::>().get(0).copied().unwrap_or_default().to_string()) + .map(|s| { + s.split("/\"") + .collect::>() + .get(0) + .copied() + .unwrap_or_default() + .to_string() + }) .collect::>(); for tag in raw_tags { if !tag.is_empty() { - tags.push(format!("@sites:{}",tag)); + tags.push(format!("@sites:{}", tag)); } } } - if video_segment.contains("href=\"/models/"){ - let raw_tags = video_segment.split("href=\"/models/").collect::>()[1..] + if video_segment.contains("href=\"/models/") { + let raw_tags = video_segment + .split("href=\"/models/") + .collect::>()[1..] .iter() - .map(|s| s.split("/\"").collect::>().get(0).copied().unwrap_or_default().to_string()) + .map(|s| { + s.split("/\"") + .collect::>() + .get(0) + .copied() + .unwrap_or_default() + .to_string() + }) .collect::>(); for tag in raw_tags { if !tag.is_empty() { - tags.push(format!("@models:{}",tag)); + tags.push(format!("@models:{}", tag)); } } } - let views_part = video_segment.split("fa fa-eye").collect::>().get(1).copied().unwrap_or_default() - .split("").collect::>().get(1).copied().unwrap_or_default() + let views_part = video_segment + .split("fa fa-eye") + .collect::>() + .get(1) + .copied() + .unwrap_or_default() + .split("") + .collect::>() + .get(1) + .copied() + .unwrap_or_default() .split("<") - .collect::>().get(0).copied().unwrap_or_default() + .collect::>() + .get(0) + .copied() + .unwrap_or_default() .to_string(); let views = parse_abbreviated_number(&views_part).unwrap_or(0) as u32; @@ -355,14 +453,11 @@ impl OkxxxProvider { ) .preview(preview_url) .views(views) - .tags(tags) - ; + .tags(tags); items.push(video_item); } return items; } - - } #[async_trait] @@ -381,10 +476,7 @@ impl Provider for OkxxxProvider { let _ = per_page; let _ = pool; let videos: std::result::Result, Error> = match query { - Some(q) => { - self.query(cache, page.parse::().unwrap_or(1), &q,) - .await - } + Some(q) => self.query(cache, page.parse::().unwrap_or(1), &q).await, None => { self.get(cache, page.parse::().unwrap_or(1), &sort) .await diff --git a/src/providers/omgxxx.rs b/src/providers/omgxxx.rs index 09b2de6..a5a3be5 100644 --- a/src/providers/omgxxx.rs +++ b/src/providers/omgxxx.rs @@ -120,17 +120,40 @@ impl OmgxxxProvider { .copied() .unwrap_or_default() .split("custom_list_models_models_list_pagination") - .collect::>().get(0).copied().unwrap_or_default(); + .collect::>() + .get(0) + .copied() + .unwrap_or_default(); for stars_element in stars_div.split(">()[1..].to_vec() { - let star_url = stars_element.split("href=\"").collect::>().get(1).copied().unwrap_or_default() + let star_url = stars_element + .split("href=\"") + .collect::>() + .get(1) + .copied() + .unwrap_or_default() .split("\"") - .collect::>().get(0).copied().unwrap_or_default(); - let star_id = star_url.split("/").collect::>().get(4).copied().unwrap_or_default().to_string(); + .collect::>() + .get(0) + .copied() + .unwrap_or_default(); + let star_id = star_url + .split("/") + .collect::>() + .get(4) + .copied() + .unwrap_or_default() + .to_string(); let star_name = stars_element .split("") - .collect::>().get(1).copied().unwrap_or_default() + .collect::>() + .get(1) + .copied() + .unwrap_or_default() .split("<") - .collect::>().get(0).copied().unwrap_or_default() + .collect::>() + .get(0) + .copied() + .unwrap_or_default() .to_string(); Self::push_unique( &stars, @@ -168,19 +191,47 @@ impl OmgxxxProvider { } let sites_div = text .split("id=\"list_content_sources_sponsors_list_items\"") - .collect::>().get(1).copied().unwrap_or_default() + .collect::>() + .get(1) + .copied() + .unwrap_or_default() .split("class=\"pagination\"") - .collect::>().get(0).copied().unwrap_or_default(); + .collect::>() + .get(0) + .copied() + .unwrap_or_default(); for sites_element in sites_div.split("class=\"headline\"").collect::>()[1..].to_vec() { - let site_url = sites_element.split("href=\"").collect::>().get(1).copied().unwrap_or_default() + let site_url = sites_element + .split("href=\"") + .collect::>() + .get(1) + .copied() + .unwrap_or_default() .split("\"") - .collect::>().get(0).copied().unwrap_or_default(); - let site_id = site_url.split("/").collect::>().get(4).copied().unwrap_or_default().to_string(); - let site_name = sites_element.split("

    ").collect::>().get(1).copied().unwrap_or_default() + .collect::>() + .get(0) + .copied() + .unwrap_or_default(); + let site_id = site_url + .split("/") + .collect::>() + .get(4) + .copied() + .unwrap_or_default() + .to_string(); + let site_name = sites_element + .split("

    ") + .collect::>() + .get(1) + .copied() + .unwrap_or_default() .split("<") - .collect::>().get(0).copied().unwrap_or_default() + .collect::>() + .get(0) + .copied() + .unwrap_or_default() .to_string(); Self::push_unique( &sites, @@ -207,22 +258,52 @@ impl OmgxxxProvider { return Ok(()); } }; - let networks_div = text.split("class=\"sites__list\"").collect::>().get(1).copied().unwrap_or_default() + let networks_div = text + .split("class=\"sites__list\"") + .collect::>() + .get(1) + .copied() + .unwrap_or_default() .split("

    ") - .collect::>().get(0).copied().unwrap_or_default(); + .collect::>() + .get(0) + .copied() + .unwrap_or_default(); for network_element in networks_div.split("sites__item").collect::>()[1..].to_vec() { if network_element.contains("sites__all") { continue; } - let network_url = network_element.split("href=\"").collect::>().get(1).copied().unwrap_or_default() + let network_url = network_element + .split("href=\"") + .collect::>() + .get(1) + .copied() + .unwrap_or_default() .split("\"") - .collect::>().get(0).copied().unwrap_or_default(); - let network_id = network_url.split("/").collect::>().get(4).copied().unwrap_or_default().to_string(); - let network_name = network_element.split(">").collect::>().get(1).copied().unwrap_or_default() + .collect::>() + .get(0) + .copied() + .unwrap_or_default(); + let network_id = network_url + .split("/") + .collect::>() + .get(4) + .copied() + .unwrap_or_default() + .to_string(); + let network_name = network_element + .split(">") + .collect::>() + .get(1) + .copied() + .unwrap_or_default() .split("<") - .collect::>().get(0).copied().unwrap_or_default() + .collect::>() + .get(0) + .copied() + .unwrap_or_default() .to_string(); Self::push_unique( &networks, @@ -420,11 +501,7 @@ impl OmgxxxProvider { } } Err(e) => { - report_provider_error_background( - "omgxxx", - "query.stars_read", - &e.to_string(), - ); + report_provider_error_background("omgxxx", "query.stars_read", &e.to_string()); } } match self.sites.read() { @@ -438,11 +515,7 @@ impl OmgxxxProvider { } } Err(e) => { - report_provider_error_background( - "omgxxx", - "query.sites_read", - &e.to_string(), - ); + report_provider_error_background("omgxxx", "query.sites_read", &e.to_string()); } } let mut video_url = format!("{}/{}/{}/{}/", self.url, search_type, search_string, page); @@ -522,11 +595,22 @@ impl OmgxxxProvider { if !html.contains("class=\"item\"") { return items; } - let raw_videos = html.split("videos_list_pagination").collect::>().get(0).copied().unwrap_or_default() + let raw_videos = html + .split("videos_list_pagination") + .collect::>() + .get(0) + .copied() + .unwrap_or_default() .split(" class=\"pagination\" ") - .collect::>().get(0).copied().unwrap_or_default() + .collect::>() + .get(0) + .copied() + .unwrap_or_default() .split("class=\"list-videos\"") - .collect::>().get(1).copied().unwrap_or_default() + .collect::>() + .get(1) + .copied() + .unwrap_or_default() .split("class=\"item\"") .collect::>()[1..] .to_vec(); @@ -535,39 +619,94 @@ impl OmgxxxProvider { // for (index, line) in vid.iter().enumerate() { // println!("Line {}: {}", index, line); // } - let video_url: String = video_segment.split(">().get(1).copied().unwrap_or_default() + let video_url: String = video_segment + .split(">() + .get(1) + .copied() + .unwrap_or_default() .split("\"") - .collect::>().get(0).copied().unwrap_or_default() + .collect::>() + .get(0) + .copied() + .unwrap_or_default() .to_string(); - let mut title = video_segment.split(" title=\"").collect::>().get(1).copied().unwrap_or_default() + let mut title = video_segment + .split(" title=\"") + .collect::>() + .get(1) + .copied() + .unwrap_or_default() .split("\"") - .collect::>().get(0).copied().unwrap_or_default() + .collect::>() + .get(0) + .copied() + .unwrap_or_default() .to_string(); // html decode title = decode(title.as_bytes()).to_string().unwrap_or(title); - let id = video_url.split("/").collect::>().get(4).copied().unwrap_or_default().to_string(); + let id = video_url + .split("/") + .collect::>() + .get(4) + .copied() + .unwrap_or_default() + .to_string(); - let thumb = match video_segment.split("img loading").collect::>().get(1).copied().unwrap_or_default() + let thumb = match video_segment + .split("img loading") + .collect::>() + .get(1) + .copied() + .unwrap_or_default() .contains("data-src=\"") { - true => video_segment.split("img loading").collect::>().get(1).copied().unwrap_or_default() + true => video_segment + .split("img loading") + .collect::>() + .get(1) + .copied() + .unwrap_or_default() .split("data-src=\"") - .collect::>().get(1).copied().unwrap_or_default() + .collect::>() + .get(1) + .copied() + .unwrap_or_default() .split("\"") - .collect::>().get(0).copied().unwrap_or_default() + .collect::>() + .get(0) + .copied() + .unwrap_or_default() .to_string(), - false => video_segment.split("img loading").collect::>().get(1).copied().unwrap_or_default() + false => video_segment + .split("img loading") + .collect::>() + .get(1) + .copied() + .unwrap_or_default() .split("data-original=\"") - .collect::>().get(1).copied().unwrap_or_default() + .collect::>() + .get(1) + .copied() + .unwrap_or_default() .split("\"") - .collect::>().get(0).copied().unwrap_or_default() + .collect::>() + .get(0) + .copied() + .unwrap_or_default() .to_string(), }; let raw_duration = video_segment .split("") - .collect::>().get(1).copied().unwrap_or_default() + .collect::>() + .get(1) + .copied() + .unwrap_or_default() .split("<") - .collect::>().get(0).copied().unwrap_or_default() + .collect::>() + .get(0) + .copied() + .unwrap_or_default() .split(" ") .collect::>() .last() @@ -577,9 +716,15 @@ impl OmgxxxProvider { let views = parse_abbreviated_number( video_segment .split("
    ") - .collect::>().get(1).copied().unwrap_or_default() + .collect::>() + .get(1) + .copied() + .unwrap_or_default() .split("<") - .collect::>().get(0).copied().unwrap_or_default() + .collect::>() + .get(0) + .copied() + .unwrap_or_default() .to_string() .as_str(), ) @@ -587,9 +732,15 @@ impl OmgxxxProvider { let preview = video_segment .split("data-preview=\"") - .collect::>().get(1).copied().unwrap_or_default() + .collect::>() + .get(1) + .copied() + .unwrap_or_default() .split("\"") - .collect::>().get(0).copied().unwrap_or_default() + .collect::>() + .get(0) + .copied() + .unwrap_or_default() .to_string(); let site_name = title .split("]") @@ -603,9 +754,15 @@ impl OmgxxxProvider { let mut tags = match video_segment.contains("class=\"models\">") { true => video_segment .split("class=\"models\">") - .collect::>().get(1).copied().unwrap_or_default() + .collect::>() + .get(1) + .copied() + .unwrap_or_default() .split("
    ") - .collect::>().get(0).copied().unwrap_or_default() + .collect::>() + .get(0) + .copied() + .unwrap_or_default() .split("href=\"") .collect::>()[1..] .into_iter() @@ -613,17 +770,38 @@ impl OmgxxxProvider { Self::push_unique( &self.stars, FilterOption { - id: s.split("/").collect::>().get(4).copied().unwrap_or_default().to_string(), - title: s.split(">").collect::>().get(1).copied().unwrap_or_default() + id: s + .split("/") + .collect::>() + .get(4) + .copied() + .unwrap_or_default() + .to_string(), + title: s + .split(">") + .collect::>() + .get(1) + .copied() + .unwrap_or_default() .split("<") - .collect::>().get(0).copied().unwrap_or_default() + .collect::>() + .get(0) + .copied() + .unwrap_or_default() .trim() .to_string(), }, ); - s.split(">").collect::>().get(1).copied().unwrap_or_default() + s.split(">") + .collect::>() + .get(1) + .copied() + .unwrap_or_default() .split("<") - .collect::>().get(0).copied().unwrap_or_default() + .collect::>() + .get(0) + .copied() + .unwrap_or_default() .trim() .to_string() }) diff --git a/src/providers/paradisehill.rs b/src/providers/paradisehill.rs index 26c1232..3bbc49c 100644 --- a/src/providers/paradisehill.rs +++ b/src/providers/paradisehill.rs @@ -1,14 +1,14 @@ -use crate::api::ClientVersion; use crate::DbPool; +use crate::api::ClientVersion; use crate::providers::Provider; use crate::status::*; use crate::util::cache::VideoCache; use crate::util::requester::Requester; -use crate::videos::VideoItem; use crate::videos::ServerOptions; +use crate::videos::VideoItem; +use async_trait::async_trait; use error_chain::error_chain; use htmlentity::entity::{ICodedDataTrait, decode}; -use async_trait::async_trait; error_chain! { foreign_links { @@ -35,7 +35,8 @@ impl ParadisehillProvider { name: "Paradisehill".to_string(), description: "Porn Movies on Paradise Hill".to_string(), premium: false, - favicon: "https://www.google.com/s2/favicons?sz=64&domain=en.paradisehill.cc".to_string(), + favicon: "https://www.google.com/s2/favicons?sz=64&domain=en.paradisehill.cc" + .to_string(), status: "active".to_string(), categories: vec![], options: vec![], @@ -49,7 +50,8 @@ impl ParadisehillProvider { page: u8, options: ServerOptions, ) -> Result> { - let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); + let mut requester = + crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); let url_str = format!("{}/all/?sort=created_at&page={}", self.url, page); @@ -99,7 +101,8 @@ impl ParadisehillProvider { options: ServerOptions, ) -> Result> { // Extract needed fields from options at the start - let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); + let mut requester = + crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); let search_string = query.replace(" ", "+"); let url_str = format!( "{}/search/?pattern={}&page={}", @@ -200,7 +203,11 @@ impl ParadisehillProvider { .unwrap_or_default() .trim() .to_string(); - let tags = if genre.is_empty() { vec![] } else { vec![genre] }; + let tags = if genre.is_empty() { + vec![] + } else { + vec![genre] + }; items.push( VideoItem::new(id, title, video_url, "paradisehill".to_string(), thumb, 0) @@ -211,7 +218,6 @@ impl ParadisehillProvider { items } - } #[async_trait] diff --git a/src/providers/perfectgirls.rs b/src/providers/perfectgirls.rs index 3da033d..565c584 100644 --- a/src/providers/perfectgirls.rs +++ b/src/providers/perfectgirls.rs @@ -1,19 +1,19 @@ -use crate::api::ClientVersion; -use crate::util::parse_abbreviated_number; use crate::DbPool; +use crate::api::ClientVersion; use crate::providers::{Provider, report_provider_error}; use crate::status::*; use crate::util::cache::VideoCache; use crate::util::flaresolverr::{FlareSolverrRequest, Flaresolverr}; +use crate::util::parse_abbreviated_number; use crate::util::time::parse_time_to_seconds; use crate::videos::{ServerOptions, VideoItem}; +use async_trait::async_trait; use error_chain::error_chain; use htmlentity::entity::{ICodedDataTrait, decode}; use std::env; use std::vec; -use wreq::{Client}; +use wreq::Client; use wreq_util::Emulation; -use async_trait::async_trait; error_chain! { foreign_links { @@ -68,12 +68,7 @@ impl PerfectgirlsProvider { cacheDuration: Some(1800), } } - async fn get( - &self, - cache: VideoCache, - page: u8, - sort: &str, - ) -> Result> { + async fn get(&self, cache: VideoCache, page: u8, sort: &str) -> Result> { let sort_string = match sort { "trending" => "/trending", "popular" => "/popular", @@ -94,13 +89,22 @@ impl PerfectgirlsProvider { }; // let proxy = Proxy::all("http://192.168.0.103:8081").unwrap(); - let client = Client::builder().cert_verification(false).emulation(Emulation::Firefox136).build()?; + let client = Client::builder() + .cert_verification(false) + .emulation(Emulation::Firefox136) + .build()?; - let mut response = client.get(video_url.clone()) - // .proxy(proxy.clone()) - .send().await?; + let mut response = client + .get(video_url.clone()) + // .proxy(proxy.clone()) + .send() + .await?; if response.status().is_redirection() { - let location = match response.headers().get("Location").and_then(|h| h.to_str().ok()) { + let location = match response + .headers() + .get("Location") + .and_then(|h| h.to_str().ok()) + { Some(location) => location, None => { report_provider_error( @@ -133,12 +137,7 @@ impl PerfectgirlsProvider { let flare_url = match env::var("FLARE_URL") { Ok(url) => url, Err(e) => { - report_provider_error( - "perfectgirls", - "get.flare_url", - &e.to_string(), - ) - .await; + report_provider_error("perfectgirls", "get.flare_url", &e.to_string()).await; return Ok(old_items); } }; @@ -169,17 +168,18 @@ impl PerfectgirlsProvider { Ok(video_items) } } - async fn query( - &self, - cache: VideoCache, - page: u8, - query: &str, - ) -> Result> { + async fn query(&self, cache: VideoCache, page: u8, query: &str) -> Result> { let search_string = query.to_lowercase().trim().replace(" ", "-"); let mut video_url = format!("{}/search/{}/{}/", self.url, search_string, page); - if search_string.starts_with("@"){ - let url_part = search_string.split("@").collect::>().get(1).copied().unwrap_or_default().replace(":", "/"); + if search_string.starts_with("@") { + let url_part = search_string + .split("@") + .collect::>() + .get(1) + .copied() + .unwrap_or_default() + .replace(":", "/"); video_url = format!("{}/{}/", self.url, url_part); } // Check our Video Cache. If the result is younger than 1 hour, we return it. @@ -198,14 +198,23 @@ impl PerfectgirlsProvider { }; // let proxy = Proxy::all("http://192.168.0.103:8081").unwrap(); - let client = Client::builder().cert_verification(false).emulation(Emulation::Firefox136).build()?; + let client = Client::builder() + .cert_verification(false) + .emulation(Emulation::Firefox136) + .build()?; - let mut response = client.get(video_url.clone()) - // .proxy(proxy.clone()) - .send().await?; + let mut response = client + .get(video_url.clone()) + // .proxy(proxy.clone()) + .send() + .await?; if response.status().is_redirection() { - let location = match response.headers().get("Location").and_then(|h| h.to_str().ok()) { + let location = match response + .headers() + .get("Location") + .and_then(|h| h.to_str().ok()) + { Some(location) => location, None => { report_provider_error( @@ -238,12 +247,7 @@ impl PerfectgirlsProvider { let flare_url = match env::var("FLARE_URL") { Ok(url) => url, Err(e) => { - report_provider_error( - "perfectgirls", - "query.flare_url", - &e.to_string(), - ) - .await; + report_provider_error("perfectgirls", "query.flare_url", &e.to_string()).await; return Ok(old_items); } }; @@ -278,7 +282,12 @@ impl PerfectgirlsProvider { return vec![]; } let mut items: Vec = Vec::new(); - let raw_videos = html.split("
    >().get(0).copied().unwrap_or_default() + let raw_videos = html + .split("
    >() + .get(0) + .copied() + .unwrap_or_default() .split("item thumb-bl thumb-bl-video video_") .collect::>()[1..] .to_vec(); @@ -287,64 +296,152 @@ impl PerfectgirlsProvider { // for (index, line) in vid.iter().enumerate() { // println!("Line {}: {}", index, line); // } - let video_url: String = format!("{}{}", self.url, video_segment.split(">().get(1).copied().unwrap_or_default() + let video_url: String = format!( + "{}{}", + self.url, + video_segment + .split(">() + .get(1) + .copied() + .unwrap_or_default() .split("\"") - .collect::>().get(0).copied().unwrap_or_default()); - let preview_url = video_segment.split("data-preview-custom=\"").collect::>().get(1).copied().unwrap_or_default() + .collect::>() + .get(0) + .copied() + .unwrap_or_default() + ); + let preview_url = video_segment + .split("data-preview-custom=\"") + .collect::>() + .get(1) + .copied() + .unwrap_or_default() .split("\"") - .collect::>().get(0).copied().unwrap_or_default() + .collect::>() + .get(0) + .copied() + .unwrap_or_default() .to_string(); - let mut title = video_segment.split("\" title=\"").collect::>().get(1).copied().unwrap_or_default() + let mut title = video_segment + .split("\" title=\"") + .collect::>() + .get(1) + .copied() + .unwrap_or_default() .split("\"") - .collect::>().get(0).copied().unwrap_or_default() + .collect::>() + .get(0) + .copied() + .unwrap_or_default() .to_string(); // html decode title = decode(title.as_bytes()).to_string().unwrap_or(title); - let id = video_url.split("/").collect::>().get(4).copied().unwrap_or_default().to_string(); - let raw_duration = video_segment.split("fa fa-clock-o").collect::>().get(1).copied().unwrap_or_default() - .split("").collect::>().get(1).copied().unwrap_or_default() + let id = video_url + .split("/") + .collect::>() + .get(4) + .copied() + .unwrap_or_default() + .to_string(); + let raw_duration = video_segment + .split("fa fa-clock-o") + .collect::>() + .get(1) + .copied() + .unwrap_or_default() + .split("") + .collect::>() + .get(1) + .copied() + .unwrap_or_default() .split("<") - .collect::>().get(0).copied().unwrap_or_default() + .collect::>() + .get(0) + .copied() + .unwrap_or_default() .to_string(); let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32; - let mut thumb = video_segment.split(" class=\"thumb lazy-load\"").collect::>().get(1).copied().unwrap_or_default() - .split("data-original=\"").collect::>().get(1).copied().unwrap_or_default() + let mut thumb = video_segment + .split(" class=\"thumb lazy-load\"") + .collect::>() + .get(1) + .copied() + .unwrap_or_default() + .split("data-original=\"") + .collect::>() + .get(1) + .copied() + .unwrap_or_default() .split("\"") - .collect::>().get(0).copied().unwrap_or_default() + .collect::>() + .get(0) + .copied() + .unwrap_or_default() .to_string(); - if thumb.starts_with("//"){ - thumb = format!("https:{}",thumb); + if thumb.starts_with("//") { + thumb = format!("https:{}", thumb); } - + let mut tags = vec![]; - if video_segment.contains("href=\"/channels/"){ - let raw_tags = video_segment.split("href=\"/channels/").collect::>()[1..] + if video_segment.contains("href=\"/channels/") { + let raw_tags = video_segment + .split("href=\"/channels/") + .collect::>()[1..] .iter() - .map(|s| s.split("/\"").collect::>().get(0).copied().unwrap_or_default().to_string()) + .map(|s| { + s.split("/\"") + .collect::>() + .get(0) + .copied() + .unwrap_or_default() + .to_string() + }) .collect::>(); for tag in raw_tags { if !tag.is_empty() { - tags.push(format!("@channels:{}",tag)); + tags.push(format!("@channels:{}", tag)); } } } - if video_segment.contains("href=\"/pornstars/"){ - let raw_tags = video_segment.split("href=\"/pornstars/").collect::>()[1..] + if video_segment.contains("href=\"/pornstars/") { + let raw_tags = video_segment + .split("href=\"/pornstars/") + .collect::>()[1..] .iter() - .map(|s| s.split("/\"").collect::>().get(0).copied().unwrap_or_default().to_string()) + .map(|s| { + s.split("/\"") + .collect::>() + .get(0) + .copied() + .unwrap_or_default() + .to_string() + }) .collect::>(); for tag in raw_tags { if !tag.is_empty() { - tags.push(format!("@pornstars:{}",tag)); + tags.push(format!("@pornstars:{}", tag)); } } } - let views_part = video_segment.split("fa fa-eye").collect::>().get(1).copied().unwrap_or_default() - .split("").collect::>().get(1).copied().unwrap_or_default() + let views_part = video_segment + .split("fa fa-eye") + .collect::>() + .get(1) + .copied() + .unwrap_or_default() + .split("") + .collect::>() + .get(1) + .copied() + .unwrap_or_default() .split("<") - .collect::>().get(0).copied().unwrap_or_default() + .collect::>() + .get(0) + .copied() + .unwrap_or_default() .to_string(); let views = parse_abbreviated_number(&views_part).unwrap_or(0) as u32; @@ -358,14 +455,11 @@ impl PerfectgirlsProvider { ) .preview(preview_url) .views(views) - .tags(tags) - ; + .tags(tags); items.push(video_item); } return items; } - - } #[async_trait] @@ -384,10 +478,7 @@ impl Provider for PerfectgirlsProvider { let _ = per_page; let _ = pool; let videos: std::result::Result, Error> = match query { - Some(q) => { - self.query(cache, page.parse::().unwrap_or(1), &q,) - .await - } + Some(q) => self.query(cache, page.parse::().unwrap_or(1), &q).await, None => { self.get(cache, page.parse::().unwrap_or(1), &sort) .await diff --git a/src/providers/perverzija.rs b/src/providers/perverzija.rs index f6ca001..528cfba 100644 --- a/src/providers/perverzija.rs +++ b/src/providers/perverzija.rs @@ -6,16 +6,15 @@ use crate::status::*; use crate::util::cache::VideoCache; use crate::util::time::parse_time_to_seconds; use crate::videos::ServerOptions; -use crate::videos::{self, VideoEmbed, VideoItem}; +use crate::videos::{self, VideoItem}; use async_trait::async_trait; use error_chain::error_chain; use futures::future::join_all; use htmlentity::entity::{ICodedDataTrait, decode}; use serde::Deserialize; use serde::Serialize; -use wreq::Version; -use std::vec; use wreq::Client; +use wreq::Version; use wreq_util::Emulation; error_chain! { @@ -78,6 +77,66 @@ impl PerverzijaProvider { } } + fn extract_between<'a>(haystack: &'a str, start: &str, end: &str) -> Option<&'a str> { + let rest = haystack.split(start).nth(1)?; + Some(rest.split(end).next().unwrap_or_default()) + } + + fn extract_iframe_src(haystack: &str) -> String { + Self::extract_between(haystack, "iframe src=\"", "\"") + .or_else(|| Self::extract_between(haystack, "iframe src="", """)) + .unwrap_or_default() + .to_string() + } + + fn extract_thumb(haystack: &str) -> String { + let img_segment = haystack.split(" String { + let mut title = Self::extract_between(haystack, "

    ", "

    ") + .or_else(|| Self::extract_between(haystack, "

    ", "

    ")) + .or_else(|| Self::extract_between(haystack, " title='", "'")) + .or_else(|| Self::extract_between(haystack, " title=\"", "\"")) + .unwrap_or_default() + .to_string(); + title = decode(title.as_bytes()).to_string().unwrap_or(title); + + if title.contains('<') && title.contains('>') { + let mut plain = String::new(); + let mut in_tag = false; + for c in title.chars() { + match c { + '<' => in_tag = true, + '>' => in_tag = false, + _ if !in_tag => plain.push(c), + _ => {} + } + } + let normalized = plain.split_whitespace().collect::>().join(" "); + if !normalized.is_empty() { + title = normalized; + } + } else { + title = title.split_whitespace().collect::>().join(" "); + } + + title.trim().to_string() + } + async fn get( &self, cache: VideoCache, @@ -204,93 +263,91 @@ impl PerverzijaProvider { fn get_video_items_from_html(&self, html: String, pool: DbPool) -> Vec { if html.is_empty() { - println!("HTML is empty"); + report_provider_error_background( + "perverzija", + "get_video_items_from_html.empty_html", + "empty html response", + ); return vec![]; } let mut items: Vec = Vec::new(); - let video_listing_content = html.split("video-listing-content").collect::>().get(1).copied().unwrap_or_default(); - let raw_videos = video_listing_content + let video_listing_content = html.split("video-listing-content").nth(1).unwrap_or(&html); + let raw_videos: Vec<&str> = video_listing_content .split("video-item post") - .collect::>()[1..] - .to_vec(); - for video_segment in &raw_videos { - let vid = video_segment.split("\n").collect::>(); - if vid.len() > 20 || vid.len() < 8 { + .skip(1) + .collect(); + + if raw_videos.is_empty() { + report_provider_error_background( + "perverzija", + "get_video_items_from_html.no_segments", + &format!("html_len={}", html.len()), + ); + return vec![]; + } + + for video_segment in raw_videos { + let title = Self::extract_title(video_segment); + + let embed_html_raw = Self::extract_between(video_segment, "data-embed='", "'") + .or_else(|| Self::extract_between(video_segment, "data-embed=\"", "\"")) + .unwrap_or_default() + .to_string(); + let embed_html = decode(embed_html_raw.as_bytes()) + .to_string() + .unwrap_or(embed_html_raw.clone()); + + let mut url_str = Self::extract_iframe_src(&embed_html); + if url_str.is_empty() { + url_str = Self::extract_iframe_src(video_segment); + } + if url_str.is_empty() { report_provider_error_background( "perverzija", - "get_video_items_from_html.snippet_shape", - &format!("unexpected snippet length={}", vid.len()), + "get_video_items_from_html.url_missing", + "missing iframe src in segment", ); continue; } - let line0 = vid.get(0).copied().unwrap_or_default(); - let line1 = vid.get(1).copied().unwrap_or_default(); - let line4 = vid.get(4).copied().unwrap_or_default(); - let line6 = vid.get(6).copied().unwrap_or_default(); - let line7 = vid.get(7).copied().unwrap_or_default(); - // for (index, line) in vid.iter().enumerate() { - // println!("Line {}: {}", index, line.to_string().trim()); - // } - let mut title = line1.split(">").collect::>().get(1).copied().unwrap_or_default() - .split("<") - .collect::>().get(0).copied().unwrap_or_default() - .to_string(); - // html decode - title = decode(title.as_bytes()).to_string().unwrap_or(title); - if !line1.contains("iframe src="") { - continue; - } - let url_str = line1.split("iframe src="").collect::>().get(1).copied().unwrap_or_default() - .split(""") - .collect::>().get(0).copied().unwrap_or_default() - .to_string() - .replace("index.php", "xs1.php"); + url_str = url_str.replace("index.php", "xs1.php"); if url_str.starts_with("https://streamtape.com/") { continue; // Skip Streamtape links } - let id = url_str.split("data=").collect::>().get(1).copied().unwrap_or_default() - .split("&") - .collect::>().get(0).copied().unwrap_or_default() + + let id_url = Self::extract_between(video_segment, "data-url='", "'") + .or_else(|| Self::extract_between(video_segment, "data-url=\"", "\"")) + .unwrap_or_default() + .to_string(); + + let mut id = url_str + .split("data=") + .nth(1) + .unwrap_or_default() + .split('&') + .next() + .unwrap_or_default() + .to_string(); + if id.is_empty() { + id = id_url + .trim_end_matches('/') + .rsplit('/') + .next() + .unwrap_or_default() + .to_string(); + } + + let raw_duration = Self::extract_between(video_segment, "time_dur\">", "<") + .or_else(|| Self::extract_between(video_segment, "class=\"time\">", "<")) + .unwrap_or("00:00") .to_string(); - let raw_duration = match vid.len() { - 10 => line6.split("time_dur\">").collect::>().get(1).copied().unwrap_or_default() - .split("<") - .collect::>().get(0).copied().unwrap_or_default() - .to_string(), - _ => "00:00".to_string(), - }; let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32; - - if !line4.contains("srcset=") - && line4.split("src=\"").collect::>().len() == 1 - { - for (index, line) in vid.iter().enumerate() { - println!("Line {}: {}\n\n", index, line); - } - } - - let mut thumb = "".to_string(); - for v in vid.clone() { - let line = v.trim(); - if line.starts_with(">().get(1).copied().unwrap_or_default() - .split("\"") - .collect::>().get(0).copied().unwrap_or_default() - .to_string(); - } - } - let embed_html = line1.split("data-embed='").collect::>().get(1).copied().unwrap_or_default() - .split("'") - .collect::>().get(0).copied().unwrap_or_default() - .to_string(); - let id_url = line1.split("data-url='").collect::>().get(1).copied().unwrap_or_default() - .split("'") - .collect::>().get(0).copied().unwrap_or_default() - .to_string(); + let thumb = Self::extract_thumb(video_segment); match pool.get() { Ok(mut conn) => { - let _ = db::insert_video(&mut conn, &id_url, &url_str); + if !id_url.is_empty() { + let _ = db::insert_video(&mut conn, &id_url, &url_str); + } } Err(e) => { report_provider_error_background( @@ -301,26 +358,36 @@ impl PerverzijaProvider { } } let referer_url = "https://xtremestream.xyz/".to_string(); - let embed = VideoEmbed::new(embed_html, url_str.clone()); - let mut tags: Vec = Vec::new(); // Placeholder for tags, adjust as needed + let mut tags: Vec = Vec::new(); - let studios_parts = line7.split("a href=\"").collect::>(); + let studios_parts = video_segment.split("a href=\"").collect::>(); for studio in studios_parts.iter().skip(1) { if studio.starts_with("https://tube.perverzija.com/studio/") { tags.push( - studio.split("/\"").collect::>().get(0).copied().unwrap_or_default() + studio + .split("/\"") + .collect::>() + .get(0) + .copied() + .unwrap_or_default() .replace("https://tube.perverzija.com/studio/", "@studio:") .to_string(), ); } } - for tag in line0.split(" ").collect::>() { - if tag.starts_with("stars-") { - let tag_name = tag.split("stars-").collect::>().get(1).copied().unwrap_or_default() - .split("\"") - .collect::>().get(0).copied().unwrap_or_default() + for tag in video_segment.split_whitespace() { + let token = + tag.trim_matches(|c: char| c == '"' || c == '\'' || c == '>' || c == '<'); + if token.starts_with("stars-") { + let tag_name = token + .split("stars-") + .nth(1) + .unwrap_or_default() + .split('"') + .next() + .unwrap_or_default() .to_string(); if !tag_name.is_empty() { tags.push(format!("@stars:{}", tag_name)); @@ -328,9 +395,11 @@ impl PerverzijaProvider { } } - for tag in line0.split(" ").collect::>() { - if tag.starts_with("tag-") { - let tag_name = tag.split("tag-").collect::>().get(1).copied().unwrap_or_default().to_string(); + for tag in video_segment.split_whitespace() { + let token = + tag.trim_matches(|c: char| c == '"' || c == '\'' || c == '>' || c == '<'); + if token.starts_with("tag-") { + let tag_name = token.split("tag-").nth(1).unwrap_or_default().to_string(); if !tag_name.is_empty() { tags.push(tag_name.replace("-", " ").to_string()); } @@ -339,7 +408,7 @@ impl PerverzijaProvider { let mut video_item = VideoItem::new( id, title, - embed.source.clone(), + url_str.clone(), "perverzija".to_string(), thumb, duration, @@ -361,7 +430,15 @@ impl PerverzijaProvider { } async fn get_video_items_from_html_query(&self, html: String, pool: DbPool) -> Vec { - let raw_videos = html.split("video-item post").collect::>()[1..].to_vec(); + let raw_videos: Vec<&str> = html.split("video-item post").skip(1).collect(); + if raw_videos.is_empty() { + report_provider_error_background( + "perverzija", + "get_video_items_from_html_query.no_segments", + &format!("html_len={}", html.len()), + ); + return vec![]; + } let futures = raw_videos .into_iter() .map(|el| self.get_video_item(el, pool.clone())); @@ -372,53 +449,39 @@ impl PerverzijaProvider { } async fn get_video_item(&self, snippet: &str, pool: DbPool) -> Result { - let vid = snippet.split("\n").collect::>(); - if vid.len() > 30 || vid.len() < 7 { + if snippet.trim().is_empty() { report_provider_error_background( "perverzija", - "get_video_item.snippet_shape", - &format!("unexpected snippet length={}", vid.len()), + "get_video_item.empty_snippet", + "snippet is empty", ); - return Err("Unexpected video snippet length".into()); + return Err("empty snippet".into()); } - let line5 = vid.get(5).copied().unwrap_or_default(); - let line6 = vid.get(6).copied().unwrap_or_default(); - let mut title = line5.split(" title=\"").collect::>().get(1).copied().unwrap_or_default() - .split("\"") - .collect::>().get(0).copied().unwrap_or_default() - .to_string(); - title = decode(title.as_bytes()).to_string().unwrap_or(title); + let title = Self::extract_title(snippet); - let thumb = match line6.split(" src=\"").collect::>().len() { - 1 => { - for (index, line) in vid.iter().enumerate() { - println!("Line {}: {}", index, line.to_string().trim()); - } - return Err("Failed to parse thumbnail URL".into()); - } - _ => line6.split(" src=\"").collect::>().get(1).copied().unwrap_or_default() - .split("\"") - .collect::>().get(0).copied().unwrap_or_default() - .to_string(), - }; + let thumb = Self::extract_thumb(snippet); let duration = 0; - let lookup_url = line5.split(" href=\"").collect::>().get(1).copied().unwrap_or_default() - .split("\"") - .collect::>().get(0).copied().unwrap_or_default() + let lookup_url = Self::extract_between(snippet, " href=\"", "\"") + .or_else(|| Self::extract_between(snippet, "data-url='", "'")) + .unwrap_or_default() .to_string(); + if lookup_url.is_empty() { + report_provider_error_background( + "perverzija", + "get_video_item.lookup_url_missing", + "missing lookup url in snippet", + ); + return Err("Failed to parse lookup url".into()); + } let referer_url = "https://xtremestream.xyz/".to_string(); let mut conn = match pool.get() { Ok(conn) => conn, Err(e) => { - report_provider_error( - "perverzija", - "get_video_item.pool_get", - &e.to_string(), - ) - .await; + report_provider_error("perverzija", "get_video_item.pool_get", &e.to_string()) + .await; return Err("couldn't get db connection from pool".into()); } }; @@ -433,9 +496,21 @@ impl PerverzijaProvider { if url_str.starts_with("!") { return Err("Video was removed".into()); } - let mut id = url_str.split("data=").collect::>().get(1).copied().unwrap_or_default().to_string(); + let mut id = url_str + .split("data=") + .collect::>() + .get(1) + .copied() + .unwrap_or_default() + .to_string(); if id.contains("&") { - id = id.split("&").collect::>().get(0).copied().unwrap_or_default().to_string() + id = id + .split("&") + .collect::>() + .get(0) + .copied() + .unwrap_or_default() + .to_string() } let mut video_item = VideoItem::new( id, @@ -481,9 +556,17 @@ impl PerverzijaProvider { } }; - let mut url_str = text.split("