From 12af9a89cd974d5176c775191e5ce4a5a8c08979 Mon Sep 17 00:00:00 2001 From: Simon Date: Tue, 7 Oct 2025 19:53:08 +0000 Subject: [PATCH] omgxxx bugfix --- src/api.rs | 6 + src/providers/beeg.rs | 364 ++++++++++++++++++++++++++++++++++++++++ src/providers/mod.rs | 5 +- src/providers/omgxxx.rs | 64 ++++--- src/videos.rs | 2 + 5 files changed, 419 insertions(+), 22 deletions(-) create mode 100644 src/providers/beeg.rs diff --git a/src/api.rs b/src/api.rs index be2d81e..8980f37 100644 --- a/src/api.rs +++ b/src/api.rs @@ -1220,6 +1220,11 @@ async fn videos_post( .as_deref() .unwrap_or("") .to_string(); + let categories = video_request + .categories + .as_deref() + .unwrap_or("") + .to_string(); let options = ServerOptions { featured: Some(featured), category: Some(category), @@ -1229,6 +1234,7 @@ async fn videos_post( requester: Some(requester), network: Some(network), stars: Some(stars), + categories: Some(categories), }; let video_items = provider .get_videos( diff --git a/src/providers/beeg.rs b/src/providers/beeg.rs new file mode 100644 index 0000000..387f022 --- /dev/null +++ b/src/providers/beeg.rs @@ -0,0 +1,364 @@ +use crate::DbPool; +use crate::api::ClientVersion; +use crate::providers::Provider; +use crate::schema::videos::star; +use crate::util::cache::VideoCache; +use crate::util::parse_abbreviated_number; +use crate::util::time::parse_time_to_seconds; +use crate::videos::{ServerOptions, VideoItem}; +use crate::{status::*, util}; +use async_trait::async_trait; +use error_chain::error_chain; +use htmlentity::entity::{ICodedDataTrait, decode}; +use serde_json::Value; +use std::sync::{Arc, RwLock}; +use std::thread; +use std::vec; + +error_chain! { + foreign_links { + Io(std::io::Error); + HttpRequest(wreq::Error); + } +} + +#[derive(Debug, Clone)] +pub struct BeegProvider { + url: String, + sites: Arc>>, + stars: Arc>>, + categories: Arc>>, +} +impl BeegProvider { + pub fn new() -> Self { + let provider = BeegProvider { + url: "https://beeg.com".to_string(), + sites: Arc::new(RwLock::new(vec![FilterOption { + id: "all".to_string(), + title: "All".to_string(), + }])), + stars: Arc::new(RwLock::new(vec![FilterOption { + id: "all".to_string(), + title: "All".to_string(), + }])), + categories: Arc::new(RwLock::new(vec![FilterOption { + id: "all".to_string(), + title: "All".to_string(), + }])), + }; + + // Kick off the background load but return immediately + provider.spawn_initial_load(); + provider + } + + fn spawn_initial_load(&self) { + let url = self.url.clone(); + let sites = Arc::clone(&self.sites); + let categories = Arc::clone(&self.categories); + let stars = Arc::clone(&self.stars); + + thread::spawn(move || { + // Create a tiny runtime just for these async tasks + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("build tokio runtime"); + + rt.block_on(async move { + // If you have a streaming sites loader, call it here too + if let Err(e) = Self::load_sites(sites).await { + eprintln!("beeg load_sites_into failed: {e}"); + } + + if let Err(e) = Self::load_categories(categories).await { + eprintln!("beeg load_categories failed: {e}"); + } + + if let Err(e) = Self::load_stars(stars).await { + eprintln!("beeg load_stars failed: {e}"); + } + }); + }); + } + + async fn load_stars(stars: Arc>>) -> Result<()> { + let mut requester = util::requester::Requester::new(); + let text = requester + .get("https://store.externulls.com/tag/facts/tags?get_original=true&slug=index") + .await + .unwrap(); + let json: serde_json::Value = serde_json::from_str::(&text).unwrap(); + let stars_array = json.get("human").unwrap().as_array().unwrap(); + for s in stars_array { + let star_name = s.get("tg_name").unwrap().as_str().unwrap().to_string(); + let star_id = s.get("tg_slug").unwrap().as_str().unwrap().to_string(); + Self::push_unique( + &stars, + FilterOption { + id: star_id, + title: star_name, + }, + ); + } + return Ok(()); + } + + async fn load_categories(categories: Arc>>) -> Result<()> { + let mut requester = util::requester::Requester::new(); + let text = requester + .get("https://store.externulls.com/tag/facts/tags?get_original=true&slug=index") + .await + .unwrap(); + let json: serde_json::Value = serde_json::from_str::(&text).unwrap(); + let stars_array = json.get("other").unwrap().as_array().unwrap(); + for s in stars_array { + let star_name = s.get("tg_name").unwrap().as_str().unwrap().to_string(); + let star_id = s.get("tg_slug").unwrap().as_str().unwrap().to_string(); + Self::push_unique( + &categories, + FilterOption { + id: star_id, + title: star_name, + }, + ); + } + return Ok(()); + } + + async fn load_sites(sites: Arc>>) -> Result<()> { + let mut requester = util::requester::Requester::new(); + let text = requester + .get("https://store.externulls.com/tag/facts/tags?get_original=true&slug=index") + .await + .unwrap(); + let json: serde_json::Value = serde_json::from_str::(&text).unwrap(); + let stars_array = json.get("productions").unwrap().as_array().unwrap(); + for s in stars_array { + let star_name = s.get("tg_name").unwrap().as_str().unwrap().to_string(); + let star_id = s.get("tg_slug").unwrap().as_str().unwrap().to_string(); + Self::push_unique( + &sites, + FilterOption { + id: star_id, + title: star_name, + }, + ); + } + return Ok(()); + } + + // Push one item with minimal lock time and dedup by id + fn push_unique(target: &Arc>>, item: FilterOption) { + if let Ok(mut vec) = target.write() { + if !vec.iter().any(|x| x.id == item.id) { + vec.push(item); + // Optional: keep it sorted for nicer UX + // vec.sort_by(|a,b| a.title.cmp(&b.title)); + } + } + } + + fn build_channel(&self, clientversion: ClientVersion) -> Channel { + let _ = clientversion; + let sites: Vec = self + .sites + .read() + .map(|g| g.clone()) // or: .map(|g| g.to_vec()) + .unwrap_or_default(); // or: .unwrap_or_else(|_| Vec::new()) + + let categories: Vec = self + .categories + .read() + .map(|g| g.clone()) // or: .map(|g| g.to_vec()) + .unwrap_or_default(); // or: .unwrap_or_else(|_| Vec::new()) + + let stars: Vec = self + .stars + .read() + .map(|g| g.clone()) // or: .map(|g| g.to_vec()) + .unwrap_or_default(); // or: .unwrap_or_else(|_| Vec::new()) + + Channel { + id: "beeg".to_string(), + name: "Beeg".to_string(), + description: "".to_string(), + premium: false, + favicon: "https://www.google.com/s2/favicons?sz=64&domain=beeg.com".to_string(), + status: "active".to_string(), + categories: vec![], + options: vec![ + ChannelOption { + id: "sites".to_string(), + title: "Sites".to_string(), + description: "Filter for different Sites".to_string(), + systemImage: "rectangle.stack".to_string(), + colorName: "green".to_string(), + options: sites, + multiSelect: false, + }, + ChannelOption { + id: "categories".to_string(), + title: "Categories".to_string(), + description: "Filter for different Networks".to_string(), + systemImage: "list.dash".to_string(), + colorName: "purple".to_string(), + options: categories, + multiSelect: false, + }, + ChannelOption { + id: "stars".to_string(), + title: "Stars".to_string(), + description: "Filter for different Pornstars".to_string(), + systemImage: "star.fill".to_string(), + colorName: "yellow".to_string(), + options: stars, + multiSelect: false, + }, + ], + nsfw: true, + cacheDuration: None, + } + } + + async fn get( + &self, + cache: VideoCache, + page: u8, + options: ServerOptions, + ) -> Result> { + let mut url = ""; + if options.categories.is_some() + && !options.categories.as_ref().unwrap().is_empty() + && options.categories.as_ref().unwrap() != "all" + { + url = options.categories.as_ref().unwrap(); + } + if options.sites.is_some() + && !options.sites.as_ref().unwrap().is_empty() + && options.sites.as_ref().unwrap() != "all" + { + url = options.sites.as_ref().unwrap(); + } + if options.stars.is_some() + && !options.stars.as_ref().unwrap().is_empty() + && options.stars.as_ref().unwrap() != "all" + { + url = options.stars.as_ref().unwrap(); + } + let video_url = format!( + "https://store.externulls.com/facts/tag?id=27173&limit=100&offset={}", + page - 1 + ); + let old_items = match cache.get(&video_url) { + Some((time, items)) => { + if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { + println!("Cache hit for URL: {}", video_url); + return Ok(items.clone()); + } else { + items.clone() + } + } + None => { + vec![] + } + }; + + let mut requester = options.requester.clone().unwrap(); + let text = requester.get(&video_url).await.unwrap(); + let json: serde_json::Value = serde_json::from_str::(&text).unwrap(); + let video_items: Vec = self.get_video_items_from_html(json.clone()); + if !video_items.is_empty() { + cache.remove(&video_url); + cache.insert(video_url.clone(), video_items.clone()); + } else { + return Ok(old_items); + } + Ok(video_items) + } + + async fn query( + &self, + cache: VideoCache, + page: u8, + query: &str, + options: ServerOptions, + ) -> Result> { + let video_url = format!( + "https://store.externulls.com/facts/tag?get_original=true&limit=100&offset={}&slug={}", + page - 1, + query.replace(" ", ""), + ); + // Check our Video Cache. If the result is younger than 1 hour, we return it. + let old_items = match cache.get(&video_url) { + Some((time, items)) => { + if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { + return Ok(items.clone()); + } else { + let _ = cache.check().await; + return Ok(items.clone()); + } + } + None => { + vec![] + } + }; + + let mut requester = options.requester.clone().unwrap(); + + let text = requester.get(&video_url).await.unwrap(); + let json: serde_json::Value = serde_json::from_str::(&text).unwrap(); + let video_items: Vec = self.get_video_items_from_html(json.clone()); + if !video_items.is_empty() { + cache.remove(&video_url); + cache.insert(video_url.clone(), video_items.clone()); + } else { + return Ok(old_items); + } + Ok(video_items) + } + + fn get_video_items_from_html(&self, json: Value) -> Vec { + let mut items: Vec = Vec::new(); + while let Some(video_items) = json.as_array() { + println!("video_items: {:?}\n\n\n", video_items); + break; + } + return items; + } +} + +#[async_trait] +impl Provider for BeegProvider { + async fn get_videos( + &self, + cache: VideoCache, + _pool: DbPool, + _sort: String, + query: Option, + page: String, + _per_page: String, + options: ServerOptions, + ) -> Vec { + let videos: std::result::Result, Error> = match query { + Some(q) => { + self.query(cache, page.parse::().unwrap_or(1), &q, options) + .await + } + None => { + self.get(cache, page.parse::().unwrap_or(1), options) + .await + } + }; + match videos { + Ok(v) => v, + Err(e) => { + println!("Error fetching videos: {}", e); + vec![] + } + } + } + fn get_channel(&self, clientversion: ClientVersion) -> crate::status::Channel { + self.build_channel(clientversion) + } +} diff --git a/src/providers/mod.rs b/src/providers/mod.rs index adf84aa..44307e8 100644 --- a/src/providers/mod.rs +++ b/src/providers/mod.rs @@ -6,7 +6,6 @@ use std::sync::Arc; use crate::{ DbPool, api::ClientVersion, - providers::omgxxx::OmgxxxProvider, status::Channel, util::cache::VideoCache, videos::{ServerOptions, VideoItem}, @@ -36,13 +35,15 @@ pub mod omgxxx; pub mod paradisehill; pub mod pornzog; pub mod youjizz; +pub mod beeg; // convenient alias pub type DynProvider = Arc; pub static ALL_PROVIDERS: Lazy> = Lazy::new(|| { let mut m = HashMap::default(); - m.insert("omgxxx", Arc::new(OmgxxxProvider::new()) as DynProvider); + m.insert("omgxxx", Arc::new(omgxxx::OmgxxxProvider::new()) as DynProvider); + m.insert("beeg", Arc::new(beeg::BeegProvider::new()) as DynProvider); // add more here as you migrate them m }); diff --git a/src/providers/omgxxx.rs b/src/providers/omgxxx.rs index 97bd562..76d47d9 100644 --- a/src/providers/omgxxx.rs +++ b/src/providers/omgxxx.rs @@ -10,8 +10,8 @@ use async_trait::async_trait; use error_chain::error_chain; use htmlentity::entity::{ICodedDataTrait, decode}; use std::sync::{Arc, RwLock}; +use std::thread; use std::vec; -use std::{thread}; error_chain! { foreign_links { @@ -91,11 +91,11 @@ impl OmgxxxProvider { break; } let stars_div = text - .split("id=\"list_models_models_list_items\"").collect::>()[1] - .split("class=\"pagination\"").collect::>()[0]; - for stars_element in - stars_div.split(">()[1..].to_vec() - { + .split("id=\"list_models_models_list_items\"") + .collect::>()[1] + .split("class=\"pagination\"") + .collect::>()[0]; + for stars_element in stars_div.split(">()[1..].to_vec() { let star_url = stars_element.split("href=\"").collect::>()[1] .split("\"") .collect::>()[0]; @@ -129,8 +129,10 @@ impl OmgxxxProvider { break; } let sites_div = text - .split("id=\"list_content_sources_sponsors_list_items\"").collect::>()[1] - .split("class=\"pagination\"").collect::>()[0]; + .split("id=\"list_content_sources_sponsors_list_items\"") + .collect::>()[1] + .split("class=\"pagination\"") + .collect::>()[0]; for sites_element in sites_div.split("class=\"headline\"").collect::>()[1..].to_vec() { @@ -301,19 +303,31 @@ impl OmgxxxProvider { && !options.network.as_ref().unwrap().is_empty() && options.network.as_ref().unwrap() != "all" { - sort_string = format!("networks/{}{}", options.network.as_ref().unwrap(), alt_sort_string); + sort_string = format!( + "networks/{}{}", + options.network.as_ref().unwrap(), + alt_sort_string + ); } if options.sites.is_some() && !options.sites.as_ref().unwrap().is_empty() && options.sites.as_ref().unwrap() != "all" { - sort_string = format!("sites/{}{}", options.sites.as_ref().unwrap(), alt_sort_string); + sort_string = format!( + "sites/{}{}", + options.sites.as_ref().unwrap(), + alt_sort_string + ); } if options.stars.is_some() && !options.stars.as_ref().unwrap().is_empty() && options.stars.as_ref().unwrap() != "all" { - sort_string = format!("models/{}{}", options.stars.as_ref().unwrap(), alt_sort_string); + sort_string = format!( + "models/{}{}", + options.stars.as_ref().unwrap(), + alt_sort_string + ); } let video_url = format!("{}/{}/{}/", self.url, sort_string, page); let old_items = match cache.get(&video_url) { @@ -350,18 +364,16 @@ impl OmgxxxProvider { options: ServerOptions, ) -> Result> { let mut search_type = "search"; + let mut search_string = query.to_string().to_lowercase().trim().replace(" ", "-"); if query.starts_with("@") { search_type = query.split(":").collect::>()[0].trim_start_matches("@"); + search_string = search_string.split(":").collect::>()[1].to_string(); } let video_url = format!( "{}/{}/{}/{}/", self.url, search_type, - query - .split(":").collect::>()[1] - .to_lowercase() - .trim() - .replace(" ", "-"), + search_string, page ); // Check our Video Cache. If the result is younger than 1 hour, we return it. @@ -395,7 +407,13 @@ impl OmgxxxProvider { fn get_site_id_from_name(&self, site_name: &str) -> Option { // site_name.to_lowercase().replace(" ", "") for site in self.sites.read().unwrap().iter() { - if site.title.to_lowercase().replace(" ", "").replace(".com", "") == site_name.to_lowercase().replace(" ", "") { + if site + .title + .to_lowercase() + .replace(" ", "") + .replace(".com", "") + == site_name.to_lowercase().replace(" ", "") + { return Some(site.id.clone()); } } @@ -479,9 +497,15 @@ impl OmgxxxProvider { .split("\"") .collect::>()[0] .to_string(); - let site_name = title.split("]").collect::>().first().unwrap_or(&"").trim_start_matches("["); - let site_id = self.get_site_id_from_name(site_name).unwrap_or("".to_string()); - println!("Site: {}\nTitle: {}", site_id, title); + let site_name = title + .split("]") + .collect::>() + .first() + .unwrap_or(&"") + .trim_start_matches("["); + let site_id = self + .get_site_id_from_name(site_name) + .unwrap_or("".to_string()); let mut tags = match video_segment.contains("class=\"models\">") { true => video_segment .split("class=\"models\">") diff --git a/src/videos.rs b/src/videos.rs index 3b9be40..d9b4b56 100644 --- a/src/videos.rs +++ b/src/videos.rs @@ -32,6 +32,7 @@ pub struct VideosRequest { pub language: Option, // pub networks: Option, // pub stars: Option, // + pub categories: Option, } #[derive(serde::Serialize, serde::Deserialize, Debug, Clone)] @@ -44,6 +45,7 @@ pub struct ServerOptions { pub requester: Option, pub network: Option, // pub stars: Option, // + pub categories: Option, // } #[derive(serde::Serialize, Debug)]