omgxxx bugfix

src/api.rs

@@ -1220,6 +1220,11 @@ async fn videos_post(
         .as_deref()
         .unwrap_or("")
         .to_string();
+    let categories = video_request
+        .categories
+        .as_deref()
+        .unwrap_or("")
+        .to_string();
     let options = ServerOptions {
         featured: Some(featured),
         category: Some(category),
@@ -1229,6 +1234,7 @@ async fn videos_post(
         requester: Some(requester),
         network: Some(network),
         stars: Some(stars),
+        categories: Some(categories),
     };
     let video_items = provider
         .get_videos(

src/providers/beeg.rs

@@ -0,0 +1,364 @@
+use crate::DbPool;
+use crate::api::ClientVersion;
+use crate::providers::Provider;
+use crate::schema::videos::star;
+use crate::util::cache::VideoCache;
+use crate::util::parse_abbreviated_number;
+use crate::util::time::parse_time_to_seconds;
+use crate::videos::{ServerOptions, VideoItem};
+use crate::{status::*, util};
+use async_trait::async_trait;
+use error_chain::error_chain;
+use htmlentity::entity::{ICodedDataTrait, decode};
+use serde_json::Value;
+use std::sync::{Arc, RwLock};
+use std::thread;
+use std::vec;
+
+error_chain! {
+    foreign_links {
+        Io(std::io::Error);
+        HttpRequest(wreq::Error);
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct BeegProvider {
+    url: String,
+    sites: Arc<RwLock<Vec<FilterOption>>>,
+    stars: Arc<RwLock<Vec<FilterOption>>>,
+    categories: Arc<RwLock<Vec<FilterOption>>>,
+}
+impl BeegProvider {
+    pub fn new() -> Self {
+        let provider = BeegProvider {
+            url: "https://beeg.com".to_string(),
+            sites: Arc::new(RwLock::new(vec![FilterOption {
+                id: "all".to_string(),
+                title: "All".to_string(),
+            }])),
+            stars: Arc::new(RwLock::new(vec![FilterOption {
+                id: "all".to_string(),
+                title: "All".to_string(),
+            }])),
+            categories: Arc::new(RwLock::new(vec![FilterOption {
+                id: "all".to_string(),
+                title: "All".to_string(),
+            }])),
+        };
+
+        // Kick off the background load but return immediately
+        provider.spawn_initial_load();
+        provider
+    }
+
+    fn spawn_initial_load(&self) {
+        let url = self.url.clone();
+        let sites = Arc::clone(&self.sites);
+        let categories = Arc::clone(&self.categories);
+        let stars = Arc::clone(&self.stars);
+
+        thread::spawn(move || {
+            // Create a tiny runtime just for these async tasks
+            let rt = tokio::runtime::Builder::new_current_thread()
+                .enable_all()
+                .build()
+                .expect("build tokio runtime");
+
+            rt.block_on(async move {
+                // If you have a streaming sites loader, call it here too
+                if let Err(e) = Self::load_sites(sites).await {
+                    eprintln!("beeg load_sites_into failed: {e}");
+                }
+
+                if let Err(e) = Self::load_categories(categories).await {
+                    eprintln!("beeg load_categories failed: {e}");
+                }
+
+                if let Err(e) = Self::load_stars(stars).await {
+                    eprintln!("beeg load_stars failed: {e}");
+                }
+            });
+        });
+    }
+
+    async fn load_stars(stars: Arc<RwLock<Vec<FilterOption>>>) -> Result<()> {
+        let mut requester = util::requester::Requester::new();
+        let text = requester
+            .get("https://store.externulls.com/tag/facts/tags?get_original=true&slug=index")
+            .await
+            .unwrap();
+        let json: serde_json::Value = serde_json::from_str::<serde_json::Value>(&text).unwrap();
+        let stars_array = json.get("human").unwrap().as_array().unwrap();
+        for s in stars_array {
+            let star_name = s.get("tg_name").unwrap().as_str().unwrap().to_string();
+            let star_id = s.get("tg_slug").unwrap().as_str().unwrap().to_string();
+            Self::push_unique(
+                &stars,
+                FilterOption {
+                    id: star_id,
+                    title: star_name,
+                },
+            );
+        }
+        return Ok(());
+    }
+
+    async fn load_categories(categories: Arc<RwLock<Vec<FilterOption>>>) -> Result<()> {
+        let mut requester = util::requester::Requester::new();
+        let text = requester
+            .get("https://store.externulls.com/tag/facts/tags?get_original=true&slug=index")
+            .await
+            .unwrap();
+        let json: serde_json::Value = serde_json::from_str::<serde_json::Value>(&text).unwrap();
+        let stars_array = json.get("other").unwrap().as_array().unwrap();
+        for s in stars_array {
+            let star_name = s.get("tg_name").unwrap().as_str().unwrap().to_string();
+            let star_id = s.get("tg_slug").unwrap().as_str().unwrap().to_string();
+            Self::push_unique(
+                &categories,
+                FilterOption {
+                    id: star_id,
+                    title: star_name,
+                },
+            );
+        }
+        return Ok(());
+    }
+
+    async fn load_sites(sites: Arc<RwLock<Vec<FilterOption>>>) -> Result<()> {
+        let mut requester = util::requester::Requester::new();
+        let text = requester
+            .get("https://store.externulls.com/tag/facts/tags?get_original=true&slug=index")
+            .await
+            .unwrap();
+        let json: serde_json::Value = serde_json::from_str::<serde_json::Value>(&text).unwrap();
+        let stars_array = json.get("productions").unwrap().as_array().unwrap();
+        for s in stars_array {
+            let star_name = s.get("tg_name").unwrap().as_str().unwrap().to_string();
+            let star_id = s.get("tg_slug").unwrap().as_str().unwrap().to_string();
+            Self::push_unique(
+                &sites,
+                FilterOption {
+                    id: star_id,
+                    title: star_name,
+                },
+            );
+        }
+        return Ok(());
+    }
+
+    // Push one item with minimal lock time and dedup by id
+    fn push_unique(target: &Arc<RwLock<Vec<FilterOption>>>, item: FilterOption) {
+        if let Ok(mut vec) = target.write() {
+            if !vec.iter().any(|x| x.id == item.id) {
+                vec.push(item);
+                // Optional: keep it sorted for nicer UX
+                // vec.sort_by(|a,b| a.title.cmp(&b.title));
+            }
+        }
+    }
+
+    fn build_channel(&self, clientversion: ClientVersion) -> Channel {
+        let _ = clientversion;
+        let sites: Vec<FilterOption> = self
+            .sites
+            .read()
+            .map(|g| g.clone()) // or: .map(|g| g.to_vec())
+            .unwrap_or_default(); // or: .unwrap_or_else(|_| Vec::new())
+
+        let categories: Vec<FilterOption> = self
+            .categories
+            .read()
+            .map(|g| g.clone()) // or: .map(|g| g.to_vec())
+            .unwrap_or_default(); // or: .unwrap_or_else(|_| Vec::new())
+
+        let stars: Vec<FilterOption> = self
+            .stars
+            .read()
+            .map(|g| g.clone()) // or: .map(|g| g.to_vec())
+            .unwrap_or_default(); // or: .unwrap_or_else(|_| Vec::new())
+
+        Channel {
+            id: "beeg".to_string(),
+            name: "Beeg".to_string(),
+            description: "".to_string(),
+            premium: false,
+            favicon: "https://www.google.com/s2/favicons?sz=64&domain=beeg.com".to_string(),
+            status: "active".to_string(),
+            categories: vec![],
+            options: vec![
+                ChannelOption {
+                    id: "sites".to_string(),
+                    title: "Sites".to_string(),
+                    description: "Filter for different Sites".to_string(),
+                    systemImage: "rectangle.stack".to_string(),
+                    colorName: "green".to_string(),
+                    options: sites,
+                    multiSelect: false,
+                },
+                ChannelOption {
+                    id: "categories".to_string(),
+                    title: "Categories".to_string(),
+                    description: "Filter for different Networks".to_string(),
+                    systemImage: "list.dash".to_string(),
+                    colorName: "purple".to_string(),
+                    options: categories,
+                    multiSelect: false,
+                },
+                ChannelOption {
+                    id: "stars".to_string(),
+                    title: "Stars".to_string(),
+                    description: "Filter for different Pornstars".to_string(),
+                    systemImage: "star.fill".to_string(),
+                    colorName: "yellow".to_string(),
+                    options: stars,
+                    multiSelect: false,
+                },
+            ],
+            nsfw: true,
+            cacheDuration: None,
+        }
+    }
+
+    async fn get(
+        &self,
+        cache: VideoCache,
+        page: u8,
+        options: ServerOptions,
+    ) -> Result<Vec<VideoItem>> {
+        let mut url = "";
+        if options.categories.is_some()
+            && !options.categories.as_ref().unwrap().is_empty()
+            && options.categories.as_ref().unwrap() != "all"
+        {
+            url = options.categories.as_ref().unwrap();
+        }
+        if options.sites.is_some()
+            && !options.sites.as_ref().unwrap().is_empty()
+            && options.sites.as_ref().unwrap() != "all"
+        {
+            url = options.sites.as_ref().unwrap();
+        }
+        if options.stars.is_some()
+            && !options.stars.as_ref().unwrap().is_empty()
+            && options.stars.as_ref().unwrap() != "all"
+        {
+            url = options.stars.as_ref().unwrap();
+        }
+        let video_url = format!(
+            "https://store.externulls.com/facts/tag?id=27173&limit=100&offset={}",
+            page - 1
+        );
+        let old_items = match cache.get(&video_url) {
+            Some((time, items)) => {
+                if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
+                    println!("Cache hit for URL: {}", video_url);
+                    return Ok(items.clone());
+                } else {
+                    items.clone()
+                }
+            }
+            None => {
+                vec![]
+            }
+        };
+
+        let mut requester = options.requester.clone().unwrap();
+        let text = requester.get(&video_url).await.unwrap();
+        let json: serde_json::Value = serde_json::from_str::<serde_json::Value>(&text).unwrap();
+        let video_items: Vec<VideoItem> = self.get_video_items_from_html(json.clone());
+        if !video_items.is_empty() {
+            cache.remove(&video_url);
+            cache.insert(video_url.clone(), video_items.clone());
+        } else {
+            return Ok(old_items);
+        }
+        Ok(video_items)
+    }
+
+    async fn query(
+        &self,
+        cache: VideoCache,
+        page: u8,
+        query: &str,
+        options: ServerOptions,
+    ) -> Result<Vec<VideoItem>> {
+        let video_url = format!(
+            "https://store.externulls.com/facts/tag?get_original=true&limit=100&offset={}&slug={}",
+            page - 1,
+            query.replace(" ", ""),
+        );
+        // Check our Video Cache. If the result is younger than 1 hour, we return it.
+        let old_items = match cache.get(&video_url) {
+            Some((time, items)) => {
+                if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
+                    return Ok(items.clone());
+                } else {
+                    let _ = cache.check().await;
+                    return Ok(items.clone());
+                }
+            }
+            None => {
+                vec![]
+            }
+        };
+
+        let mut requester = options.requester.clone().unwrap();
+
+        let text = requester.get(&video_url).await.unwrap();
+        let json: serde_json::Value = serde_json::from_str::<serde_json::Value>(&text).unwrap();
+        let video_items: Vec<VideoItem> = self.get_video_items_from_html(json.clone());
+        if !video_items.is_empty() {
+            cache.remove(&video_url);
+            cache.insert(video_url.clone(), video_items.clone());
+        } else {
+            return Ok(old_items);
+        }
+        Ok(video_items)
+    }
+
+    fn get_video_items_from_html(&self, json: Value) -> Vec<VideoItem> {
+        let mut items: Vec<VideoItem> = Vec::new();
+        while let Some(video_items) = json.as_array() {
+            println!("video_items: {:?}\n\n\n", video_items);
+            break;
+        }
+        return items;
+    }
+}
+
+#[async_trait]
+impl Provider for BeegProvider {
+    async fn get_videos(
+        &self,
+        cache: VideoCache,
+        _pool: DbPool,
+        _sort: String,
+        query: Option<String>,
+        page: String,
+        _per_page: String,
+        options: ServerOptions,
+    ) -> Vec<VideoItem> {
+        let videos: std::result::Result<Vec<VideoItem>, Error> = match query {
+            Some(q) => {
+                self.query(cache, page.parse::<u8>().unwrap_or(1), &q, options)
+                    .await
+            }
+            None => {
+                self.get(cache, page.parse::<u8>().unwrap_or(1), options)
+                    .await
+            }
+        };
+        match videos {
+            Ok(v) => v,
+            Err(e) => {
+                println!("Error fetching videos: {}", e);
+                vec![]
+            }
+        }
+    }
+    fn get_channel(&self, clientversion: ClientVersion) -> crate::status::Channel {
+        self.build_channel(clientversion)
+    }
+}

src/providers/mod.rs

@@ -6,7 +6,6 @@ use std::sync::Arc;
 use crate::{
     DbPool,
     api::ClientVersion,
-    providers::omgxxx::OmgxxxProvider,
     status::Channel,
     util::cache::VideoCache,
     videos::{ServerOptions, VideoItem},
@@ -36,13 +35,15 @@ pub mod omgxxx;
 pub mod paradisehill;
 pub mod pornzog;
 pub mod youjizz;
+pub mod beeg;
 
 // convenient alias
 pub type DynProvider = Arc<dyn Provider>;
 
 pub static ALL_PROVIDERS: Lazy<HashMap<&'static str, DynProvider>> = Lazy::new(|| {
     let mut m = HashMap::default();
-    m.insert("omgxxx", Arc::new(OmgxxxProvider::new()) as DynProvider);
+    m.insert("omgxxx", Arc::new(omgxxx::OmgxxxProvider::new()) as DynProvider);
+    m.insert("beeg", Arc::new(beeg::BeegProvider::new()) as DynProvider);
     // add more here as you migrate them
     m
 });

src/providers/omgxxx.rs

@@ -10,8 +10,8 @@ use async_trait::async_trait;
 use error_chain::error_chain;
 use htmlentity::entity::{ICodedDataTrait, decode};
 use std::sync::{Arc, RwLock};
+use std::thread;
 use std::vec;
-use std::{thread};
 
 error_chain! {
     foreign_links {
@@ -91,11 +91,11 @@ impl OmgxxxProvider {
                 break;
             }
             let stars_div = text
-                .split("id=\"list_models_models_list_items\"").collect::<Vec<&str>>()[1]
+                .split("id=\"list_models_models_list_items\"")
-                .split("class=\"pagination\"").collect::<Vec<&str>>()[0];
+                .collect::<Vec<&str>>()[1]
-            for stars_element in
+                .split("class=\"pagination\"")
-                stars_div.split("<a ").collect::<Vec<&str>>()[1..].to_vec()
+                .collect::<Vec<&str>>()[0];
-            {
+            for stars_element in stars_div.split("<a ").collect::<Vec<&str>>()[1..].to_vec() {
                 let star_url = stars_element.split("href=\"").collect::<Vec<&str>>()[1]
                     .split("\"")
                     .collect::<Vec<&str>>()[0];
@@ -129,8 +129,10 @@ impl OmgxxxProvider {
                 break;
             }
             let sites_div = text
-                .split("id=\"list_content_sources_sponsors_list_items\"").collect::<Vec<&str>>()[1]
+                .split("id=\"list_content_sources_sponsors_list_items\"")
-                .split("class=\"pagination\"").collect::<Vec<&str>>()[0];
+                .collect::<Vec<&str>>()[1]
+                .split("class=\"pagination\"")
+                .collect::<Vec<&str>>()[0];
             for sites_element in
                 sites_div.split("class=\"headline\"").collect::<Vec<&str>>()[1..].to_vec()
             {
@@ -301,19 +303,31 @@ impl OmgxxxProvider {
             && !options.network.as_ref().unwrap().is_empty()
             && options.network.as_ref().unwrap() != "all"
         {
-            sort_string = format!("networks/{}{}", options.network.as_ref().unwrap(), alt_sort_string);
+            sort_string = format!(
+                "networks/{}{}",
+                options.network.as_ref().unwrap(),
+                alt_sort_string
+            );
         }
         if options.sites.is_some()
             && !options.sites.as_ref().unwrap().is_empty()
             && options.sites.as_ref().unwrap() != "all"
         {
-            sort_string = format!("sites/{}{}", options.sites.as_ref().unwrap(), alt_sort_string);
+            sort_string = format!(
+                "sites/{}{}",
+                options.sites.as_ref().unwrap(),
+                alt_sort_string
+            );
         }
         if options.stars.is_some()
             && !options.stars.as_ref().unwrap().is_empty()
             && options.stars.as_ref().unwrap() != "all"
         {
-            sort_string = format!("models/{}{}", options.stars.as_ref().unwrap(), alt_sort_string);
+            sort_string = format!(
+                "models/{}{}",
+                options.stars.as_ref().unwrap(),
+                alt_sort_string
+            );
         }
         let video_url = format!("{}/{}/{}/", self.url, sort_string, page);
         let old_items = match cache.get(&video_url) {
@@ -350,18 +364,16 @@ impl OmgxxxProvider {
         options: ServerOptions,
     ) -> Result<Vec<VideoItem>> {
         let mut search_type = "search";
+        let mut search_string = query.to_string().to_lowercase().trim().replace(" ", "-");
         if query.starts_with("@") {
             search_type = query.split(":").collect::<Vec<&str>>()[0].trim_start_matches("@");
+            search_string = search_string.split(":").collect::<Vec<&str>>()[1].to_string();
         }
         let video_url = format!(
             "{}/{}/{}/{}/",
             self.url,
             search_type,
-            query
+            search_string,
-                .split(":").collect::<Vec<&str>>()[1]
-                .to_lowercase()
-                .trim()
-                .replace(" ", "-"),
             page
         );
         // Check our Video Cache. If the result is younger than 1 hour, we return it.
@@ -395,7 +407,13 @@ impl OmgxxxProvider {
     fn get_site_id_from_name(&self, site_name: &str) -> Option<String> {
         // site_name.to_lowercase().replace(" ", "")
         for site in self.sites.read().unwrap().iter() {
-            if site.title.to_lowercase().replace(" ", "").replace(".com", "") == site_name.to_lowercase().replace(" ", "") {
+            if site
+                .title
+                .to_lowercase()
+                .replace(" ", "")
+                .replace(".com", "")
+                == site_name.to_lowercase().replace(" ", "")
+            {
                 return Some(site.id.clone());
             }
         }
@@ -479,9 +497,15 @@ impl OmgxxxProvider {
                 .split("\"")
                 .collect::<Vec<&str>>()[0]
                 .to_string();
-            let site_name = title.split("]").collect::<Vec<&str>>().first().unwrap_or(&"").trim_start_matches("[");
+            let site_name = title
-            let site_id = self.get_site_id_from_name(site_name).unwrap_or("".to_string());
+                .split("]")
-            println!("Site: {}\nTitle: {}", site_id, title);
+                .collect::<Vec<&str>>()
+                .first()
+                .unwrap_or(&"")
+                .trim_start_matches("[");
+            let site_id = self
+                .get_site_id_from_name(site_name)
+                .unwrap_or("".to_string());
             let mut tags = match video_segment.contains("class=\"models\">") {
                 true => video_segment
                     .split("class=\"models\">")

src/videos.rs

@@ -32,6 +32,7 @@ pub struct VideosRequest {
     pub language: Option<String>, //
     pub networks: Option<String>, //
     pub stars: Option<String>, //
+    pub categories: Option<String>,
 }
 
 #[derive(serde::Serialize, serde::Deserialize, Debug, Clone)]
@@ -44,6 +45,7 @@ pub struct ServerOptions {
     pub requester: Option<Requester>,
     pub network: Option<String>, //
     pub stars: Option<String>, //
+    pub categories: Option<String>, //
 }
 
 #[derive(serde::Serialize, Debug)]