hentaihaven

archive/tube8.rs

@@ -0,0 +1,448 @@
+use crate::DbPool;
+use crate::api::ClientVersion;
+use crate::providers::Provider;
+use crate::status::*;
+use crate::util::cache::VideoCache;
+use crate::util::parse_abbreviated_number;
+use crate::util::time::parse_time_to_seconds;
+use crate::videos::{ServerOptions, VideoItem};
+use async_trait::async_trait;
+use error_chain::error_chain;
+use htmlentity::entity::{ICodedDataTrait, decode};
+use std::sync::{Arc, RwLock};
+use std::vec;
+
+error_chain! {
+    foreign_links {
+        Io(std::io::Error);
+        HttpRequest(wreq::Error);
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct Tube8Provider {
+    url: String,
+    sites: Arc<RwLock<Vec<FilterOption>>>,
+    stars: Arc<RwLock<Vec<FilterOption>>>,
+}
+impl Tube8Provider {
+    pub fn new() -> Self {
+        let provider = Tube8Provider {
+            url: "https://www.tube8.com".to_string(),
+            sites: Arc::new(RwLock::new(vec![FilterOption {
+                id: "all".to_string(),
+                title: "All".to_string(),
+            }])),
+            stars: Arc::new(RwLock::new(vec![FilterOption {
+                id: "all".to_string(),
+                title: "All".to_string(),
+            }])),
+        };
+
+        // Kick off the background load but return immediately
+        provider
+    }
+
+    // Push one item with minimal lock time and dedup by id
+    fn push_unique(target: &Arc<RwLock<Vec<FilterOption>>>, item: FilterOption) {
+        if let Ok(mut vec) = target.write() {
+            if !vec.iter().any(|x| x.id == item.id) {
+                vec.push(item);
+                // Optional: keep it sorted for nicer UX
+                // vec.sort_by(|a,b| a.title.cmp(&b.title));
+            }
+        }
+    }
+
+    fn build_channel(&self, clientversion: ClientVersion) -> Channel {
+        let _ = clientversion;
+        let sites: Vec<FilterOption> = self
+            .sites
+            .read()
+            .map(|g| g.clone()) // or: .map(|g| g.to_vec())
+            .unwrap_or_default(); // or: .unwrap_or_else(|_| Vec::new())
+
+        let stars: Vec<FilterOption> = self
+            .stars
+            .read()
+            .map(|g| g.clone()) // or: .map(|g| g.to_vec())
+            .unwrap_or_default(); // or: .unwrap_or_else(|_| Vec::new())
+
+        Channel {
+            id: "tube8".to_string(),
+            name: "Tube8".to_string(),
+            description: "Tube8 Videos".to_string(),
+            premium: false,
+            favicon: "https://www.google.com/s2/favicons?sz=64&domain=www.tube8.com".to_string(),
+            status: "active".to_string(),
+            categories: vec![],
+            options: vec![
+                ChannelOption {
+                    id: "sort".to_string(),
+                    title: "Sort".to_string(),
+                    description: "Sort the Videos".to_string(),
+                    systemImage: "list.number".to_string(),
+                    colorName: "blue".to_string(),
+                    options: vec![
+                        FilterOption {
+                            id: "rating".into(),
+                            title: "Rating".into(),
+                        },
+                        FilterOption {
+                            id: "mostviewed.html".into(),
+                            title: "Most Viewed".into(),
+                        },
+                        FilterOption {
+                            id: "longest.html".into(),
+                            title: "Duration".into(),
+                        },
+                        FilterOption {
+                            id: "newest.html".into(),
+                            title: "Newest".into(),
+                        },
+                    ],
+                    multiSelect: false,
+                },
+                ChannelOption {
+                    id: "sites".to_string(),
+                    title: "Sites".to_string(),
+                    description: "Filter for different Sites".to_string(),
+                    systemImage: "rectangle.stack".to_string(),
+                    colorName: "green".to_string(),
+                    options: sites,
+                    multiSelect: false,
+                },
+                ChannelOption {
+                    id: "stars".to_string(),
+                    title: "Stars".to_string(),
+                    description: "Filter for different Pornstars".to_string(),
+                    systemImage: "star.fill".to_string(),
+                    colorName: "yellow".to_string(),
+                    options: stars,
+                    multiSelect: false,
+                },
+            ],
+            nsfw: true,
+            cacheDuration: None,
+        }
+    }
+
+    async fn get(
+        &self,
+        cache: VideoCache,
+        page: u8,
+        sort: &str,
+        options: ServerOptions,
+    ) -> Result<Vec<VideoItem>> {
+        let mut sort_string: String = match sort {
+            "mostviewed.html" => "mostviewed.html/".to_string(),
+            "longest.html" => "longest.html/".to_string(),
+            "newest.html" => "newest.html/".to_string(),
+            _ => "".to_string(),
+        };
+        if options.sites.is_some()
+            && !options.sites.as_ref().unwrap().is_empty()
+            && options.sites.as_ref().unwrap() != "all"
+        {
+            sort_string = match sort {
+                "mostviewed.html" => "?orderBy=mv&page=".to_string(),
+                "longest.html" => "?orderBy=ln&page=".to_string(),
+                "newest.html" => "?page=".to_string(),
+                _ => "?orderBy=tr&page=".to_string(),
+            };
+        }
+        if options.stars.is_some()
+            && !options.stars.as_ref().unwrap().is_empty()
+            && options.stars.as_ref().unwrap() != "all"
+        {
+            sort_string = match sort {
+                "mostviewed.html" => "views/?page=".to_string(),
+                "longest.html" => "duration/?page=".to_string(),
+                "newest.html" => "?page=".to_string(),
+                _ => "rating/?page=".to_string(),
+            };
+        }
+        let video_url = format!("{}/{}{}", self.url, sort_string, page);
+        let old_items = match cache.get(&video_url) {
+            Some((time, items)) => {
+                if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
+                    return Ok(items.clone());
+                } else {
+                    items.clone()
+                }
+            }
+            None => {
+                vec![]
+            }
+        };
+
+        let mut requester = options.requester.clone().unwrap();
+        let text = requester.get(&video_url, None).await.unwrap();
+        let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
+        if !video_items.is_empty() {
+            cache.remove(&video_url);
+            cache.insert(video_url.clone(), video_items.clone());
+        } else {
+            return Ok(old_items);
+        }
+        Ok(video_items)
+    }
+
+    async fn query(
+        &self,
+        cache: VideoCache,
+        page: u8,
+        query: &str,
+        options: ServerOptions,
+    ) -> Result<Vec<VideoItem>> {
+        let mut sort_string: String = match options.sort.as_ref().unwrap().as_str() {
+            "mostviewed.html" => "&orderby=views&page=".to_string(),
+            "longest.html" => "&orderby=longest&page=".to_string(),
+            "newest.html" => "&orderby=newest&page=".to_string(),
+            _ => "&orderby=rating&page=".to_string(),
+        };
+        let mut search_string = query.to_string().to_ascii_lowercase().trim().to_string();
+        let mut video_url = format!(
+            "{}/searches.html/?q={}{}{}",
+            self.url, query, sort_string, page
+        );
+        video_url = video_url.replace(" ", "+");
+        match self
+            .stars
+            .read()
+            .unwrap()
+            .iter()
+            .find(|s| s.title.to_ascii_lowercase() == search_string)
+        {
+            Some(star) => {
+                sort_string = match options.sort.as_ref().unwrap().as_str() {
+                    "mostviewed.html" => "views/?page=".to_string(),
+                    "longest.html" => "duration/?page=".to_string(),
+                    "newest.html" => "?page=".to_string(),
+                    _ => "rating/?page=".to_string(),
+                };
+                video_url = format!("{}/{}{}{}", self.url, star.id, sort_string, page);
+            }
+            _ => {}
+        }
+        match self
+            .sites
+            .read()
+            .unwrap()
+            .iter()
+            .find(|s| s.title.to_ascii_lowercase() == search_string)
+        {
+            Some(site) => {
+                sort_string = match options.sort.as_ref().unwrap().as_str() {
+                    "mostviewed.html" => "?orderBy=mv&page=".to_string(),
+                    "longest.html" => "?orderBy=ln&page=".to_string(),
+                    "newest.html" => "?page=".to_string(),
+                    _ => "?orderBy=tr&page=".to_string(),
+                };
+                video_url = format!("{}/{}{}{}", self.url, site.id, sort_string, page);
+            }
+            _ => {}
+        }
+        // Check our Video Cache. If the result is younger than 1 hour, we return it.
+        let old_items = match cache.get(&video_url) {
+            Some((time, items)) => {
+                if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
+                    return Ok(items.clone());
+                } else {
+                    let _ = cache.check().await;
+                    return Ok(items.clone());
+                }
+            }
+            None => {
+                vec![]
+            }
+        };
+
+        let mut requester = options.requester.clone().unwrap();
+
+        let text = requester.get(&video_url, None).await.unwrap();
+        let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
+        if !video_items.is_empty() {
+            cache.remove(&video_url);
+            cache.insert(video_url.clone(), video_items.clone());
+        } else {
+            return Ok(old_items);
+        }
+        Ok(video_items)
+    }
+
+    fn get_video_items_from_html(&self, html: String) -> Vec<VideoItem> {
+        if html.is_empty() {
+            println!("HTML is empty");
+            return vec![];
+        }
+        let mut items: Vec<VideoItem> = Vec::new();
+        if !html.contains("video-box ") {
+            return items;
+        }
+        let raw_videos = html.split("id=\"pagination\"").collect::<Vec<&str>>()[0]
+            .split("-thumbs")
+            .collect::<Vec<&str>>()[1]
+            .split("video-box ")
+            .collect::<Vec<&str>>()[1..]
+            .to_vec();
+        for video_segment in &raw_videos {
+            // let vid = video_segment.split("\n").collect::<Vec<&str>>();
+            // for (index, line) in vid.iter().enumerate() {
+            //     println!("Line {}: {}", index, line);
+            // }
+            let video_url: String = format!("{}{}", self.url, video_segment.split("<a href=\"").collect::<Vec<&str>>()[1]
+                .split("\"")
+                .collect::<Vec<&str>>()[0]
+                .to_string());
+            let mut title = video_segment.split("alt=\"").collect::<Vec<&str>>()[1]
+                .split("\"")
+                .collect::<Vec<&str>>()[0]
+                .to_string();
+            // html decode
+            title = decode(title.as_bytes()).to_string().unwrap_or(title);
+            let id = video_url.split("/").collect::<Vec<&str>>()[4].to_string();
+
+            let thumb = match video_segment.split("thumb-image ").collect::<Vec<&str>>()[1]
+                .contains("data-src=\"")
+            {
+                true => video_segment.split("thumb-image ").collect::<Vec<&str>>()[1]
+                    .split("data-src=\"")
+                    .collect::<Vec<&str>>()[1]
+                    .split("\"")
+                    .collect::<Vec<&str>>()[0]
+                    .to_string(),
+                false => video_segment.split("thumb-image ").collect::<Vec<&str>>()[1]
+                    .split("data-poster=\"")
+                    .collect::<Vec<&str>>()[1]
+                    .split("\"")
+                    .collect::<Vec<&str>>()[0]
+                    .to_string(),
+            };
+            let raw_duration = video_segment
+                .split("video-duration ")
+                .collect::<Vec<&str>>()[1]
+                .split("</span>")
+                .collect::<Vec<&str>>()[0]
+                .split("<span>")
+                .collect::<Vec<&str>>()
+                .last()
+                .unwrap_or(&"")
+                .replace("\n", "")
+                .trim()
+                .to_string();
+            let duration = parse_time_to_seconds(raw_duration.as_str()).unwrap_or(0) as u32;
+            let views = parse_abbreviated_number(
+                video_segment
+                    .split("<span class='info-views'>")
+                    .collect::<Vec<&str>>()[1]
+                    .split("<")
+                    .collect::<Vec<&str>>()[0]
+                    .to_string()
+                    .as_str(),
+            )
+            .unwrap_or(0) as u32;
+            let site_name = title
+                .split("]")
+                .collect::<Vec<&str>>()
+                .first()
+                .unwrap_or(&"")
+                .trim_start_matches("[");
+            let site_id = self
+                .get_site_id_from_name(site_name)
+                .unwrap_or("".to_string());
+            let mut tags = match video_segment.contains("class=\"models\">") {
+                true => video_segment
+                    .split("class=\"models\">")
+                    .collect::<Vec<&str>>()[1]
+                    .split("</div>")
+                    .collect::<Vec<&str>>()[0]
+                    .split("href=\"")
+                    .collect::<Vec<&str>>()[1..]
+                    .into_iter()
+                    .map(|s| {
+                        Self::push_unique(
+                            &self.stars,
+                            FilterOption {
+                                id: s.split("/").collect::<Vec<&str>>()[4].to_string(),
+                                title: s.split(">").collect::<Vec<&str>>()[1]
+                                    .split("<")
+                                    .collect::<Vec<&str>>()[0]
+                                    .trim()
+                                    .to_string(),
+                            },
+                        );
+                        s.split(">").collect::<Vec<&str>>()[1]
+                            .split("<")
+                            .collect::<Vec<&str>>()[0]
+                            .trim()
+                            .to_string()
+                    })
+                    .collect::<Vec<String>>()
+                    .to_vec(),
+                false => vec![],
+            };
+            if !site_id.is_empty() {
+                Self::push_unique(
+                    &self.sites,
+                    FilterOption {
+                        id: site_id,
+                        title: site_name.to_string(),
+                    },
+                );
+                tags.push(site_name.to_string());
+            }
+
+            let video_item = VideoItem::new(
+                id,
+                title,
+                video_url.to_string(),
+                "omgxxx".to_string(),
+                thumb,
+                duration,
+            )
+            .views(views)
+            .preview(preview)
+            .tags(tags);
+            items.push(video_item);
+        }
+        return items;
+    }
+}
+
+#[async_trait]
+impl Provider for Tube8Provider {
+    async fn get_videos(
+        &self,
+        cache: VideoCache,
+        pool: DbPool,
+        sort: String,
+        query: Option<String>,
+        page: String,
+        per_page: String,
+        options: ServerOptions,
+    ) -> Vec<VideoItem> {
+        let _ = per_page;
+        let _ = pool;
+        let videos: std::result::Result<Vec<VideoItem>, Error> = match query {
+            Some(q) => {
+                self.query(cache, page.parse::<u8>().unwrap_or(1), &q, options)
+                    .await
+            }
+            None => {
+                self.get(cache, page.parse::<u8>().unwrap_or(1), &sort, options)
+                    .await
+            }
+        };
+        match videos {
+            Ok(v) => v,
+            Err(e) => {
+                println!("Error fetching videos: {}", e);
+                vec![]
+            }
+        }
+    }
+    fn get_channel(&self, clientversion: ClientVersion) -> Option<crate::status::Channel> {
+        Some(self.build_channel(clientversion))
+    }
+}

src/providers/hentaihaven.rs

@@ -1,62 +1,93 @@
 use crate::api::ClientVersion;
 use crate::providers::Provider;
-use crate::schema::videos::url;
+use crate::status::*;
-use crate::status::{Channel, ChannelOption, FilterOption};
 use crate::util::cache::VideoCache;
-use crate::util::flaresolverr::{FlareSolverrRequest, Flaresolverr};
+use crate::util::discord::{format_error_chain, send_discord_error_report};
-use crate::util::parse_abbreviated_number;
+use crate::util::requester::Requester;
-use crate::util::time::parse_time_to_seconds;
+use crate::videos::{ServerOptions, VideoFormat, VideoItem};
-use crate::videos::{ServerOptions, VideoItem};
+use crate::{DbPool, db};
-use crate::{DbPool, videos};
+
 use async_trait::async_trait;
 use error_chain::error_chain;
+use futures::future::join_all;
 use htmlentity::entity::{ICodedDataTrait, decode};
-use std::env;
+use std::sync::{Arc, RwLock};
 use std::vec;
-use wreq::{Client, Proxy};
+use titlecase::Titlecase;
-use wreq_util::Emulation;
+use wreq::Version;
 
 error_chain! {
     foreign_links {
         Io(std::io::Error);
         HttpRequest(wreq::Error);
+        Json(serde_json::Error);
+    }
+    errors {
+        Parse(msg: String) {
+            description("parse error")
+            display("parse error: {}", msg)
+        }
     }
 }
 
 #[derive(Debug, Clone)]
 pub struct HentaihavenProvider {
     url: String,
+    categories: Arc<RwLock<Vec<FilterOption>>>,
 }
+
 impl HentaihavenProvider {
     pub fn new() -> Self {
-        HentaihavenProvider {
+        let provider = Self {
             url: "https://hentaihaven.xxx".to_string(),
-        }
+            categories: Arc::new(RwLock::new(vec![])),
+        };
+        provider
     }
 
     fn build_channel(&self, clientversion: ClientVersion) -> Channel {
         let _ = clientversion;
-
         Channel {
             id: "hentaihaven".to_string(),
-            name: "WORK IN PROGRESS Hentai Haven".to_string(),
+            name: "Hentai Haven".to_string(),
-            description: "".to_string(),
+            description: "Watch Free Hentai Videos HD!".to_string(),
             premium: false,
             favicon: "https://www.google.com/s2/favicons?sz=64&domain=hentaihaven.xxx".to_string(),
             status: "active".to_string(),
-            categories: vec![],
+            categories: self
+                .categories
+                .read()
+                .unwrap()
+                .iter()
+                .map(|c| c.title.clone())
+                .collect(),
             options: vec![],
             nsfw: true,
             cacheDuration: None,
         }
     }
 
-    async fn get(&self, cache: VideoCache, page: u8, sort: &str) -> Result<Vec<VideoItem>> {
+    fn push_unique(target: &Arc<RwLock<Vec<FilterOption>>>, item: FilterOption) {
-        let video_url = format!("{}/page/{}/?m_orderby={}", self.url, page, sort);
+        if let Ok(mut vec) = target.write() {
+            if !vec.iter().any(|x| x.id == item.id) {
+                vec.push(item);
+            }
+        }
+    }
+
+    async fn get(
+        &self,
+        cache: VideoCache,
+        page: u8,
+        sort: &str,
+        options: ServerOptions,
+        pool: DbPool,
+    ) -> Result<Vec<VideoItem>> {
+        let _ = sort;
+        let video_url = format!("{}/hentai/page/{}/", self.url, page);
         let old_items = match cache.get(&video_url) {
             Some((time, items)) => {
-                if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
+                if time.elapsed().unwrap_or_default().as_secs() < 60 * 60 * 24 {
-                    println!("Cache hit for URL: {}", video_url);
                     return Ok(items.clone());
                 } else {
                     items.clone()
@@ -67,58 +98,22 @@ impl HentaihavenProvider {
             }
         };
 
-        let proxy = Proxy::all("http://192.168.0.103:8081").unwrap();
+        let mut requester = options.requester.clone().unwrap();
-        let client = Client::builder()
+        let text = requester
-            .cert_verification(false)
+            .get(&video_url, Some(Version::HTTP_2))
-            .emulation(Emulation::Firefox136)
+            .await
-            .build()?;
+            .unwrap();
-
+        if page > 1
-        let mut response = client
+            && !text.contains(&format!(
-            .get(video_url.clone())
+                "<li class=\"page-item active\"><span class=\"page-link\">{}</span>",
-            .proxy(proxy.clone())
+                page
-            .send()
+            ))
-            .await?;
+        {
-        if response.status().is_redirection() {
+            return Ok(vec![]);
-            println!(
-                "Redirection detected, following to: {}",
-                response.headers()["Location"].to_str().unwrap()
-            );
-            response = client
-                .get(response.headers()["Location"].to_str().unwrap())
-                // .proxy(proxy)
-                .send()
-                .await?;
         }
-        if response.status().is_success() {
+        let video_items: Vec<VideoItem> = self
-            let text = response.text().await?;
+            .get_video_items_from_html(text.clone(), &mut requester, pool.clone())
-            let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
-            if !video_items.is_empty() {
-                cache.remove(&video_url);
-                cache.insert(video_url.clone(), video_items.clone());
-            } else {
-                return Ok(old_items);
-            }
-            Ok(video_items)
-        } else {
-            let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set");
-            let flare = Flaresolverr::new(flare_url);
-            let result = flare
-                .solve(FlareSolverrRequest {
-                    cmd: "request.get".to_string(),
-                    url: video_url.clone(),
-                    maxTimeout: 60000,
-                })
             .await;
-            let video_items = match result {
-                Ok(res) => {
-                    // println!("FlareSolverr response: {}", res);
-                    self.get_video_items_from_html(res.solution.response)
-                }
-                Err(e) => {
-                    println!("Error solving FlareSolverr: {}", e);
-                    return Err("Failed to solve FlareSolverr".into());
-                }
-            };
         if !video_items.is_empty() {
             cache.remove(&video_url);
             cache.insert(video_url.clone(), video_items.clone());
@@ -127,19 +122,24 @@ impl HentaihavenProvider {
         }
         Ok(video_items)
     }
-    }
-    async fn query(&self, cache: VideoCache, page: u8, query: &str) -> Result<Vec<VideoItem>> {
-        let search_string = query.to_lowercase().trim().replace(" ", "-");
-        let mut video_url = format!("{}/search/{}/{}/", self.url, search_string, page);
 
-        if search_string.starts_with("@") {
+    async fn query(
-            let url_part = search_string.split("@").collect::<Vec<&str>>()[1].replace(":", "/");
+        &self,
-            video_url = format!("{}/{}/", self.url, url_part);
+        cache: VideoCache,
-        }
+        page: u8,
+        query: &str,
+        options: ServerOptions,
+        pool: DbPool,
+    ) -> Result<Vec<VideoItem>> {
+        let video_url = format!(
+            "{}/?s={}",
+            self.url,
+            query.replace(" ", "+"),
+        );
         // Check our Video Cache. If the result is younger than 1 hour, we return it.
         let old_items = match cache.get(&video_url) {
             Some((time, items)) => {
-                if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
+                if time.elapsed().unwrap_or_default().as_secs() < 60 * 60 * 24 {
                     return Ok(items.clone());
                 } else {
                     let _ = cache.check().await;
@@ -151,53 +151,18 @@ impl HentaihavenProvider {
             }
         };
 
-        let proxy = Proxy::all("http://192.168.0.103:8081").unwrap();
+        let mut requester = options.requester.clone().unwrap();
-        let client = Client::builder()
+        let text = requester
-            .cert_verification(false)
+            .get(&video_url, Some(Version::HTTP_2))
-            .emulation(Emulation::Firefox136)
+            .await
-            .build()?;
+            .unwrap();
-
+        if page > 1
-        let mut response = client
+        {
-            .get(video_url.clone())
+            return Ok(vec![]);
-            .proxy(proxy.clone())
-            .send()
-            .await?;
-
-        if response.status().is_redirection() {
-            response = client
-                .get(self.url.clone() + response.headers()["Location"].to_str().unwrap())
-                // .proxy(proxy)
-                .send()
-                .await?;
         }
-
+        let video_items: Vec<VideoItem> = self
-        if response.status().is_success() {
+            .get_video_items_from_html_search(text.clone(), &mut requester, pool)
-            let text = response.text().await?;
-            let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
-            if !video_items.is_empty() {
-                cache.remove(&video_url);
-                cache.insert(video_url.clone(), video_items.clone());
-            } else {
-                return Ok(old_items);
-            }
-            Ok(video_items)
-        } else {
-            let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set");
-            let flare = Flaresolverr::new(flare_url);
-            let result = flare
-                .solve(FlareSolverrRequest {
-                    cmd: "request.get".to_string(),
-                    url: video_url.clone(),
-                    maxTimeout: 60000,
-                })
             .await;
-            let video_items = match result {
-                Ok(res) => self.get_video_items_from_html(res.solution.response),
-                Err(e) => {
-                    println!("Error solving FlareSolverr: {}", e);
-                    return Err("Failed to solve FlareSolverr".into());
-                }
-            };
         if !video_items.is_empty() {
             cache.remove(&video_url);
             cache.insert(video_url.clone(), video_items.clone());
@@ -206,125 +171,298 @@ impl HentaihavenProvider {
         }
         Ok(video_items)
     }
-    }
 
-    fn get_video_items_from_html(&self, html: String) -> Vec<VideoItem> {
+    async fn get_video_items_from_html(
-        if html.is_empty() {
+        &self,
-            println!("HTML is empty");
+        html: String,
+        requester: &mut Requester,
+        pool: DbPool,
+    ) -> Vec<VideoItem> {
+        if html.is_empty() || html.contains("404 Not Found") {
             return vec![];
         }
-        let mut items: Vec<VideoItem> = Vec::new();
-        let raw_videos = html.split("\"wp-pagenavi\"").collect::<Vec<&str>>()[0]
-            .split("page-item-detail video")
-            .collect::<Vec<&str>>()[1..]
-            .to_vec();
-        for video_segment in &raw_videos {
-            let vid = video_segment.split("\n").collect::<Vec<&str>>();
-            for (index, line) in vid.iter().enumerate() {
-                println!("Line {}: {}", index, line);
-            }
 
-            let episode_count = video_segment
+        let block = match html
-                .split("chapter font-meta")
+            .split("previouspostslink")
-                .collect::<Vec<&str>>()[1]
+            .next()
-                .split("class=\"btn-link\">")
+            .and_then(|s| s.split("vraven_manga_list").nth(1))
-                .collect::<Vec<&str>>()[1]
-                .split("<")
-                .collect::<Vec<&str>>()[0]
-                .split(" ")
-                .collect::<Vec<&str>>()[2]
-                .to_string()
-                .parse::<i32>()
-                .unwrap();
-            let season = video_segment
-                .split("chapter font-meta")
-                .collect::<Vec<&str>>()[1]
-                .split("class=\"btn-link\">")
-                .collect::<Vec<&str>>()[1]
-                .split("<")
-                .collect::<Vec<&str>>()[0]
-                .split(" ")
-                .collect::<Vec<&str>>()[1]
-                == "Season";
-            let mut url_part_list = video_segment
-                .split("chapter font-meta")
-                .collect::<Vec<&str>>()[1]
-                .split("href=\"")
-                .collect::<Vec<&str>>()[1]
-                .split("\"")
-                .collect::<Vec<&str>>()[0]
-                .split("/")
-                .collect::<Vec<&str>>()[4]
-                .split("-")
-                .collect::<Vec<&str>>();
-            if url_part_list.len() > 5 {
-                if let Some(pos) = url_part_list.iter().rposition(|x| *x == "no") {
-                    url_part_list.remove(pos);
-                }
-            }
-            url_part_list.truncate(5);
-            let url_part = url_part_list.join("-");
-            for i in 1..=episode_count {
-                let mut video_url = format!(
-                    "https://master-lengs.org/api/v3/hh/{}-{}-eng/master.m3u8",
-                    url_part, i
-                );
-                if season {
-                    video_url = format!(
-                        "https://master-lengs.org/api/v3/hh/{}-season-eng/master.m3u8",
-                        url_part
-                    );
-                }
-                let title = format!(
-                    "{} - {}",
-                    video_segment.split("title=\"").collect::<Vec<&str>>()[1]
-                        .split("\"")
-                        .collect::<Vec<&str>>()[0]
-                        .to_string(),
-                    i
-                );
-                let id = format!("{}-{}", url_part, i);
-
-                let thumb = match video_segment.split("<img").collect::<Vec<&str>>()[1]
-                    .split("")
-                    .collect::<Vec<&str>>()[0]
-                    .contains("data-src=\"")
         {
-                    true => video_segment.split("<img ").collect::<Vec<&str>>()[1]
+            Some(b) => b,
-                        .split("data-src=\"")
+            None => {
-                        .collect::<Vec<&str>>()[1]
+                eprint!("Hentai Haven Provider: Failed to get block from html");
-                        .split("\"")
+                let e = Error::from(ErrorKind::Parse("html".into()));
-                        .collect::<Vec<&str>>()[0]
+                send_discord_error_report(
-                        .replace(" ", "%20")
+                    e.to_string(),
-                        .to_string(),
+                    Some(format_error_chain(&e)),
-                    false => video_segment.split("<img ").collect::<Vec<&str>>()[1]
+                    Some("Hentai Haven Provider"),
-                        .split("src=\"")
+                    Some(&format!("Failed to get block from html:\n```{html}\n```")),
-                        .collect::<Vec<&str>>()[1]
+                    file!(),
-                        .split("\"")
+                    line!(),
-                        .collect::<Vec<&str>>()[0]
+                    module_path!(),
-                        .replace(" ", "%20")
-                        .to_string(),
-                };
-                items.push(
-                    VideoItem::new(
-                        id,
-                        title,
-                        video_url.clone(),
-                        "hentaihaven".to_string(),
-                        thumb,
-                        0, // duration is not available
                 )
-                    .formats(vec![videos::VideoFormat::new(
+                .await;
-                        video_url.clone(),
+                return vec![];
-                        "1080".to_string(),
+            }
-                        "m3u8".to_string(),
+        };
-                    )])
+
-                    .aspect_ratio(0.73),
+        let futures = block
+            .split("id=\"manga-item-")
+            .skip(1)
+            .map(|el| self.get_video_item(el.to_string(), pool.clone(), requester.clone()));
+        join_all(futures)
+            .await
+            .into_iter()
+            .inspect(|r| {
+                if let Err(e) = r {
+                    eprint!("Hentai Haven Provider: Failed to get video item:{}\n", e);
+                    // Prepare data to move into the background task
+                    let msg = e.to_string();
+                    let chain = format_error_chain(&e);
+
+                    // Spawn the report into the background - NO .await here
+                    tokio::spawn(async move {
+                        let _ = send_discord_error_report(
+                            msg,
+                            Some(chain),
+                            Some("Hentai Haven Provider"),
+                            Some("Failed to get video item"),
+                            file!(), // Note: these might report the utility line
+                            line!(), // better to hardcode or pass from outside
+                            module_path!(),
+                        )
+                        .await;
+                    });
+                }
+            })
+            .filter_map(Result::ok)
+            .collect()
+    }
+
+    async fn get_video_items_from_html_search(
+        &self,
+        html: String,
+        requester: &mut Requester,
+        pool: DbPool,
+    ) -> Vec<VideoItem> {
+        if html.is_empty() || html.contains("404 Not Found") {
+            return vec![];
+        }
+
+        let block = match html
+            .split("<footer")
+            .next()
+            .and_then(|s| s.split("content-area").nth(1))
+        {
+            Some(b) => b,
+            None => {
+                eprint!("Hentai Haven Provider: Failed to get block from html");
+                let e = Error::from(ErrorKind::Parse("html".into()));
+                send_discord_error_report(
+                    e.to_string(),
+                    Some(format_error_chain(&e)),
+                    Some("Hentai Haven Provider"),
+                    Some(&format!("Failed to get block from html:\n```{html}\n```")),
+                    file!(),
+                    line!(),
+                    module_path!(),
+                )
+                .await;
+                return vec![];
+            }
+        };
+
+        let futures = block
+            .split("c-tabs-item__content col-6 col-md-12")
+            .skip(1)
+            .map(|el| self.get_video_item(el.to_string(), pool.clone(), requester.clone()));
+        join_all(futures)
+            .await
+            .into_iter()
+            .inspect(|r| {
+                if let Err(e) = r {
+                    eprint!("Hentai Haven Provider: Failed to get video item:{}\n", e);
+                    // Prepare data to move into the background task
+                    let msg = e.to_string();
+                    let chain = format_error_chain(&e);
+
+                    // Spawn the report into the background - NO .await here
+                    tokio::spawn(async move {
+                        let _ = send_discord_error_report(
+                            msg,
+                            Some(chain),
+                            Some("Hentai Haven Provider"),
+                            Some("Failed to get video item"),
+                            file!(), // Note: these might report the utility line
+                            line!(), // better to hardcode or pass from outside
+                            module_path!(),
+                        )
+                        .await;
+                    });
+                }
+            })
+            .filter_map(Result::ok)
+            .collect()
+    }
+
+    async fn get_video_item(
+        &self,
+        seg: String,
+        pool: DbPool,
+        mut requester: Requester,
+    ) -> Result<VideoItem> {
+        let video_url = seg
+            .split("a href=\"")
+            .nth(1)
+            .and_then(|s| s.split('"').next())
+            .ok_or_else(|| ErrorKind::Parse("video url\n\n{seg}".into()))?
+            .to_string();
+        let mut conn = pool.get().expect("couldn't get db connection from pool");
+        let db_result = db::get_video(&mut conn, video_url.clone());
+        drop(conn);
+        match db_result {
+            Ok(Some(video)) => {
+                let video_item = VideoItem::from(video);
+                match video_item {
+                    Ok(item) => return Ok(item),
+                    Err(e) => {
+                        eprint!("Failed to convert video from DB result: {}\n", e);
+                    }
+                }
+            }
+            Ok(None) => {
+                // continue to fetch and parse the video
+            }
+            Err(e) => {
+                eprint!("Database error: {}\n", e);
+                // continue to fetch and parse the video even if there's a DB error
+            }
+        }
+        let html = requester
+            .get(&video_url, Some(Version::HTTP_2))
+            .await
+            .map_err(|e| Error::from(format!("Failed to fetch video page: {}", e)))?;
+
+        let mut title = html
+            .split("<h1>")
+            .nth(1)
+            .and_then(|s| s.split("</h1>").next())
+            .ok_or_else(|| ErrorKind::Parse(format!("video title\n\n{seg}").into()))?
+            .trim()
+            .to_string();
+        title = decode(title.as_bytes())
+            .to_string()
+            .unwrap_or(title)
+            .titlecase();
+        let id = video_url
+            .split('/')
+            .nth(4)
+            .and_then(|s| s.split('.').next())
+            .ok_or_else(|| ErrorKind::Parse("video id\n\n{seg}".into()))?
+            .to_string();
+        let thumb = html
+            .split("og:image\" content=\"")
+            .nth(1)
+            .and_then(|s| s.split('"').next())
+            .unwrap_or("")
+            .to_string();
+        let raw_tags: Vec<FilterOption> = html
+            .split("Genre(s)")
+            .nth(1)
+            .unwrap_or_default()
+            .split("Release")
+            .nth(0)
+            .unwrap_or_default()
+            .split("a href=\"")
+            .skip(1)
+            .map(|tag_block| {
+                let id = tag_block
+                    .split("\"")
+                    .nth(1)
+                    .and_then(|s| s.split('"').next())
+                    .unwrap_or("")
+                    .to_string();
+                let title = tag_block
+                    .split('>')
+                    .nth(1)
+                    .and_then(|s| s.split('<').next())
+                    .map(|s| {
+                        decode(s.as_bytes())
+                            .to_string()
+                            .unwrap_or(s.to_string())
+                            .titlecase()
+                    })
+                    .unwrap_or("".to_string());
+                FilterOption {
+                    id: id.to_ascii_lowercase().replace(" ", "+"),
+                    title: title.clone(),
+                }
+            })
+            .collect::<Vec<FilterOption>>();
+        for tag in &raw_tags {
+            Self::push_unique(&self.categories, tag.clone());
+        }
+        let tags = raw_tags.into_iter().map(|t| t.title).collect();
+        let views = html
+            .split("Viewed")
+            .last()
+            .and_then(|s| s.split("summary-content\">").nth(1))
+            .and_then(|s| s.split(" Total").nth(0))
+            .map(|s| 
+                s.trim().parse::<u32>().unwrap_or(0))
+            .unwrap_or(0);
+        let mut formats = vec![];
+        let episode_block = html
+            .split("manga-chapters-holder")
+            .nth(1)
+            .unwrap_or_default()
+            .split("vraven_read")
+            .nth(0)
+            .unwrap_or_default();
+        for episode in episode_block.split("wp-manga-chapter").skip(1) {
+            let ep_thumbnail = episode
+                .split(" src=\"")
+                .nth(1)
+                .and_then(|s| s.split('"').next())
+                .unwrap_or_default();
+            let episode_title = episode
+                .split("<div>")
+                .nth(1)
+                .and_then(|s| s.split('<').next())
+                .unwrap_or_default()
+                .trim()
+                .to_string();
+            let episode_id = ep_thumbnail.split('/').nth(5).unwrap_or_default();
+            let episode_url = format!(
+                "https://master-lengs.org/api/v3/hh/{}/master.m3u8",
+                episode_id
             );
+            let format = VideoFormat::new(episode_url, "1080p".to_string(), "m3u8".to_string())
+                .format_id(episode_title);
+            formats.push(format);
         }
+        if formats.is_empty() {
+            let e = Error::from(format!("No formats found for video URL: {}", video_url));
+            return Err(e);
         }
-        return items;
+        if formats.len() > 1 {
-        //return items;
+            title = format!("{} ({} Episodes)", title, formats.len());
+        }
+
+        let video_item =
+            VideoItem::new(id, title, video_url.clone(), "hentaihaven".into(), thumb, 0)
+                .formats(formats)
+                .tags(tags)
+                .views(views);
+
+        let mut conn = pool.get().expect("couldn't get db connection from pool");
+        let _ = db::insert_video(
+            &mut conn,
+            &video_url,
+            &serde_json::to_string(&video_item).unwrap_or_default(),
+        );
+        drop(conn);
+
+        Ok(video_item)
     }
 }
 
@@ -337,29 +475,23 @@ impl Provider for HentaihavenProvider {
         sort: String,
         query: Option<String>,
         page: String,
-        per_page: String,
+        _per_page: String,
         options: ServerOptions,
     ) -> Vec<VideoItem> {
-        let _ = options;
+        let page = page.parse::<u8>().unwrap_or(1);
-        let _ = per_page;
+
-        let _ = pool;
+        let res = match query {
-        let videos: std::result::Result<Vec<VideoItem>, Error> = match query {
+            Some(q) => self.to_owned().query(cache, page, &q, options, pool).await,
-            Some(q) => self.query(cache, page.parse::<u8>().unwrap_or(1), &q).await,
+            None => self.get(cache, page, &sort, options, pool).await,
-            None => {
-                self.get(cache, page.parse::<u8>().unwrap_or(1), &sort)
-                    .await
-            }
         };
-        match videos {
+
-            Ok(v) => v,
+        res.unwrap_or_else(|e| {
-            Err(e) => {
+            eprintln!("hentai haven error: {e}");
-                println!("Error fetching videos: {}", e);
             vec![]
-            }
+        })
-        }
     }
 
-    fn get_channel(&self, clientversion: ClientVersion) -> Option<crate::status::Channel> {
+    fn get_channel(&self, v: ClientVersion) -> Option<Channel> {
-        Some(self.build_channel(clientversion))
+        Some(self.build_channel(v))
     }
 }

src/providers/mod.rs

@@ -42,6 +42,7 @@ pub mod hypnotube;
 pub mod freepornvideosxxx;
 pub mod hentaihaven;
 pub mod chaturbate;
+// pub mod tube8;
 
 // convenient alias
 pub type DynProvider = Arc<dyn Provider>;
@@ -63,6 +64,7 @@ pub static ALL_PROVIDERS: Lazy<HashMap<&'static str, DynProvider>> = Lazy::new(|
     m.insert("freepornvideosxxx", Arc::new(freepornvideosxxx::FreepornvideosxxxProvider::new()) as DynProvider);
     m.insert("hentaihaven", Arc::new(hentaihaven::HentaihavenProvider::new()) as DynProvider);
     m.insert("chaturbate", Arc::new(chaturbate::ChaturbateProvider::new()) as DynProvider);
+    // m.insert("tube8", Arc::new(tube8::Tube8Provider::new()) as DynProvider);
     // add more here as you migrate them
     m
 });

src/videos.rs

@@ -144,6 +144,9 @@ impl VideoItem {
             aspectRatio: None,
         }
     }
+    pub fn from(s: String) -> Result<Self, serde_json::Error> {
+        serde_json::from_str::<VideoItem>(&s)
+    }
     pub fn tags(mut self, tags: Vec<String>) -> Self {
         if tags.is_empty(){
             return self;