use crate::DbPool; use crate::api::ClientVersion; use crate::providers::{Provider, report_provider_error, report_provider_error_background}; use crate::status::*; use crate::util::cache::VideoCache; use crate::util::parse_abbreviated_number; use crate::util::requester::Requester; use crate::util::time::parse_time_to_seconds; use crate::videos::{ServerOptions, VideoFormat, VideoItem}; use async_trait::async_trait; use chrono::NaiveDate; use error_chain::error_chain; use futures::stream::{self, StreamExt}; use htmlentity::entity::{ICodedDataTrait, decode}; use regex::Regex; use scraper::{ElementRef, Html, Selector}; use std::sync::{Arc, RwLock}; use std::{thread, vec}; pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata = crate::providers::ProviderChannelMetadata { group_id: "onlyfans", tags: &["creator", "amateur", "clips"], }; error_chain! { foreign_links { Io(std::io::Error); HttpRequest(wreq::Error); Json(serde_json::Error); } errors { Parse(msg: String) { description("parse error") display("parse error: {}", msg) } } } #[derive(Debug, Clone)] pub struct ShooshtimeProvider { url: String, categories: Arc>>, channels: Arc>>, pornstars: Arc>>, tags: Arc>>, uploaders: Arc>>, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum FilterKind { Standard, MemberVideos, } #[derive(Debug, Clone)] struct QueryTarget { kind: FilterKind, url: String, } impl ShooshtimeProvider { pub fn new() -> Self { let provider = Self { url: "https://shooshtime.com".to_string(), categories: Arc::new(RwLock::new(vec![FilterOption { id: "all".to_string(), title: "All".to_string(), }])), channels: Arc::new(RwLock::new(vec![FilterOption { id: "all".to_string(), title: "All".to_string(), }])), pornstars: Arc::new(RwLock::new(vec![FilterOption { id: "all".to_string(), title: "All".to_string(), }])), tags: Arc::new(RwLock::new(vec![])), uploaders: Arc::new(RwLock::new(vec![])), }; provider.spawn_initial_load(); provider } fn spawn_initial_load(&self) { let url = self.url.clone(); let categories = Arc::clone(&self.categories); let channels = Arc::clone(&self.channels); let pornstars = Arc::clone(&self.pornstars); let tags = Arc::clone(&self.tags); let uploaders = Arc::clone(&self.uploaders); thread::spawn(move || { let rt = match tokio::runtime::Builder::new_current_thread() .enable_all() .build() { Ok(rt) => rt, Err(error) => { report_provider_error_background( "shooshtime", "spawn_initial_load.runtime_build", &error.to_string(), ); return; } }; rt.block_on(async move { if let Err(error) = Self::load_categories(&url, Arc::clone(&categories)).await { report_provider_error_background( "shooshtime", "load_categories", &error.to_string(), ); } if let Err(error) = Self::load_channels(&url, Arc::clone(&channels)).await { report_provider_error_background( "shooshtime", "load_channels", &error.to_string(), ); } if let Err(error) = Self::load_pornstars(&url, Arc::clone(&pornstars)).await { report_provider_error_background( "shooshtime", "load_pornstars", &error.to_string(), ); } if let Err(error) = Self::load_tags(&url, Arc::clone(&tags)).await { report_provider_error_background("shooshtime", "load_tags", &error.to_string()); } if let Err(error) = Self::load_uploaders(&url, Arc::clone(&uploaders)).await { report_provider_error_background( "shooshtime", "load_uploaders", &error.to_string(), ); } }); }); } fn build_channel(&self, _clientversion: ClientVersion) -> Channel { let categories = self .categories .read() .map(|value| value.clone()) .unwrap_or_default(); let channels = self .channels .read() .map(|value| value.clone()) .unwrap_or_default(); let pornstars = self .pornstars .read() .map(|value| value.clone()) .unwrap_or_default(); Channel { id: "shooshtime".to_string(), name: "Shooshtime".to_string(), description: "Videos, tags, pornstars, channels, and user uploads from Shooshtime." .to_string(), premium: false, favicon: "https://www.google.com/s2/favicons?sz=64&domain=shooshtime.com".to_string(), status: "active".to_string(), categories: categories.iter().map(|value| value.title.clone()).collect(), options: vec![ ChannelOption { id: "sort".to_string(), title: "Sort".to_string(), description: "Sort the videos".to_string(), systemImage: "list.number".to_string(), colorName: "blue".to_string(), options: vec![ FilterOption { id: "new".to_string(), title: "Newest".to_string(), }, FilterOption { id: "viewed".to_string(), title: "Most Viewed".to_string(), }, FilterOption { id: "rated".to_string(), title: "Top Rated".to_string(), }, FilterOption { id: "comments".to_string(), title: "Most Commented".to_string(), }, FilterOption { id: "recommended".to_string(), title: "Recommended".to_string(), }, ], multiSelect: false, }, ChannelOption { id: "categories".to_string(), title: "Categories".to_string(), description: "Browse a Shooshtime category".to_string(), systemImage: "square.grid.2x2".to_string(), colorName: "orange".to_string(), options: categories, multiSelect: false, }, ChannelOption { id: "stars".to_string(), title: "Pornstars".to_string(), description: "Browse a Shooshtime pornstar".to_string(), systemImage: "star.fill".to_string(), colorName: "yellow".to_string(), options: pornstars, multiSelect: false, }, ChannelOption { id: "sites".to_string(), title: "Channels".to_string(), description: "Browse a Shooshtime channel".to_string(), systemImage: "rectangle.stack".to_string(), colorName: "purple".to_string(), options: channels, multiSelect: false, }, ], nsfw: true, cacheDuration: Some(1800), } } fn push_unique(target: &Arc>>, item: FilterOption) { if let Ok(mut values) = target.write() { if !values.iter().any(|value| value.id == item.id) { values.push(item); } } } fn normalize_url(&self, url: &str) -> String { if url.is_empty() { return String::new(); } if url.starts_with("http://") || url.starts_with("https://") { return url.to_string(); } if url.starts_with("//") { return format!("https:{url}"); } if url.starts_with('/') { return format!("{}{}", self.url, url); } format!("{}/{}", self.url, url.trim_start_matches("./")) } fn normalize_title(title: &str) -> String { title .trim() .trim_start_matches('#') .split_whitespace() .collect::>() .join(" ") .to_ascii_lowercase() } fn decode_html(text: &str) -> String { decode(text.as_bytes()) .to_string() .unwrap_or_else(|_| text.to_string()) } fn collapse_whitespace(text: &str) -> String { text.split_whitespace().collect::>().join(" ") } fn text_of(element: &ElementRef<'_>) -> String { Self::collapse_whitespace(&element.text().collect::>().join(" ")) } fn parse_duration(text: &str) -> u32 { let raw = Self::collapse_whitespace(text); if raw.is_empty() { return 0; } if raw.contains(':') { return parse_time_to_seconds(&raw) .and_then(|seconds| u32::try_from(seconds).ok()) .unwrap_or(0); } raw.parse::().unwrap_or(0) } fn parse_views(text: &str) -> Option { let cleaned = text .replace("views", "") .replace("view", "") .replace(' ', "") .trim() .to_string(); parse_abbreviated_number(&cleaned) } fn parse_percent(text: &str) -> Option { let cleaned = text.trim().trim_end_matches('%').trim(); if cleaned.is_empty() { return None; } cleaned.parse::().ok() } fn parse_uploaded_at(date_text: &str) -> Option { NaiveDate::parse_from_str(date_text.trim(), "%d %B %Y") .ok() .and_then(|date| date.and_hms_opt(0, 0, 0)) .map(|value| value.and_utc().timestamp() as u64) } fn strip_counter_suffix(text: &str) -> String { let mut parts = text.split_whitespace().collect::>(); while parts .last() .is_some_and(|value| value.chars().all(|ch| ch.is_ascii_digit())) { parts.pop(); } while parts .last() .is_some_and(|value| value.eq_ignore_ascii_case("subscribers")) { parts.pop(); } parts.join(" ") } fn selector(value: &str) -> Result { Selector::parse(value) .map_err(|error| Error::from(format!("selector `{value}` parse failed: {error}"))) } fn regex(value: &str) -> Result { Regex::new(value).map_err(|error| Error::from(format!("regex `{value}` failed: {error}"))) } async fn fetch_html(url: &str) -> Result { let mut requester = Requester::new(); requester .get(url, None) .await .map_err(|error| Error::from(format!("request failed for {url}: {error}"))) } async fn load_categories( base_url: &str, categories: Arc>>, ) -> Result<()> { let html = Self::fetch_html(&format!("{base_url}/categories/")).await?; let document = Html::parse_document(&html); let link_selector = Self::selector("a[href]")?; for link in document.select(&link_selector) { let Some(href) = link.value().attr("href") else { continue; }; let normalized = href.trim_end_matches('/'); if !normalized.starts_with(&format!("{base_url}/categories/")) { continue; } let remainder = normalized .strip_prefix(&format!("{base_url}/categories/")) .unwrap_or_default(); if remainder.is_empty() || remainder.contains('/') || matches!(remainder, "viewed" | "rated" | "videos") { continue; } let title = Self::text_of(&link); if title.is_empty() { continue; } Self::push_unique( &categories, FilterOption { id: format!("{base_url}/categories/{remainder}/"), title, }, ); } Ok(()) } async fn load_channels(base_url: &str, channels: Arc>>) -> Result<()> { let link_selector = Self::selector("a[href]")?; for page in 1..=5 { let url = if page == 1 { format!("{base_url}/channels/") } else { format!("{base_url}/channels/{page}/") }; let html = Self::fetch_html(&url).await?; let document = Html::parse_document(&html); for link in document.select(&link_selector) { let Some(href) = link.value().attr("href") else { continue; }; let normalized = href.trim_end_matches('/'); if !normalized.starts_with(&format!("{base_url}/channels/")) { continue; } let remainder = normalized .strip_prefix(&format!("{base_url}/channels/")) .unwrap_or_default(); if remainder.is_empty() || remainder.contains('/') { continue; } let title = Self::text_of(&link); if title.is_empty() { continue; } Self::push_unique( &channels, FilterOption { id: format!("{base_url}/channels/{remainder}/"), title, }, ); } } Ok(()) } async fn load_pornstars( base_url: &str, pornstars: Arc>>, ) -> Result<()> { let link_selector = Self::selector("a[href]")?; for page in 1..=5 { let url = if page == 1 { format!("{base_url}/pornstars/") } else { format!("{base_url}/pornstars/{page}/") }; let html = Self::fetch_html(&url).await?; let document = Html::parse_document(&html); for link in document.select(&link_selector) { let Some(href) = link.value().attr("href") else { continue; }; let normalized = href.trim_end_matches('/'); if !normalized.starts_with(&format!("{base_url}/pornstars/")) { continue; } let remainder = normalized .strip_prefix(&format!("{base_url}/pornstars/")) .unwrap_or_default(); if remainder.is_empty() || remainder.contains('/') || matches!(remainder, "celebrities" | "amateurs" | "couples" | "all") { continue; } let title = Self::text_of(&link); if title.is_empty() { continue; } Self::push_unique( &pornstars, FilterOption { id: format!("{base_url}/pornstars/{remainder}/"), title, }, ); } } Ok(()) } async fn load_tags(base_url: &str, tags: Arc>>) -> Result<()> { let html = Self::fetch_html(&format!("{base_url}/tags/")).await?; let document = Html::parse_document(&html); let link_selector = Self::selector("a[href]")?; for link in document.select(&link_selector) { let Some(href) = link.value().attr("href") else { continue; }; let normalized = href.trim_end_matches('/'); if !normalized.starts_with(&format!("{base_url}/tags/")) { continue; } let remainder = normalized .strip_prefix(&format!("{base_url}/tags/")) .unwrap_or_default(); if remainder.is_empty() || remainder.contains('/') { continue; } let title = Self::text_of(&link) .trim_start_matches('#') .trim() .to_string(); if title.is_empty() { continue; } Self::push_unique( &tags, FilterOption { id: format!("{base_url}/tags/{remainder}/"), title, }, ); } Ok(()) } async fn load_uploaders( base_url: &str, uploaders: Arc>>, ) -> Result<()> { let link_selector = Self::selector("a[href]")?; for page in 1..=5 { let url = if page == 1 { format!("{base_url}/members/") } else { format!("{base_url}/members/{page}/") }; let html = Self::fetch_html(&url).await?; let document = Html::parse_document(&html); for link in document.select(&link_selector) { let Some(href) = link.value().attr("href") else { continue; }; let normalized = href.trim_end_matches('/'); if !normalized.starts_with(&format!("{base_url}/members/")) { continue; } let remainder = normalized .strip_prefix(&format!("{base_url}/members/")) .unwrap_or_default(); if remainder.is_empty() || remainder.contains('/') { continue; } if !remainder.chars().all(|value| value.is_ascii_digit()) { continue; } let title = Self::text_of(&link) .replace("no photo", "") .trim() .to_string(); if title.is_empty() { continue; } Self::push_unique( &uploaders, FilterOption { id: format!("{base_url}/members/{remainder}/videos/"), title, }, ); } } Ok(()) } fn normalize_sort(sort: &str) -> &'static str { match sort { "viewed" => "viewed", "rated" => "rated", "comments" => "comments", "recommended" => "recommended", _ => "new", } } fn search_sort_param(sort: &str) -> Option<&'static str> { match Self::normalize_sort(sort) { "viewed" => Some("video_viewed"), "rated" => Some("rating"), "comments" => Some("most_commented"), "recommended" => Some("most_favourited"), _ => None, } } fn append_query_param(url: &str, key: &str, value: &str) -> String { let separator = if url.contains('?') { "&" } else { "?" }; format!("{url}{separator}{key}={value}") } fn build_top_level_url(&self, page: u8, sort: &str) -> String { let base = match Self::normalize_sort(sort) { "viewed" => format!("{}/videos/viewed/", self.url), "rated" => format!("{}/videos/rated/", self.url), "comments" => format!("{}/videos/comments/", self.url), "recommended" => format!("{}/videos/recommended/", self.url), _ => format!("{}/videos/", self.url), }; if page > 1 { format!("{base}{page}/") } else { base } } fn build_standard_filtered_url(&self, base: &str, page: u8, sort: &str) -> String { let mut url = if page > 1 { format!("{}{page}/", base.trim_end_matches('/').to_string() + "/") } else { base.to_string() }; if let Some(sort_by) = Self::search_sort_param(sort) { url = Self::append_query_param(&url, "sort_by", sort_by); } url } fn build_member_filtered_url(&self, base: &str, page: u8, sort: &str) -> String { let mut url = base.to_string(); if let Some(sort_by) = Self::search_sort_param(sort) { url = Self::append_query_param(&url, "sort_by", sort_by); } if page > 1 { url = Self::append_query_param(&url, "from_videos", &page.to_string()); } url } fn build_search_url(&self, query: &str, page: u8, sort: &str) -> String { let encoded_query = url::form_urlencoded::byte_serialize(query.as_bytes()).collect::(); let mut url = if page > 1 { format!("{}/search/{page}/?q={encoded_query}", self.url) } else { format!("{}/search/?q={encoded_query}", self.url) }; if let Some(sort_by) = Self::search_sort_param(sort) { url = Self::append_query_param(&url, "sort_by", sort_by); } url } fn resolve_option_target(&self, options: &ServerOptions) -> Option { if let Some(category) = options.categories.as_deref() { if !category.is_empty() && category != "all" { return Some(QueryTarget { kind: FilterKind::Standard, url: category.to_string(), }); } } if let Some(pornstar) = options.stars.as_deref() { if !pornstar.is_empty() && pornstar != "all" { return Some(QueryTarget { kind: FilterKind::Standard, url: pornstar.to_string(), }); } } if let Some(channel) = options.sites.as_deref() { if !channel.is_empty() && channel != "all" { return Some(QueryTarget { kind: FilterKind::Standard, url: channel.to_string(), }); } } None } fn match_filter(options: &[FilterOption], query: &str) -> Option { let normalized_query = Self::normalize_title(query); options .iter() .find(|value| Self::normalize_title(&value.title) == normalized_query) .map(|value| QueryTarget { kind: if value.id.contains("/members/") { FilterKind::MemberVideos } else { FilterKind::Standard }, url: value.id.clone(), }) } fn resolve_query_target(&self, query: &str) -> Option { if let Ok(uploaders) = self.uploaders.read() { if let Some(target) = Self::match_filter(&uploaders, query) { return Some(target); } } if let Ok(pornstars) = self.pornstars.read() { if let Some(target) = Self::match_filter(&pornstars, query) { return Some(target); } } if let Ok(channels) = self.channels.read() { if let Some(target) = Self::match_filter(&channels, query) { return Some(target); } } if let Ok(tags) = self.tags.read() { if let Some(target) = Self::match_filter(&tags, query) { return Some(target); } } if let Ok(categories) = self.categories.read() { if let Some(target) = Self::match_filter(&categories, query) { return Some(target); } } None } fn build_url_for_target(&self, target: &QueryTarget, page: u8, sort: &str) -> String { match target.kind { FilterKind::Standard => self.build_standard_filtered_url(&target.url, page, sort), FilterKind::MemberVideos => self.build_member_filtered_url(&target.url, page, sort), } } fn parse_list_videos(&self, html: &str) -> Result> { let document = Html::parse_document(html); let card_selector = Self::selector("div.item.thumb")?; let video_link_selector = Self::selector("a[href*=\"/videos/\"]")?; let image_selector = Self::selector("img")?; let duration_selector = Self::selector("span.duration")?; let title_selector = Self::selector("span.description, span.description.secondary")?; let info_selector = Self::selector("ul.info li.val")?; let uploader_selector = Self::selector("ul.info a[href*=\"/members/\"]")?; let mut items = Vec::new(); for card in document.select(&card_selector) { let Some(link) = card.select(&video_link_selector).next() else { continue; }; let href = link.value().attr("href").unwrap_or_default(); if href.is_empty() { continue; } let page_url = self.normalize_url(href); let id = page_url .trim_end_matches('/') .split('/') .nth_back(1) .unwrap_or_default() .to_string(); if id.is_empty() { continue; } let image = card.select(&image_selector).next(); let thumb = image .and_then(|value| { value .value() .attr("data-src") .or_else(|| value.value().attr("src")) }) .map(|value| self.normalize_url(value)) .unwrap_or_default(); let preview = image .and_then(|value| value.value().attr("data-preview")) .map(|value| self.normalize_url(value)); let title = card .select(&title_selector) .next() .map(|value| Self::decode_html(&Self::text_of(&value))) .unwrap_or_else(|| { Self::decode_html(link.value().attr("title").unwrap_or_default()) }); if title.is_empty() { continue; } let duration = card .select(&duration_selector) .next() .map(|value| Self::parse_duration(&Self::text_of(&value))) .unwrap_or(0); let views = card .select(&info_selector) .next() .and_then(|value| Self::parse_views(&Self::text_of(&value))); let uploader_link = card.select(&uploader_selector).next(); let uploader = uploader_link .map(|value| Self::text_of(&value)) .filter(|value| !value.is_empty()); let uploader_url = uploader_link .and_then(|value| value.value().attr("href")) .map(|value| { let absolute = self.normalize_url(value); if absolute.ends_with('/') { format!("{absolute}videos/") } else { format!("{absolute}/videos/") } }); if let (Some(name), Some(url)) = (&uploader, &uploader_url) { Self::push_unique( &self.uploaders, FilterOption { id: url.clone(), title: name.clone(), }, ); } let mut item = VideoItem::new( id, title, page_url, "shooshtime".to_string(), thumb, duration, ); if let Some(views) = views { item = item.views(views); } if let Some(uploader) = uploader { item = item.uploader(uploader); } if let Some(uploader_url) = uploader_url { item = item.uploader_url(uploader_url); } if let Some(preview) = preview { item = item.preview(preview); } items.push(item); } Ok(items) } fn extract_js_value(block: &str, regex: &Regex) -> Option { regex .captures(block) .and_then(|value| value.get(1)) .map(|value| value.as_str().replace("\\/", "/").replace("\\'", "'")) } fn apply_detail_video( &self, mut item: VideoItem, html: &str, page_url: &str, ) -> Result { let flashvars_regex = Self::regex(r#"(?s)var\s+flashvars\s*=\s*\{(.*?)\};"#)?; let value_regex = |key: &str| Self::regex(&format!(r#"{key}:\s*'([^']*)'"#)); let flashvars = flashvars_regex .captures(html) .and_then(|value| value.get(1)) .map(|value| value.as_str().to_string()) .unwrap_or_default(); let video_url_regex = value_regex("video_url")?; let video_url_text_regex = value_regex("video_url_text")?; let video_alt_url_regex = value_regex("video_alt_url")?; let video_alt_url_text_regex = value_regex("video_alt_url_text")?; let preview_url_regex = value_regex("preview_url")?; let primary_url = Self::extract_js_value(&flashvars, &video_url_regex) .map(|value| self.normalize_url(&value)); let primary_quality = Self::extract_js_value(&flashvars, &video_url_text_regex) .unwrap_or_else(|| "480p".to_string()); let alt_url = Self::extract_js_value(&flashvars, &video_alt_url_regex) .map(|value| self.normalize_url(&value)); let alt_quality = Self::extract_js_value(&flashvars, &video_alt_url_text_regex) .unwrap_or_else(|| "720p".to_string()); let preview_url = Self::extract_js_value(&flashvars, &preview_url_regex) .map(|value| self.normalize_url(&value)); let mut formats = Vec::new(); if let Some(url) = &primary_url { formats.push( VideoFormat::new(url.clone(), primary_quality.clone(), "mp4".to_string()) .format_id(primary_quality.clone()) .http_header("Referer".to_string(), page_url.to_string()), ); } if let Some(url) = &alt_url { formats.push( VideoFormat::new(url.clone(), alt_quality.clone(), "mp4".to_string()) .format_id(alt_quality.clone()) .http_header("Referer".to_string(), page_url.to_string()), ); } let document = Html::parse_document(html); let block_selector = Self::selector(".block-video")?; let uploader_selector = Self::selector("a[href*=\"/members/\"]")?; let rating_selector = Self::selector(".headline .value, .headline .rating, .value")?; let title_selector = Self::selector(".headline h1, .headline .title, h1")?; let about_section_selector = Self::selector("#tab2")?; let info_value_selector = Self::selector("ul.video-info li.item span.value")?; let tags_outer_selector = Self::selector(".tags-outer")?; let tags_link_selector = Self::selector("a[href]")?; let pornstar_selector = Self::selector("a[href*=\"/pornstars/\"]")?; let block = document.select(&block_selector).next(); let uploader_link = block .as_ref() .and_then(|value| value.select(&uploader_selector).next()); let uploader = uploader_link .as_ref() .map(|value| Self::strip_counter_suffix(&Self::text_of(value))) .filter(|value| !value.is_empty()); let uploader_url = uploader_link .as_ref() .and_then(|value| value.value().attr("href")) .map(|value| { let absolute = self.normalize_url(value); if absolute.ends_with('/') { format!("{absolute}videos/") } else { format!("{absolute}/videos/") } }); if let (Some(name), Some(url)) = (&uploader, &uploader_url) { Self::push_unique( &self.uploaders, FilterOption { id: url.clone(), title: name.clone(), }, ); } let title = block .as_ref() .and_then(|value| value.select(&title_selector).next()) .map(|value| Self::decode_html(&Self::text_of(&value))) .filter(|value| !value.is_empty()); let rating = block.as_ref().and_then(|value| { value.select(&rating_selector).find_map(|node| { let text = Self::text_of(&node); if text.contains('%') { Self::parse_percent(&text) } else { None } }) }); let about = document.select(&about_section_selector).next(); let info_values = about .as_ref() .map(|value| { value .select(&info_value_selector) .map(|node| Self::text_of(&node)) .collect::>() }) .unwrap_or_default(); let views = info_values.first().and_then(|value| { let normalized = value.replace(' ', ""); parse_abbreviated_number(&normalized) }); let uploaded_at = info_values .get(1) .and_then(|value| Self::parse_uploaded_at(value)); let mut tags = Vec::new(); if let Some(about_value) = about.as_ref() { for section in about_value.select(&tags_outer_selector) { let title_text = section .select(&Self::selector("h4.title")?) .next() .map(|value| Self::text_of(&value).to_ascii_lowercase()) .unwrap_or_default(); let values = section .select(&tags_link_selector) .filter_map(|link| { let text = Self::text_of(&link); if text.is_empty() { return None; } Some(( text, link.value().attr("href").unwrap_or_default().to_string(), )) }) .collect::>(); if title_text.contains("categories") { for (text, href) in values { tags.push(text.clone()); let normalized = self.normalize_url(&href); Self::push_unique( &self.categories, FilterOption { id: normalized, title: text, }, ); } } else if title_text.contains("tags") { for (text, href) in values { tags.push(text.clone()); let normalized = self.normalize_url(&href); Self::push_unique( &self.tags, FilterOption { id: normalized, title: text, }, ); } } else if title_text.contains("models") { for (text, href) in values { tags.push(text.clone()); let normalized = self.normalize_url(&href); Self::push_unique( &self.pornstars, FilterOption { id: normalized, title: text, }, ); } } } } if let Some(block_value) = block.as_ref() { for pornstar in block_value.select(&pornstar_selector) { let text = Self::strip_counter_suffix(&Self::text_of(&pornstar)); if text.is_empty() { continue; } let normalized = self.normalize_url(pornstar.value().attr("href").unwrap_or_default()); Self::push_unique( &self.pornstars, FilterOption { id: normalized, title: text.clone(), }, ); tags.push(text); } } tags.sort(); tags.dedup(); if let Some(title) = title { item.title = title; } if !formats.is_empty() { item = item.formats(formats); } if let Some(views) = views { item = item.views(views); } if let Some(rating) = rating { item = item.rating(rating); } if let Some(uploaded_at) = uploaded_at { item = item.uploaded_at(uploaded_at); } if let Some(uploader) = uploader { item = item.uploader(uploader); } if let Some(uploader_url) = uploader_url { item = item.uploader_url(uploader_url); } if !tags.is_empty() { item = item.tags(tags); } if item.preview.is_none() { if let Some(preview) = preview_url.as_ref() { item = item.preview(preview.clone()); } } if item.thumb.is_empty() { if let Some(thumb) = preview_url { item.thumb = thumb; } } Ok(item) } async fn enrich_video(&self, item: VideoItem, options: &ServerOptions) -> VideoItem { let page_url = item.url.clone(); let original_item = item.clone(); let mut requester = match options.requester.clone() { Some(requester) => requester, None => Requester::new(), }; let html = match requester.get(&page_url, None).await { Ok(html) => html, Err(error) => { report_provider_error_background( "shooshtime", "enrich_video.request", &format!("url={}; error={error}", page_url), ); return item; } }; match self.apply_detail_video(item, &html, &page_url) { Ok(item) => item, Err(error) => { report_provider_error_background( "shooshtime", "enrich_video.parse", &format!("url={}; error={error}", page_url), ); original_item } } } async fn fetch_items_for_url( &self, cache: VideoCache, url: String, options: &ServerOptions, ) -> Result> { if let Some((time, items)) = cache.get(&url) { if time.elapsed().unwrap_or_default().as_secs() < 300 { return Ok(items.clone()); } } let mut requester = crate::providers::requester_or_default(options, module_path!(), "missing_requester"); let html = match requester.get(&url, None).await { Ok(html) => html, Err(error) => { report_provider_error( "shooshtime", "fetch_items_for_url.request", &format!("url={url}; error={error}"), ) .await; return Ok(vec![]); } }; let list_videos = self.parse_list_videos(&html)?; if list_videos.is_empty() { return Ok(vec![]); } let items = stream::iter(list_videos.into_iter().map(|video| { let provider = self.clone(); let options = options.clone(); async move { provider.enrich_video(video, &options).await } })) .buffer_unordered(6) .collect::>() .await; if !items.is_empty() { cache.insert(url, items.clone()); } Ok(items) } async fn get( &self, cache: VideoCache, page: u8, sort: &str, options: ServerOptions, ) -> Result> { let target = self.resolve_option_target(&options); let url = match target { Some(target) => self.build_url_for_target(&target, page, sort), None => self.build_top_level_url(page, sort), }; self.fetch_items_for_url(cache, url, &options).await } async fn query( &self, cache: VideoCache, page: u8, query: &str, sort: &str, options: ServerOptions, ) -> Result> { let url = match self.resolve_query_target(query) { Some(target) => self.build_url_for_target(&target, page, sort), None => self.build_search_url(query, page, sort), }; self.fetch_items_for_url(cache, url, &options).await } } #[async_trait] impl Provider for ShooshtimeProvider { async fn get_videos( &self, cache: VideoCache, pool: DbPool, sort: String, query: Option, page: String, per_page: String, options: ServerOptions, ) -> Vec { let _ = pool; let _ = per_page; let page = page.parse::().unwrap_or(1); let result = match query { Some(query) if !query.trim().is_empty() => { self.query(cache, page, &query, &sort, options).await } _ => self.get(cache, page, &sort, options).await, }; match result { Ok(videos) => videos, Err(error) => { report_provider_error_background("shooshtime", "get_videos", &error.to_string()); vec![] } } } fn get_channel(&self, clientversion: ClientVersion) -> Option { Some(self.build_channel(clientversion)) } } #[cfg(test)] mod tests { use super::ShooshtimeProvider; use crate::videos::VideoItem; #[test] fn preserves_list_thumb_when_detail_has_preview_url() { let provider = ShooshtimeProvider::new(); let item = VideoItem::new( "123".to_string(), "Example".to_string(), "https://shooshtime.com/videos/example/123/".to_string(), "shooshtime".to_string(), "https://shooshtime.com/list-thumb.jpg".to_string(), 0, ); let html = r#" "#; let enriched = provider .apply_detail_video(item, html, "https://shooshtime.com/videos/example/123/") .unwrap(); assert_eq!(enriched.thumb, "https://shooshtime.com/list-thumb.jpg"); assert_eq!( enriched.preview.as_deref(), Some("https://shooshtime.com/detail-thumb.jpg") ); } }