diff --git a/src/providers/heavyfetish.rs b/src/providers/heavyfetish.rs index 5ef8a36..4cc081b 100644 --- a/src/providers/heavyfetish.rs +++ b/src/providers/heavyfetish.rs @@ -11,13 +11,13 @@ use crate::util::time::parse_time_to_seconds; use crate::videos::{ServerOptions, VideoFormat, VideoItem}; use async_trait::async_trait; use error_chain::error_chain; -use futures::stream::{self, StreamExt}; use htmlentity::entity::{ICodedDataTrait, decode}; use regex::Regex; use scraper::{ElementRef, Html, Selector}; use std::collections::HashMap; use std::sync::{Arc, RwLock}; use std::{thread, vec}; +use url::Url; pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata = crate::providers::ProviderChannelMetadata { @@ -354,6 +354,31 @@ impl HeavyfetishProvider { .to_string() } + fn video_id_from_page_url(page_url: &str) -> String { + let Ok(parsed_url) = Url::parse(page_url) else { + return String::new(); + }; + + let segments = parsed_url + .path_segments() + .map(|value| value.collect::>()) + .unwrap_or_default(); + + if segments.is_empty() { + return String::new(); + } + + if let Some(index) = segments.iter().position(|segment| *segment == "videos") { + if let Some(primary) = segments.get(index + 1) { + if !primary.is_empty() { + return (*primary).to_string(); + } + } + } + + segments.last().copied().unwrap_or_default().to_string() + } + fn push_unique(target: &Arc>>, item: FilterOption) { if item.id.is_empty() || item.title.is_empty() { return; @@ -698,12 +723,7 @@ impl HeavyfetishProvider { }; let href = link.value().attr("href").unwrap_or_default(); let page_url = self.normalize_url(href); - let id = page_url - .trim_end_matches('/') - .split('/') - .nth_back(1) - .unwrap_or_default() - .to_string(); + let id = Self::video_id_from_page_url(&page_url); if id.is_empty() || page_url.is_empty() { continue; @@ -885,11 +905,6 @@ impl HeavyfetishProvider { } } - let formats = self.build_formats(html, page_url)?; - if !formats.is_empty() { - item = item.formats(formats); - } - let uploader_link = document.select(&uploader_selector).next(); let uploader = uploader_link .as_ref() @@ -1072,20 +1087,11 @@ impl HeavyfetishProvider { return Ok(vec![]); } - let limited_items = list_items + let items = list_items .into_iter() .take(per_page_limit.max(1)) .collect::>(); - let items = stream::iter(limited_items.into_iter().map(|item| { - let provider = self.clone(); - let options = options.clone(); - async move { provider.enrich_video(item, &options).await } - })) - .buffer_unordered(4) - .collect::>() - .await; - if !items.is_empty() { cache.insert(url, items.clone()); } @@ -1239,4 +1245,20 @@ mod tests { Some("https://heavyfetish.com/list-preview.mp4") ); } + + #[test] + fn extracts_video_id_from_slug_only_layout() { + let id = HeavyfetishProvider::video_id_from_page_url( + "https://heavyfetish.com/videos/mistress-gaia-silky-feet-sniffer/", + ); + assert_eq!(id, "mistress-gaia-silky-feet-sniffer"); + } + + #[test] + fn extracts_video_id_from_legacy_numeric_layout() { + let id = HeavyfetishProvider::video_id_from_page_url( + "https://heavyfetish.com/videos/120660/example/", + ); + assert_eq!(id, "120660"); + } }