From 03c4e4d3c495773240ccfbfd259eeed6bcb34a82 Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 26 Jun 2026 16:55:24 +0000 Subject: [PATCH] paradisehill fixes --- src/providers/paradisehill.rs | 148 +++++++++++++++++++++++++--------- 1 file changed, 108 insertions(+), 40 deletions(-) diff --git a/src/providers/paradisehill.rs b/src/providers/paradisehill.rs index 909923a..3514b7e 100644 --- a/src/providers/paradisehill.rs +++ b/src/providers/paradisehill.rs @@ -1,13 +1,14 @@ use crate::DbPool; use crate::api::ClientVersion; -use crate::providers::Provider; +use crate::providers::{ + Provider, report_provider_error, report_provider_error_background, requester_or_default, +}; use crate::status::*; use crate::util::cache::VideoCache; -use crate::util::requester::Requester; -use crate::videos::ServerOptions; -use crate::videos::VideoItem; +use crate::videos::{ServerOptions, VideoFormat, VideoItem}; use async_trait::async_trait; use error_chain::error_chain; +use futures::stream::{self, StreamExt}; use htmlentity::entity::{ICodedDataTrait, decode}; pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata = @@ -24,10 +25,13 @@ error_chain! { } } +const ENRICH_CONCURRENCY: usize = 4; + #[derive(Debug, Clone)] pub struct ParadisehillProvider { url: String, } + impl ParadisehillProvider { pub fn new() -> Self { ParadisehillProvider { @@ -50,6 +54,75 @@ impl ParadisehillProvider { cacheDuration: None, } } + + // Extracts all part MP4 URLs from the `var videoList = [...]` JS block. + // Each videoList entry is one sequential part ("Part 1", "Part 2", …). + fn extract_all_parts(html: &str) -> Vec { + let start = match html.find("var videoList = ") { + Some(i) => i, + None => return vec![], + }; + let slice = &html[start..]; + let end = slice.find("];").map(|i| i + 2).unwrap_or(slice.len()); + let slice = &slice[..end]; + + let mut parts = Vec::new(); + let mut pos = 0; + while let Some(rel) = slice[pos..].find("\"src\":\"") { + let src_start = pos + rel + "\"src\":\"".len(); + if let Some(src_end) = slice[src_start..].find('"') { + let url = slice[src_start..src_start + src_end].replace("\\/", "/"); + if url.starts_with("http") { + parts.push(url); + } + } + pos += rel + 1; + } + parts + } + + async fn enrich_item(mut item: VideoItem, options: ServerOptions) -> VideoItem { + let mut requester = requester_or_default(&options, module_path!(), "enrich_item"); + match requester.get(&item.url, None).await { + Ok(html) => { + let parts = Self::extract_all_parts(&html); + if !parts.is_empty() { + item.formats = Some( + parts + .into_iter() + .enumerate() + .map(|(i, url)| { + VideoFormat::new( + url, + format!("Part {}", i + 1), + "mp4".to_string(), + ) + }) + .collect(), + ); + } + } + Err(e) => { + report_provider_error_background( + "paradisehill", + "enrich_item", + &e.to_string(), + ); + } + } + item + } + + async fn enrich_all(stubs: Vec, options: ServerOptions) -> Vec { + stream::iter(stubs.into_iter().map(|item| { + let opts = options.clone(); + async move { Self::enrich_item(item, opts).await } + })) + .buffer_unordered(ENRICH_CONCURRENCY) + .collect::>() + .await + } + async fn get( &self, cache: VideoCache, @@ -57,7 +130,7 @@ impl ParadisehillProvider { options: ServerOptions, ) -> Result> { let mut requester = - crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); + requester_or_default(&options, module_path!(), "missing_requester"); let url_str = format!("{}/all/?sort=created_at&page={}", self.url, page); @@ -69,15 +142,13 @@ impl ParadisehillProvider { items.clone() } } - None => { - vec![] - } + None => vec![], }; let text = match requester.get(&url_str, None).await { Ok(text) => text, Err(e) => { - crate::providers::report_provider_error( + report_provider_error( "paradisehill", "get.request", &format!("url={url_str}; error={e}"), @@ -86,17 +157,20 @@ impl ParadisehillProvider { return Ok(old_items); } }; - // Pass a reference to options if needed, or reconstruct as needed - let video_items: Vec = self - .get_video_items_from_html(text.clone(), requester) - .await; - if !video_items.is_empty() { + + let stubs = self.parse_listing_html(&text); + if stubs.is_empty() { + return Ok(old_items); + } + + let items = Self::enrich_all(stubs, options).await; + if !items.is_empty() { cache.remove(&url_str); - cache.insert(url_str.clone(), video_items.clone()); + cache.insert(url_str.clone(), items.clone()); } else { return Ok(old_items); } - Ok(video_items) + Ok(items) } async fn query( @@ -106,15 +180,14 @@ impl ParadisehillProvider { query: &str, options: ServerOptions, ) -> Result> { - // Extract needed fields from options at the start let mut requester = - crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); + requester_or_default(&options, module_path!(), "missing_requester"); let search_string = query.replace(" ", "+"); let url_str = format!( "{}/search/?pattern={}&page={}", self.url, search_string, page ); - // Check our Video Cache. If the result is younger than 1 hour, we return it. + let old_items = match cache.get(&url_str) { Some((time, items)) => { if time.elapsed().unwrap_or_default().as_secs() < 60 * 60 { @@ -124,14 +197,13 @@ impl ParadisehillProvider { return Ok(items.clone()); } } - None => { - vec![] - } + None => vec![], }; + let text = match requester.get(&url_str, None).await { Ok(text) => text, Err(e) => { - crate::providers::report_provider_error( + report_provider_error( "paradisehill", "query.request", &format!("url={url_str}; error={e}"), @@ -140,25 +212,24 @@ impl ParadisehillProvider { return Ok(old_items); } }; - let video_items: Vec = self - .get_video_items_from_html(text.clone(), requester) - .await; - if !video_items.is_empty() { + + let stubs = self.parse_listing_html(&text); + if stubs.is_empty() { + return Ok(old_items); + } + + let items = Self::enrich_all(stubs, options).await; + if !items.is_empty() { cache.remove(&url_str); - cache.insert(url_str.clone(), video_items.clone()); + cache.insert(url_str.clone(), items.clone()); } else { return Ok(old_items); } - Ok(video_items) + Ok(items) } - async fn get_video_items_from_html( - &self, - html: String, - _requester: Requester, - ) -> Vec { + fn parse_listing_html(&self, html: &str) -> Vec { if html.is_empty() { - println!("HTML is empty"); return vec![]; } let mut items: Vec = Vec::new(); @@ -217,7 +288,7 @@ impl ParadisehillProvider { items.push( VideoItem::new(id, title, video_url, "paradisehill".to_string(), thumb, 0) - .aspect_ratio(0.697674419 as f32) + .aspect_ratio(0.697674419_f32) .tags(tags), ); } @@ -246,10 +317,7 @@ impl Provider for ParadisehillProvider { self.query(cache, page.parse::().unwrap_or(1), &q, options) .await } - None => { - self.get(cache, page.parse::().unwrap_or(1), options) - .await - } + None => self.get(cache, page.parse::().unwrap_or(1), options).await, }; match videos { Ok(v) => v,