paradisehill fixes

src/providers/paradisehill.rs

@@ -1,13 +1,14 @@
 use crate::DbPool;
 use crate::api::ClientVersion;
-use crate::providers::Provider;
+use crate::providers::{
+    Provider, report_provider_error, report_provider_error_background, requester_or_default,
+};
 use crate::status::*;
 use crate::util::cache::VideoCache;
-use crate::util::requester::Requester;
+use crate::videos::{ServerOptions, VideoFormat, VideoItem};
-use crate::videos::ServerOptions;
-use crate::videos::VideoItem;
 use async_trait::async_trait;
 use error_chain::error_chain;
+use futures::stream::{self, StreamExt};
 use htmlentity::entity::{ICodedDataTrait, decode};
 
 pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata =
@@ -24,10 +25,13 @@ error_chain! {
     }
 }
 
+const ENRICH_CONCURRENCY: usize = 4;
+
 #[derive(Debug, Clone)]
 pub struct ParadisehillProvider {
     url: String,
 }
+
 impl ParadisehillProvider {
     pub fn new() -> Self {
         ParadisehillProvider {
@@ -50,6 +54,75 @@ impl ParadisehillProvider {
             cacheDuration: None,
         }
     }
+
+    // Extracts all part MP4 URLs from the `var videoList = [...]` JS block.
+    // Each videoList entry is one sequential part ("Part 1", "Part 2", …).
+    fn extract_all_parts(html: &str) -> Vec<String> {
+        let start = match html.find("var videoList = ") {
+            Some(i) => i,
+            None => return vec![],
+        };
+        let slice = &html[start..];
+        let end = slice.find("];").map(|i| i + 2).unwrap_or(slice.len());
+        let slice = &slice[..end];
+
+        let mut parts = Vec::new();
+        let mut pos = 0;
+        while let Some(rel) = slice[pos..].find("\"src\":\"") {
+            let src_start = pos + rel + "\"src\":\"".len();
+            if let Some(src_end) = slice[src_start..].find('"') {
+                let url = slice[src_start..src_start + src_end].replace("\\/", "/");
+                if url.starts_with("http") {
+                    parts.push(url);
+                }
+            }
+            pos += rel + 1;
+        }
+        parts
+    }
+
+    async fn enrich_item(mut item: VideoItem, options: ServerOptions) -> VideoItem {
+        let mut requester = requester_or_default(&options, module_path!(), "enrich_item");
+        match requester.get(&item.url, None).await {
+            Ok(html) => {
+                let parts = Self::extract_all_parts(&html);
+                if !parts.is_empty() {
+                    item.formats = Some(
+                        parts
+                            .into_iter()
+                            .enumerate()
+                            .map(|(i, url)| {
+                                VideoFormat::new(
+                                    url,
+                                    format!("Part {}", i + 1),
+                                    "mp4".to_string(),
+                                )
+                            })
+                            .collect(),
+                    );
+                }
+            }
+            Err(e) => {
+                report_provider_error_background(
+                    "paradisehill",
+                    "enrich_item",
+                    &e.to_string(),
+                );
+            }
+        }
+        item
+    }
+
+    async fn enrich_all(stubs: Vec<VideoItem>, options: ServerOptions) -> Vec<VideoItem> {
+        stream::iter(stubs.into_iter().map(|item| {
+            let opts = options.clone();
+            async move { Self::enrich_item(item, opts).await }
+        }))
+        .buffer_unordered(ENRICH_CONCURRENCY)
+        .collect::<Vec<_>>()
+        .await
+    }
+
     async fn get(
         &self,
         cache: VideoCache,
@@ -57,7 +130,7 @@ impl ParadisehillProvider {
         options: ServerOptions,
     ) -> Result<Vec<VideoItem>> {
         let mut requester =
-            crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
+            requester_or_default(&options, module_path!(), "missing_requester");
 
         let url_str = format!("{}/all/?sort=created_at&page={}", self.url, page);
 
@@ -69,15 +142,13 @@ impl ParadisehillProvider {
                     items.clone()
                 }
             }
-            None => {
+            None => vec![],
-                vec![]
-            }
         };
 
         let text = match requester.get(&url_str, None).await {
             Ok(text) => text,
             Err(e) => {
-                crate::providers::report_provider_error(
+                report_provider_error(
                     "paradisehill",
                     "get.request",
                     &format!("url={url_str}; error={e}"),
@@ -86,17 +157,20 @@ impl ParadisehillProvider {
                 return Ok(old_items);
             }
         };
-        // Pass a reference to options if needed, or reconstruct as needed
+
-        let video_items: Vec<VideoItem> = self
+        let stubs = self.parse_listing_html(&text);
-            .get_video_items_from_html(text.clone(), requester)
+        if stubs.is_empty() {
-            .await;
+            return Ok(old_items);
-        if !video_items.is_empty() {
+        }
+
+        let items = Self::enrich_all(stubs, options).await;
+        if !items.is_empty() {
             cache.remove(&url_str);
-            cache.insert(url_str.clone(), video_items.clone());
+            cache.insert(url_str.clone(), items.clone());
         } else {
             return Ok(old_items);
         }
-        Ok(video_items)
+        Ok(items)
     }
 
     async fn query(
@@ -106,15 +180,14 @@ impl ParadisehillProvider {
         query: &str,
         options: ServerOptions,
     ) -> Result<Vec<VideoItem>> {
-        // Extract needed fields from options at the start
         let mut requester =
-            crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
+            requester_or_default(&options, module_path!(), "missing_requester");
         let search_string = query.replace(" ", "+");
         let url_str = format!(
             "{}/search/?pattern={}&page={}",
             self.url, search_string, page
         );
-        // Check our Video Cache. If the result is younger than 1 hour, we return it.
+
         let old_items = match cache.get(&url_str) {
             Some((time, items)) => {
                 if time.elapsed().unwrap_or_default().as_secs() < 60 * 60 {
@@ -124,14 +197,13 @@ impl ParadisehillProvider {
                     return Ok(items.clone());
                 }
             }
-            None => {
+            None => vec![],
-                vec![]
-            }
         };
+
         let text = match requester.get(&url_str, None).await {
             Ok(text) => text,
             Err(e) => {
-                crate::providers::report_provider_error(
+                report_provider_error(
                     "paradisehill",
                     "query.request",
                     &format!("url={url_str}; error={e}"),
@@ -140,25 +212,24 @@ impl ParadisehillProvider {
                 return Ok(old_items);
             }
         };
-        let video_items: Vec<VideoItem> = self
+
-            .get_video_items_from_html(text.clone(), requester)
+        let stubs = self.parse_listing_html(&text);
-            .await;
+        if stubs.is_empty() {
-        if !video_items.is_empty() {
+            return Ok(old_items);
+        }
+
+        let items = Self::enrich_all(stubs, options).await;
+        if !items.is_empty() {
             cache.remove(&url_str);
-            cache.insert(url_str.clone(), video_items.clone());
+            cache.insert(url_str.clone(), items.clone());
         } else {
             return Ok(old_items);
         }
-        Ok(video_items)
+        Ok(items)
     }
 
-    async fn get_video_items_from_html(
+    fn parse_listing_html(&self, html: &str) -> Vec<VideoItem> {
-        &self,
-        html: String,
-        _requester: Requester,
-    ) -> Vec<VideoItem> {
         if html.is_empty() {
-            println!("HTML is empty");
             return vec![];
         }
         let mut items: Vec<VideoItem> = Vec::new();
@@ -217,7 +288,7 @@ impl ParadisehillProvider {
 
             items.push(
                 VideoItem::new(id, title, video_url, "paradisehill".to_string(), thumb, 0)
-                    .aspect_ratio(0.697674419 as f32)
+                    .aspect_ratio(0.697674419_f32)
                     .tags(tags),
             );
         }
@@ -246,10 +317,7 @@ impl Provider for ParadisehillProvider {
                 self.query(cache, page.parse::<u8>().unwrap_or(1), &q, options)
                     .await
             }
-            None => {
+            None => self.get(cache, page.parse::<u8>().unwrap_or(1), options).await,
-                self.get(cache, page.parse::<u8>().unwrap_or(1), options)
-                    .await
-            }
         };
         match videos {
             Ok(v) => v,