fixes and cleanup

src/providers/hqporner.rs

@@ -9,10 +9,9 @@ use crate::util::time::parse_time_to_seconds;
 use crate::videos::{ServerOptions, VideoFormat, VideoItem};
 use async_trait::async_trait;
 use error_chain::error_chain;
-use futures::future::join_all;
+use futures::stream::{FuturesUnordered, StreamExt};
 use htmlentity::entity::{ICodedDataTrait, decode};
 use std::sync::{Arc, RwLock};
-use std::thread::sleep;
 use std::{thread, vec};
 use titlecase::Titlecase;
 
@@ -263,17 +262,37 @@ impl HqpornerProvider {
             })
             .unwrap_or_default();
 
-        let futures = raw_videos
-            .into_iter()
-            .map(|el| self.get_video_item(el, requester.clone()));
+        // Limit concurrent detail-page requests to reduce transient connect errors.
+        let mut in_flight = FuturesUnordered::new();
+        let mut iter = raw_videos.into_iter();
+        let mut items = Vec::new();
+        const MAX_IN_FLIGHT: usize = 6;
 
-        join_all(futures)
-            .await
-            .into_iter()
-            .inspect(|r| {
-                if let Err(e) = r {
+        loop {
+            while in_flight.len() < MAX_IN_FLIGHT {
+                let Some(seg) = iter.next() else {
+                    break;
+                };
+                in_flight.push(self.get_video_item(seg, requester.clone()));
+            }
+
+            let Some(result) = in_flight.next().await else {
+                break;
+            };
+            match result {
+                Ok(item)
+                    if item
+                        .formats
+                        .as_ref()
+                        .map(|formats| !formats.is_empty())
+                        .unwrap_or(false) =>
+                {
+                    items.push(item);
+                }
+                Ok(_) => {}
+                Err(e) => {
                     let msg = e.to_string();
-                    let chain = format_error_chain(e);
+                    let chain = format_error_chain(&e);
                     tokio::spawn(async move {
                         let _ = send_discord_error_report(
                             msg,
@@ -287,10 +306,10 @@ impl HqpornerProvider {
                         .await;
                     });
                 }
-            })
-            .filter_map(Result::ok)
-            .filter(|item| item.formats.as_ref().map(|formats| !formats.is_empty()).unwrap_or(false))
-            .collect()
+            }
+        }
+
+        items
     }
 
     async fn get_video_item(&self, seg: String, mut requester: Requester) -> Result<VideoItem> {
@@ -319,13 +338,22 @@ impl HqpornerProvider {
             .and_then(|s| s.split('.').next())
             .ok_or_else(|| ErrorKind::Parse(format!("id \n{seg}").into()))?
             .to_string();
-        let thumb = format!(
-            "https:{}",
-            seg.split("onmouseleave='defaultImage(\"")
-                .nth(1)
-                .and_then(|s| s.split('"').next())
-                .ok_or_else(|| ErrorKind::Parse(format!("thumb \n{seg}").into()))?
-        );
+        let thumb_raw = seg
+            .split("onmouseleave='defaultImage(\"")
+            .nth(1)
+            .and_then(|s| s.split('"').next())
+            .ok_or_else(|| ErrorKind::Parse(format!("thumb \n{seg}").into()))?;
+        let thumb_abs = if thumb_raw.starts_with("//") {
+            format!("https:{}", thumb_raw)
+        } else if thumb_raw.starts_with("http://") || thumb_raw.starts_with("https://") {
+            thumb_raw.to_string()
+        } else {
+            format!("https://{}", thumb_raw.trim_start_matches('/'))
+        };
+        let thumb = match thumb_abs.strip_prefix("https://") {
+            Some(path) => format!("https://hottub.spacemoehre.de/proxy/hqporner-thumb/{path}"),
+            None => thumb_abs,
+        };
         let raw_duration = seg
             .split("<span class=\"icon fa-clock-o meta-data\">")
             .nth(1)
@@ -335,7 +363,7 @@ impl HqpornerProvider {
         let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
 
         let (tags, formats) = self.extract_media(&video_url, &mut requester).await?;
-        
+
         Ok(
             VideoItem::new(id, title, video_url, "hqporner".into(), thumb, duration)
                 .formats(formats)
@@ -350,17 +378,35 @@ impl HqpornerProvider {
     ) -> Result<(Vec<String>, Vec<VideoFormat>)> {
         let mut formats = vec![];
         let mut tags = vec![];
-        let resp = requester
-            .get_raw_with_headers(
-                url,
-                vec![("Referer".to_string(), "https://hqporner.com/".into())],
-            )
+        let headers = vec![("Referer".to_string(), "https://hqporner.com/".into())];
+        let mut text = match self
+            .fetch_text_with_retries(requester, url, &headers, 3)
             .await
-            .map_err(|e| Error::from(format!("Request failed: {}", e)))?;
-        let text = resp
-            .text()
-            .await
-            .map_err(|e| Error::from(format!("Text conversion failed: {}", e)))?;
+        {
+            Ok(text) => text,
+            Err(primary_err) => {
+                if url.contains("://hqporner.com/") {
+                    let fallback_url = url.replace("://hqporner.com/", "://www.hqporner.com/");
+                    self.fetch_text_with_retries(requester, &fallback_url, &headers, 3)
+                        .await
+                        .map_err(|fallback_err| {
+                            Error::from(format!(
+                                "Request failed: primary={primary_err}; fallback={fallback_err}"
+                            ))
+                        })?
+                } else {
+                    return Err(Error::from(format!("Request failed: {}", primary_err)));
+                }
+            }
+        };
+
+        if text.is_empty() && url.contains("://hqporner.com/") {
+            let fallback_url = url.replace("://hqporner.com/", "://www.hqporner.com/");
+            text = self
+                .fetch_text_with_retries(requester, &fallback_url, &headers, 3)
+                .await
+                .unwrap_or_default();
+        }
 
         if text.contains("Why do I see it?") {
             return Ok((tags, formats));
@@ -395,22 +441,11 @@ impl HqpornerProvider {
                 .and_then(|s| s.split('\'').next())
                 .ok_or("No player link")?
         );
-        let mut r = requester
-            .get_raw_with_headers(
-                &player_url,
-                vec![("Referer".to_string(), "https://hqporner.com/".into())],
-            ).await;
-        
-        if let Err(_e) = &r {
-            sleep(std::time::Duration::from_secs(1));
-            r = requester
-            .get_raw_with_headers(
-                &player_url,
-                vec![("Referer".to_string(), "https://hqporner.com/".into())],
-            ).await;
-        }
-        let response = match r {
-            Ok(response) => response,
+        let response_text = match self
+            .fetch_text_with_retries(requester, &player_url, &headers, 2)
+            .await
+        {
+            Ok(text) => text,
             Err(e) => {
                 let err = format!("altplayer request failed: {e}");
                 send_discord_error_report(
@@ -426,13 +461,10 @@ impl HqpornerProvider {
                 return Ok((tags, formats));
             }
         };
-        let text2 = response
-            .text()
-            .await
-            .map_err(|e| Error::from(format!("Text conversion failed: {}", e)))?;
+        let text2 = response_text;
 
         // Check for error response
-        if text2.starts_with("ERR:"){
+        if text2.starts_with("ERR:") {
             return Ok((tags, formats));
         }
 
@@ -467,6 +499,37 @@ impl HqpornerProvider {
 
         Ok((tags, formats))
     }
+
+    async fn fetch_text_with_retries(
+        &self,
+        requester: &mut Requester,
+        url: &str,
+        headers: &[(String, String)],
+        max_attempts: u8,
+    ) -> std::result::Result<String, String> {
+        let mut last_err = String::new();
+
+        for attempt in 1..=max_attempts {
+            match requester.get_raw_with_headers(url, headers.to_vec()).await {
+                Ok(resp) => match resp.text().await {
+                    Ok(text) => return Ok(text),
+                    Err(e) => {
+                        last_err =
+                            format!("text read failed (attempt {attempt}/{max_attempts}): {e}");
+                    }
+                },
+                Err(e) => {
+                    last_err = format!("request failed (attempt {attempt}/{max_attempts}): {e}");
+                }
+            }
+
+            if attempt < max_attempts {
+                tokio::time::sleep(std::time::Duration::from_millis(250 * attempt as u64)).await;
+            }
+        }
+
+        Err(last_err)
+    }
 }
 
 #[async_trait]
@@ -488,7 +551,15 @@ impl Provider for HqpornerProvider {
         };
         res.unwrap_or_else(|e| {
             eprintln!("Hqporner error: {e}");
-            let _ = send_discord_error_report(e.to_string(), Some(format_error_chain(&e)), None, None, file!(), line!(), module_path!());
+            let _ = send_discord_error_report(
+                e.to_string(),
+                Some(format_error_chain(&e)),
+                None,
+                None,
+                file!(),
+                line!(),
+                module_path!(),
+            );
             vec![]
         })
     }