hsex page >1 fix

src/providers/hsex.rs

@@ -10,14 +10,16 @@ use crate::util::requester::Requester;
 use crate::util::time::parse_time_to_seconds;
 use crate::videos::{ServerOptions, VideoFormat, VideoItem};
 use async_trait::async_trait;
-use chrono::{DateTime, Duration, NaiveDate, Utc};
+use chrono::{DateTime, Duration as ChronoDuration, NaiveDate, Utc};
 use error_chain::error_chain;
 use futures::stream::{self, StreamExt};
 use htmlentity::entity::{ICodedDataTrait, decode};
 use regex::Regex;
 use scraper::{ElementRef, Html, Selector};
 use std::sync::{Arc, RwLock};
+use std::time::Duration as StdDuration;
 use std::{thread, vec};
+use tokio::time::timeout;
 use url::Url;
 use wreq::Version;
 
@@ -546,11 +548,11 @@ impl HsexProvider {
         let unit = captures.get(2)?.as_str();
         let now = Utc::now();
         let timestamp = match unit {
-            "分钟" => now - Duration::minutes(amount),
+            "分钟" => now - ChronoDuration::minutes(amount),
-            "小时" => now - Duration::hours(amount),
+            "小时" => now - ChronoDuration::hours(amount),
-            "天" => now - Duration::days(amount),
+            "天" => now - ChronoDuration::days(amount),
-            "月" => now - Duration::days(amount * 30),
+            "月" => now - ChronoDuration::days(amount * 30),
-            "年" => now - Duration::days(amount * 365),
+            "年" => now - ChronoDuration::days(amount * 365),
             _ => return None,
         };
         Some(timestamp.timestamp() as u64)
@@ -786,8 +788,13 @@ impl HsexProvider {
 
     async fn enrich_video(&self, item: VideoItem, options: &ServerOptions) -> VideoItem {
         let mut requester = requester_or_default(options, CHANNEL_ID, "enrich_video");
-        match self.fetch_html(&mut requester, &item.url, &format!("{}/", self.url)).await {
+        let detail_fetch = timeout(
-            Ok(html) => match self.apply_detail_video(item.clone(), &html, &item.url) {
+            StdDuration::from_secs(6),
+            self.fetch_html(&mut requester, &item.url, &format!("{}/", self.url)),
+        )
+        .await;
+        match detail_fetch {
+            Ok(Ok(html)) => match self.apply_detail_video(item.clone(), &html, &item.url) {
                 Ok(enriched) => enriched,
                 Err(error) => {
                     report_provider_error_background(
@@ -798,10 +805,14 @@ impl HsexProvider {
                     item
                 }
             },
-            Err(error) => {
+            Ok(Err(error)) => {
                 report_provider_error_background(CHANNEL_ID, "fetch_detail", &error.to_string());
                 item
             }
+            Err(_) => {
+                report_provider_error_background(CHANNEL_ID, "fetch_detail_timeout", &item.url);
+                item
+            }
         }
     }
 
@@ -810,6 +821,7 @@ impl HsexProvider {
         cache: VideoCache,
         url: String,
         per_page_limit: usize,
+        enrich_details: bool,
         options: &ServerOptions,
     ) -> Result<Vec<VideoItem>> {
         if let Some((time, items)) = cache.get(&url) {
@@ -819,7 +831,12 @@ impl HsexProvider {
         }
 
         let mut requester = requester_or_default(options, CHANNEL_ID, "fetch_items_for_url");
-        let html = self.fetch_html(&mut requester, &url, &format!("{}/", self.url)).await?;
+        let html = timeout(
+            StdDuration::from_secs(10),
+            self.fetch_html(&mut requester, &url, &format!("{}/", self.url)),
+        )
+        .await
+        .map_err(|_| Error::from(format!("list request timed out for {url}")))??;
         let list_items = self.parse_list_videos(&html)?;
         if list_items.is_empty() {
             return Ok(vec![]);
@@ -830,6 +847,11 @@ impl HsexProvider {
             .take(per_page_limit.max(1))
             .collect::<Vec<_>>();
 
+        if !enrich_details {
+            cache.insert(url, limited_items.clone());
+            return Ok(limited_items);
+        }
+
         let items = stream::iter(limited_items.into_iter().map(|item| {
             let provider = self.clone();
             let options = options.clone();
@@ -856,7 +878,7 @@ impl HsexProvider {
     ) -> Result<Vec<VideoItem>> {
         let target = self.resolve_option_target(&options, sort);
         let url = self.build_url_for_target(&target, page);
-        self.fetch_items_for_url(cache, url, per_page_limit, &options)
+        self.fetch_items_for_url(cache, url, per_page_limit, page <= 1, &options)
             .await
     }
 
@@ -871,7 +893,7 @@ impl HsexProvider {
     ) -> Result<Vec<VideoItem>> {
         let target = self.resolve_query_target(query, sort);
         let url = self.build_url_for_target(&target, page);
-        self.fetch_items_for_url(cache, url, per_page_limit, &options)
+        self.fetch_items_for_url(cache, url, per_page_limit, page <= 1, &options)
             .await
     }
 }
@@ -917,6 +939,8 @@ impl Provider for HsexProvider {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::util::cache::VideoCache;
+    use crate::util::requester::Requester;
 
     fn provider() -> HsexProvider {
         HsexProvider {
@@ -995,4 +1019,32 @@ mod tests {
             Some("https://cdn1.hdcdn.online/hls/1183662/index.m3u8")
         );
     }
+
+    #[tokio::test]
+    #[ignore]
+    async fn fetches_page_two_items() {
+        let provider = provider();
+        let options = ServerOptions {
+            featured: None,
+            category: None,
+            sites: None,
+            filter: Some("new".to_string()),
+            language: None,
+            public_url_base: None,
+            requester: Some(Requester::new()),
+            network: None,
+            stars: None,
+            categories: None,
+            duration: None,
+            sort: Some("new".to_string()),
+            sexuality: None,
+        };
+
+        let videos = provider
+            .get(VideoCache::new(), 2, "new", 10, options)
+            .await
+            .expect("page 2 should load");
+
+        assert!(!videos.is_empty(), "page 2 should return items");
+    }
 }