diff --git a/src/providers/hsex.rs b/src/providers/hsex.rs index 6f44525..9a2b68e 100644 --- a/src/providers/hsex.rs +++ b/src/providers/hsex.rs @@ -10,14 +10,16 @@ use crate::util::requester::Requester; use crate::util::time::parse_time_to_seconds; use crate::videos::{ServerOptions, VideoFormat, VideoItem}; use async_trait::async_trait; -use chrono::{DateTime, Duration, NaiveDate, Utc}; +use chrono::{DateTime, Duration as ChronoDuration, NaiveDate, Utc}; use error_chain::error_chain; use futures::stream::{self, StreamExt}; use htmlentity::entity::{ICodedDataTrait, decode}; use regex::Regex; use scraper::{ElementRef, Html, Selector}; use std::sync::{Arc, RwLock}; +use std::time::Duration as StdDuration; use std::{thread, vec}; +use tokio::time::timeout; use url::Url; use wreq::Version; @@ -546,11 +548,11 @@ impl HsexProvider { let unit = captures.get(2)?.as_str(); let now = Utc::now(); let timestamp = match unit { - "分钟" => now - Duration::minutes(amount), - "小时" => now - Duration::hours(amount), - "天" => now - Duration::days(amount), - "月" => now - Duration::days(amount * 30), - "年" => now - Duration::days(amount * 365), + "分钟" => now - ChronoDuration::minutes(amount), + "小时" => now - ChronoDuration::hours(amount), + "天" => now - ChronoDuration::days(amount), + "月" => now - ChronoDuration::days(amount * 30), + "年" => now - ChronoDuration::days(amount * 365), _ => return None, }; Some(timestamp.timestamp() as u64) @@ -786,8 +788,13 @@ impl HsexProvider { async fn enrich_video(&self, item: VideoItem, options: &ServerOptions) -> VideoItem { let mut requester = requester_or_default(options, CHANNEL_ID, "enrich_video"); - match self.fetch_html(&mut requester, &item.url, &format!("{}/", self.url)).await { - Ok(html) => match self.apply_detail_video(item.clone(), &html, &item.url) { + let detail_fetch = timeout( + StdDuration::from_secs(6), + self.fetch_html(&mut requester, &item.url, &format!("{}/", self.url)), + ) + .await; + match detail_fetch { + Ok(Ok(html)) => match self.apply_detail_video(item.clone(), &html, &item.url) { Ok(enriched) => enriched, Err(error) => { report_provider_error_background( @@ -798,10 +805,14 @@ impl HsexProvider { item } }, - Err(error) => { + Ok(Err(error)) => { report_provider_error_background(CHANNEL_ID, "fetch_detail", &error.to_string()); item } + Err(_) => { + report_provider_error_background(CHANNEL_ID, "fetch_detail_timeout", &item.url); + item + } } } @@ -810,6 +821,7 @@ impl HsexProvider { cache: VideoCache, url: String, per_page_limit: usize, + enrich_details: bool, options: &ServerOptions, ) -> Result> { if let Some((time, items)) = cache.get(&url) { @@ -819,7 +831,12 @@ impl HsexProvider { } let mut requester = requester_or_default(options, CHANNEL_ID, "fetch_items_for_url"); - let html = self.fetch_html(&mut requester, &url, &format!("{}/", self.url)).await?; + let html = timeout( + StdDuration::from_secs(10), + self.fetch_html(&mut requester, &url, &format!("{}/", self.url)), + ) + .await + .map_err(|_| Error::from(format!("list request timed out for {url}")))??; let list_items = self.parse_list_videos(&html)?; if list_items.is_empty() { return Ok(vec![]); @@ -830,6 +847,11 @@ impl HsexProvider { .take(per_page_limit.max(1)) .collect::>(); + if !enrich_details { + cache.insert(url, limited_items.clone()); + return Ok(limited_items); + } + let items = stream::iter(limited_items.into_iter().map(|item| { let provider = self.clone(); let options = options.clone(); @@ -856,7 +878,7 @@ impl HsexProvider { ) -> Result> { let target = self.resolve_option_target(&options, sort); let url = self.build_url_for_target(&target, page); - self.fetch_items_for_url(cache, url, per_page_limit, &options) + self.fetch_items_for_url(cache, url, per_page_limit, page <= 1, &options) .await } @@ -871,7 +893,7 @@ impl HsexProvider { ) -> Result> { let target = self.resolve_query_target(query, sort); let url = self.build_url_for_target(&target, page); - self.fetch_items_for_url(cache, url, per_page_limit, &options) + self.fetch_items_for_url(cache, url, per_page_limit, page <= 1, &options) .await } } @@ -917,6 +939,8 @@ impl Provider for HsexProvider { #[cfg(test)] mod tests { use super::*; + use crate::util::cache::VideoCache; + use crate::util::requester::Requester; fn provider() -> HsexProvider { HsexProvider { @@ -995,4 +1019,32 @@ mod tests { Some("https://cdn1.hdcdn.online/hls/1183662/index.m3u8") ); } + + #[tokio::test] + #[ignore] + async fn fetches_page_two_items() { + let provider = provider(); + let options = ServerOptions { + featured: None, + category: None, + sites: None, + filter: Some("new".to_string()), + language: None, + public_url_base: None, + requester: Some(Requester::new()), + network: None, + stars: None, + categories: None, + duration: None, + sort: Some("new".to_string()), + sexuality: None, + }; + + let videos = provider + .get(VideoCache::new(), 2, "new", 10, options) + .await + .expect("page 2 should load"); + + assert!(!videos.is_empty(), "page 2 should return items"); + } }