diff --git a/src/providers/rule34video.rs b/src/providers/rule34video.rs index c690e22..85920bb 100644 --- a/src/providers/rule34video.rs +++ b/src/providers/rule34video.rs @@ -1,6 +1,6 @@ use crate::DbPool; use crate::api::ClientVersion; -use crate::providers::{Provider, probe_first_archive, requester_or_default, slug_dashed}; +use crate::providers::{Provider, requester_or_default, slug_dashed}; use crate::status::*; use crate::util::cache::VideoCache; use crate::util::discord::send_discord_error_report; @@ -13,11 +13,22 @@ use error_chain::error_chain; use futures::stream::{self, StreamExt}; use htmlentity::entity::{ICodedDataTrait, decode}; use scraper::{Html, Selector}; -use std::time::{SystemTime, UNIX_EPOCH}; +use std::collections::HashMap; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use percent_encoding::{NON_ALPHANUMERIC, utf8_percent_encode}; use std::vec; -const ENRICH_CONCURRENCY: usize = 6; +const ENRICH_CONCURRENCY: usize = 4; +/// The signed `v-acctoken` format URLs last ~12h; keep details well inside that. +const DETAIL_TTL: Duration = Duration::from_secs(2 * 60 * 60); +const DETAIL_CACHE_SIZE: usize = 5000; +const TARGET_TTL: Duration = Duration::from_secs(6 * 60 * 60); +const TARGET_CACHE_SIZE: usize = 2000; +/// A probed archive smaller than this loses to keyword search: "elf girl" is a +/// one-video tag, while the search finds hundreds. +const MIN_PROBED_ARCHIVE_ITEMS: usize = 4; pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata = crate::providers::ProviderChannelMetadata { @@ -61,15 +72,43 @@ impl Target { } } -#[derive(Debug, Clone)] +#[derive(Clone)] pub struct Rule34videoProvider { url: String, + /// Enriched items (formats, tags, artist) by video URL. Every listing costs + /// ~23 detail pages, again for the prefetched next page; refetching them + /// on each request is what runs into the site's 429 rate limit. + details: VideoCache, + /// Bare query -> resolved target, so later pages skip the archive probes. + targets: Arc>>, } impl Rule34videoProvider { pub fn new() -> Self { + let mut details = VideoCache::new(); + details.max_size(DETAIL_CACHE_SIZE); Rule34videoProvider { url: "https://rule34video.com".to_string(), + details, + targets: Arc::new(Mutex::new(HashMap::new())), + } + } + + fn remembered_target(&self, query: &str) -> Option { + let targets = self.targets.lock().ok()?; + let (time, target) = targets.get(&slug_dashed(query))?; + (time.elapsed() < TARGET_TTL).then(|| target.clone()) + } + + fn remember_target(&self, query: &str, target: &Target) { + if let Ok(mut targets) = self.targets.lock() { + if targets.len() >= TARGET_CACHE_SIZE { + targets.retain(|_, (time, _)| time.elapsed() < TARGET_TTL); + if targets.len() >= TARGET_CACHE_SIZE { + targets.clear(); + } + } + targets.insert(slug_dashed(query), (Instant::now(), target.clone())); } } @@ -170,29 +209,37 @@ impl Rule34videoProvider { /// A free-form query that names an artist, category or tag goes to that /// archive (probed on the site; unknown names 404), else keyword search. - async fn resolve_keyword(&self, query: &str, options: &ServerOptions) -> Target { + /// The second value is false when a probe failed (rate limit, outage) and + /// the answer must not be remembered. + async fn resolve_keyword(&self, query: &str, options: &ServerOptions) -> (Target, bool) { let slug = slug_dashed(query); if slug.is_empty() { - return Target::Search(query.trim().to_string()); + return (Target::Search(query.trim().to_string()), true); } let candidates = [ Target::Artist(slug.clone()), Target::Category(slug.clone()), Target::Tag(slug), ]; - let urls: Vec = candidates - .iter() - .filter_map(|t| t.archive_path()) - .map(|path| format!("{}/{path}/", self.url)) - .collect(); let mut requester = requester_or_default(options, "rule34video", "resolve_keyword"); let referer = format!("{}/", self.url); - if let Some((index, _)) = probe_first_archive(&mut requester, &urls, &referer, |body| { - body.contains("item thumb video_") - }) - .await - { - return candidates[index].clone(); + let mut conclusive = true; + for candidate in &candidates { + let Some(path) = candidate.archive_path() else { + continue; + }; + let url = format!("{}/{path}/", self.url); + let headers = vec![("Referer".to_string(), referer.clone())]; + match requester.get_probe(&url, headers).await { + Ok(Some(body)) => { + let items = self.get_video_items_from_html(body).len(); + if items >= MIN_PROBED_ARCHIVE_ITEMS { + return (candidate.clone(), true); + } + } + Ok(None) => {} + Err(_) => conclusive = false, + } } // Characters are tagged "name (franchise)": ask the site's own tag search. @@ -205,13 +252,17 @@ impl Rule34videoProvider { ("Referer".to_string(), referer), ("X-Requested-With".to_string(), "XMLHttpRequest".to_string()), ]; - if let Ok(Some(body)) = requester.get_probe(&suggest_url, headers).await { - let titles = Self::parse_tag_suggestions(&body); - if let Some(title) = Self::best_tag_title(&titles, query) { - return Target::Tag(slug_dashed(&title)); + match requester.get_probe(&suggest_url, headers).await { + Ok(Some(body)) => { + let titles = Self::parse_tag_suggestions(&body); + if let Some(title) = Self::best_tag_title(&titles, query) { + return (Target::Tag(slug_dashed(&title)), conclusive); + } } + Ok(None) => {} + Err(_) => conclusive = false, } - Target::Search(query.trim().to_string()) + (Target::Search(query.trim().to_string()), conclusive) } async fn resolve_target(&self, query: Option<&str>, options: &ServerOptions) -> Target { @@ -221,7 +272,14 @@ impl Rule34videoProvider { if let Some(target) = Self::parse_prefixed(query) { return target; } - self.resolve_keyword(query, options).await + if let Some(target) = self.remembered_target(query) { + return target; + } + let (target, conclusive) = self.resolve_keyword(query, options).await; + if conclusive { + self.remember_target(query, &target); + } + target } async fn archive( @@ -254,20 +312,35 @@ impl Rule34videoProvider { self.url ); let headers = vec![("Referer".to_string(), format!("{}/{path}/", self.url))]; + let stale = || cache.get(&index).map(|(_, items)| items).unwrap_or_default(); let text = match requester.get_probe(&url, headers).await { Ok(Some(text)) => text, // A 404 is an unknown archive (or a page past the end). Ok(None) => return Ok(vec![]), - Err(e) => return Err(format!("archive fetch failed for {url}: {e}").into()), + Err(e) => { + eprintln!("rule34video archive fetch failed for {url}: {e}"); + return Ok(stale()); + } }; let video_items = self.get_video_items_from_html(text); let video_items = self.enrich_items(video_items, requester).await; - if !video_items.is_empty() { - cache.insert(index, video_items.clone()); + if video_items.is_empty() { + return Ok(stale()); } + cache.insert(index, video_items.clone()); Ok(video_items) } + /// `/search/{q}/` takes dashes for spaces. A raw `/` splits the path (404), + /// `%20` is refused (403) and `&` cuts the query short, so keep only words. + fn search_path_segment(query: &str) -> String { + slug_dashed(query) + .split('-') + .map(|word| utf8_percent_encode(word, NON_ALPHANUMERIC).to_string()) + .collect::>() + .join("-") + } + /// Helper to safely extract a string between two delimiters fn extract_between<'a>(content: &'a str, start_pat: &str, end_pat: &str) -> Option<&'a str> { let start_idx = content.find(start_pat)? + start_pat.len(); @@ -383,7 +456,7 @@ impl Rule34videoProvider { let url = format!( "{}/search/{}/?mode=async&function=get_block&block_id=custom_list_videos_videos_list_search&tag_ids=&sort_by={}&from_videos={}&from_albums={}&_={}", self.url, - query.replace(" ", "-"), + Self::search_path_segment(query), sort_val, page, page, @@ -513,25 +586,54 @@ impl Rule34videoProvider { artists } + fn cached_detail(&self, item: &mut VideoItem) -> bool { + let Some((time, cached)) = self.details.get(&item.url) else { + return false; + }; + let Some(cached) = cached.into_iter().next() else { + return false; + }; + if time.elapsed().unwrap_or(DETAIL_TTL) >= DETAIL_TTL { + return false; + } + item.formats = cached.formats; + item.tags = cached.tags; + item.uploader = cached.uploader; + item.uploaderUrl = cached.uploaderUrl; + item.uploaderId = cached.uploaderId; + true + } + async fn enrich_items(&self, items: Vec, requester: Requester) -> Vec { let referer = format!("{}/", self.url); let base = self.url.clone(); + // Once the site rate-limits a detail page, the rest of the batch keeps + // the listing data instead of digging the hole deeper. + let throttled = Arc::new(AtomicBool::new(false)); stream::iter(items.into_iter().map(|mut item| { let mut requester = requester.clone(); let referer = referer.clone(); let base = base.clone(); + let throttled = throttled.clone(); async move { - if item.url.is_empty() { + if item.url.is_empty() || self.cached_detail(&mut item) { return item; } - let html = requester - .get_with_headers( - &item.url, - vec![("Referer".to_string(), referer)], - None, - ) - .await - .unwrap_or_default(); + if throttled.load(Ordering::Relaxed) { + return item; + } + // `get_probe` hands challenges to the full fallback chain but + // reports a lasting 429 as an error rather than escalating every + // detail page to FlareSolverr. + let headers = vec![("Referer".to_string(), referer)]; + let html = match requester.get_probe(&item.url, headers).await { + Ok(Some(html)) => html, + Ok(None) => return item, + Err(_) => { + throttled.store(true, Ordering::Relaxed); + return item; + } + }; let (formats, mut tags) = Self::parse_detail(&html); if !formats.is_empty() { item.formats = Some(formats); @@ -550,6 +652,9 @@ impl Rule34videoProvider { if !tags.is_empty() { item.tags = Some(tags); } + if item.formats.is_some() { + self.details.insert(item.url.clone(), vec![item.clone()]); + } item } })) @@ -563,15 +668,15 @@ impl Rule34videoProvider { return vec![]; } - // Safely isolate the video listing section - let video_listing = match Self::extract_between( - &html, - "id=\"custom_list_videos", - "
content, - None => return vec![], + // Isolate the video listing section. A result that fits on one page has + // no pagination block, so the listing then runs to the end. + let Some(start) = html.find("id=\"custom_list_videos") else { + return vec![]; }; + let video_listing = &html[start..]; + let video_listing = video_listing + .find("
"#; + let items = Rule34videoProvider::new().get_video_items_from_html(html.to_string()); + assert_eq!(items.len(), 1); + assert_eq!(items[0].id, "7"); + } + + #[test] + fn search_segment_uses_dashes_and_drops_path_breaking_punctuation() { + assert_eq!(Rule34videoProvider::search_path_segment("elf girl"), "elf-girl"); + assert_eq!(Rule34videoProvider::search_path_segment("2b/9s"), "2b-9s"); + assert_eq!(Rule34videoProvider::search_path_segment("Tifa & Aerith"), "tifa-aerith"); + assert_eq!(Rule34videoProvider::search_path_segment("Pokémon"), "pok%C3%A9mon"); + } + + #[test] + fn remembers_resolved_targets_by_slug() { + let provider = Rule34videoProvider::new(); + provider.remember_target("Samus Aran", &Target::Tag("samus-aran-metroid".into())); + assert_eq!( + provider.remembered_target("samus aran"), + Some(Target::Tag("samus-aran-metroid".into())) + ); + assert_eq!(provider.remembered_target("samus"), None); + } } diff --git a/src/util/requester.rs b/src/util/requester.rs index bca2843..e5d015b 100644 --- a/src/util/requester.rs +++ b/src/util/requester.rs @@ -674,6 +674,18 @@ impl Requester { url::Url::parse(url).ok().map(|u| u.path().to_string()) } + /// `Url` percent-encodes a non-ASCII path ("/tags/pok%C3%A9mon/") while the + /// response URI keeps it raw ("/tags/pokémon/"): compare them decoded. + fn same_path(asked: &str, landed: &str) -> bool { + let normalize = |path: &str| { + percent_encoding::percent_decode_str(path) + .decode_utf8_lossy() + .trim_end_matches('/') + .to_lowercase() + }; + normalize(asked) == normalize(landed) + } + /// Existence probe for archive pages (pornstar / studio / tag / category). /// /// Reads the raw response so a real 404/410 answers immediately with @@ -687,7 +699,11 @@ impl Requester { headers: Vec<(String, String)>, ) -> Result, AnyErr> { let mut last_err = String::new(); - for _ in 0..2 { + for attempt in 0..2 { + if attempt > 0 { + // An immediate retry of a 429 is all but certain to hit it again. + tokio::time::sleep(std::time::Duration::from_millis(1500)).await; + } match self.get_raw_with_headers(url, headers.clone()).await { Ok(response) => { let status = response.status().as_u16(); @@ -696,9 +712,7 @@ impl Requester { // lands on a different path is a miss. let redirected_away = Self::path_of(url) .zip(Some(response.uri().path().to_string())) - .is_some_and(|(asked, landed)| { - !asked.trim_end_matches('/').eq_ignore_ascii_case(landed.trim_end_matches('/')) - }); + .is_some_and(|(asked, landed)| !Self::same_path(&asked, &landed)); let body = response.text().await.unwrap_or_default(); if status == 404 || status == 410 { return Ok(None); @@ -893,7 +907,9 @@ impl Requester { rate_limit_retries, MAX_RATE_LIMIT_RETRIES ); - tokio::time::sleep(std::time::Duration::from_secs(1)).await; + // 1s, 2s, 4s: a site-wide limit needs longer than a second to lift. + tokio::time::sleep(std::time::Duration::from_secs(1 << (rate_limit_retries - 1))) + .await; continue; } else { crate::flow_debug!( @@ -1027,6 +1043,14 @@ impl Requester { mod tests { use super::Requester; + #[test] + fn probe_paths_compare_percent_decoded() { + let asked = Requester::path_of("https://rule34video.com/tags/pokémon-species/").unwrap(); + assert!(Requester::same_path(&asked, "/tags/pokémon-species/")); + assert!(Requester::same_path("/Models/X", "/models/x/")); + assert!(!Requester::same_path(&asked, "/search/")); + } + #[test] fn new_requesters_share_cookie_jar() { let a = Requester::new();