diff --git a/src/providers/rule34video.rs b/src/providers/rule34video.rs index 4305b8a..be7bcba 100644 --- a/src/providers/rule34video.rs +++ b/src/providers/rule34video.rs @@ -5,25 +5,21 @@ use crate::status::*; use crate::util::cache::VideoCache; use crate::util::discord::send_discord_error_report; use crate::util::parse_abbreviated_number; -use crate::util::requester::Requester; use crate::util::time::parse_time_to_seconds; -use crate::videos::{ServerOptions, VideoFormat, VideoItem}; +use crate::videos::{ServerOptions, VideoItem}; use async_trait::async_trait; use error_chain::error_chain; -use futures::stream::{self, StreamExt}; use htmlentity::entity::{ICodedDataTrait, decode}; use scraper::{Html, Selector}; use std::collections::HashMap; -use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use percent_encoding::{NON_ALPHANUMERIC, utf8_percent_encode}; use std::vec; -const ENRICH_CONCURRENCY: usize = 4; -/// The signed `v-acctoken` format URLs last ~12h; keep details well inside that. -const DETAIL_TTL: Duration = Duration::from_secs(2 * 60 * 60); -const DETAIL_CACHE_SIZE: usize = 5000; +/// Hands out `/proxy/rule34video/` URLs instead of the page URLs. Switched off; +/// the proxy and resolver stay in place (`proxies/rule34video.rs`). +const USE_MEDIA_PROXY: bool = false; const TARGET_TTL: Duration = Duration::from_secs(6 * 60 * 60); const TARGET_CACHE_SIZE: usize = 2000; /// A probed archive smaller than this loses to keyword search: "elf girl" is a @@ -75,21 +71,14 @@ impl Target { #[derive(Clone)] pub struct Rule34videoProvider { url: String, - /// Enriched items (formats, tags, artist) by video URL. Every listing costs - /// ~23 detail pages, again for the prefetched next page; refetching them - /// on each request is what runs into the site's 429 rate limit. - details: VideoCache, /// Bare query -> resolved target, so later pages skip the archive probes. targets: Arc>>, } impl Rule34videoProvider { pub fn new() -> Self { - let mut details = VideoCache::new(); - details.max_size(DETAIL_CACHE_SIZE); Rule34videoProvider { url: "https://rule34video.com".to_string(), - details, targets: Arc::new(Mutex::new(HashMap::new())), } } @@ -325,7 +314,7 @@ impl Rule34videoProvider { } }; let video_items = self.get_video_items_from_html(text); - let video_items = self.enrich_items(video_items, requester).await; + let video_items = Self::proxy_items(video_items, &options); if video_items.is_empty() { return Ok(stale()); } @@ -406,7 +395,7 @@ impl Rule34videoProvider { "".to_string() }); let video_items = self.get_video_items_from_html(text); - let video_items = self.enrich_items(video_items, requester).await; + let video_items = Self::proxy_items(video_items, &options); if !video_items.is_empty() { cache.insert(index, video_items.clone()); @@ -479,7 +468,7 @@ impl Rule34videoProvider { "".to_string() }); let video_items = self.get_video_items_from_html(text); - let video_items = self.enrich_items(video_items, requester).await; + let video_items = Self::proxy_items(video_items, &options); if !video_items.is_empty() { cache.insert(index, video_items.clone()); @@ -492,177 +481,18 @@ impl Rule34videoProvider { } } - /// Reads a single-quoted JS string value (`key: '...'`) from the - /// `flashvars` object, honouring `\'` escapes. - fn flashvar(block: &str, key: &str) -> Option { - let marker = format!("{}: '", key); - let start = block.find(&marker)? + marker.len(); - let mut out = String::new(); - let mut chars = block[start..].chars(); - while let Some(c) = chars.next() { - match c { - '\\' => out.push(chars.next()?), - '\'' => return Some(out), - other => out.push(other), + /// Playback resolves the video page on demand through `/proxy/rule34video/`, + /// instead of fetching every listed page up front (which ran into 429s). + fn proxy_items(mut items: Vec, options: &ServerOptions) -> Vec { + if !USE_MEDIA_PROXY { + return items; + } + for item in &mut items { + if let Some(path) = item.url.split("/video/").nth(1).filter(|p| !p.is_empty()) { + item.url = crate::providers::build_proxy_url(options, "rule34video", path); } } - None - } - - /// The page URL is no longer resolvable by yt-dlp's Rule34Video - /// extractor ("No video formats found"), so pull the signed mp4 URLs - /// (360p/480p/720p, valid ~12h, no cookie/Referer needed) and the tags - /// straight from the detail page's `flashvars`. - fn parse_detail(html: &str) -> (Vec, Vec) { - let Some(start) = html.find("var flashvars") else { - return (vec![], vec![]); - }; - let block = &html[start..]; - let block = &block[..block.len().min(6000)]; - - let mut formats: Vec<(u32, VideoFormat)> = Vec::new(); - for (url_key, text_key) in [ - ("video_url", "video_url_text"), - ("video_alt_url", "video_alt_url_text"), - ("video_alt_url2", "video_alt_url2_text"), - ("video_alt_url3", "video_alt_url3_text"), - ] { - let Some(url) = Self::flashvar(block, url_key) else { - continue; - }; - if !url.starts_with("http") { - continue; - } - let quality = Self::flashvar(block, text_key).unwrap_or_else(|| "auto".to_string()); - let height = quality.trim_end_matches('p').parse::().unwrap_or(0); - let mut format = VideoFormat::new(url, quality.clone(), "mp4".to_string()) - .format_id(format!("mp4-{}", quality)); - if height > 0 { - format = format.height(height); - } - formats.push((height, format)); - } - formats.sort_by(|a, b| b.0.cmp(&a.0)); - - let mut tags: Vec = Vec::new(); - for key in ["video_categories", "video_tags"] { - for tag in Self::flashvar(block, key).unwrap_or_default().split(',') { - let tag = tag.trim(); - if !tag.is_empty() && !tags.iter().any(|t| t.eq_ignore_ascii_case(tag)) { - tags.push(tag.to_string()); - } - } - } - - (formats.into_iter().map(|(_, f)| f).collect(), tags) - } - - /// Artists credited on a video page: `(name, slug)` from the "Artist" group. - fn parse_artists(html: &str) -> Vec<(String, String)> { - let document = Html::parse_document(html); - let Ok(group_sel) = Selector::parse("[data-suggest-type=\"model\"] a[href*=\"/models/\"]") - else { - return vec![]; - }; - let name_sel = Selector::parse(".name").ok(); - let mut artists: Vec<(String, String)> = Vec::new(); - for link in document.select(&group_sel) { - let href = link.value().attr("href").unwrap_or_default(); - let slug = href - .split("/models/") - .nth(1) - .unwrap_or_default() - .trim_matches('/') - .to_string(); - let name = name_sel - .as_ref() - .and_then(|sel| link.select(sel).next()) - .map(|n| n.text().collect::()) - .unwrap_or_else(|| link.text().collect::()); - let name = name.split_whitespace().collect::>().join(" "); - if slug.is_empty() || name.is_empty() || artists.iter().any(|(_, s)| *s == slug) { - continue; - } - artists.push((name, slug)); - } - artists - } - - fn cached_detail(&self, item: &mut VideoItem) -> bool { - let Some((time, cached)) = self.details.get(&item.url) else { - return false; - }; - let Some(cached) = cached.into_iter().next() else { - return false; - }; - if time.elapsed().unwrap_or(DETAIL_TTL) >= DETAIL_TTL { - return false; - } - item.formats = cached.formats; - item.tags = cached.tags; - item.uploader = cached.uploader; - item.uploaderUrl = cached.uploaderUrl; - item.uploaderId = cached.uploaderId; - true - } - - async fn enrich_items(&self, items: Vec, requester: Requester) -> Vec { - let referer = format!("{}/", self.url); - let base = self.url.clone(); - // Once the site rate-limits a detail page, the rest of the batch keeps - // the listing data instead of digging the hole deeper. - let throttled = Arc::new(AtomicBool::new(false)); - stream::iter(items.into_iter().map(|mut item| { - let mut requester = requester.clone(); - let referer = referer.clone(); - let base = base.clone(); - let throttled = throttled.clone(); - async move { - if item.url.is_empty() || self.cached_detail(&mut item) { - return item; - } - if throttled.load(Ordering::Relaxed) { - return item; - } - // `get_probe` hands challenges to the full fallback chain but - // reports a lasting 429 as an error rather than escalating every - // detail page to FlareSolverr. - let headers = vec![("Referer".to_string(), referer)]; - let html = match requester.get_probe(&item.url, headers).await { - Ok(Some(html)) => html, - Ok(None) => return item, - Err(_) => { - throttled.store(true, Ordering::Relaxed); - return item; - } - }; - let (formats, mut tags) = Self::parse_detail(&html); - if !formats.is_empty() { - item.formats = Some(formats); - } - let artists = Self::parse_artists(&html); - if let Some((name, slug)) = artists.first() { - item.uploader = Some(name.clone()); - item.uploaderUrl = Some(format!("{base}/models/{slug}/")); - item.uploaderId = Some(format!("rule34video:{slug}")); - } - for (name, _) in &artists { - if !tags.iter().any(|t| t.eq_ignore_ascii_case(name)) { - tags.push(name.clone()); - } - } - if !tags.is_empty() { - item.tags = Some(tags); - } - if item.formats.is_some() { - self.details.insert(item.url.clone(), vec![item.clone()]); - } - item - } - })) - .buffered(ENRICH_CONCURRENCY) - .collect::>() - .await + items } fn get_video_items_from_html(&self, html: String) -> Vec { @@ -830,22 +660,6 @@ mod tests { assert_eq!(items[0].duration, 88); } - #[test] - fn parses_flashvars_formats_and_tags() { - let html = r#"var flashvars = { video_id: '1', video_categories: '3D, Cyberpunk 2077', video_tags: 'bondage, panam palmer (cyberpunk), 3d', video_url: 'https://rule34video.com/get_file/a_360.mp4/?v-acctoken=1', video_url_text: '360p', video_alt_url: 'https://rule34video.com/get_file/a_480p.mp4/?v-acctoken=2', video_alt_url_text: '480p', video_alt_url2: 'https://rule34video.com/get_file/a_720p.mp4/?v-acctoken=3', video_alt_url2_text: '720p', video_title: 'It\'s' };"#; - let (formats, tags) = Rule34videoProvider::parse_detail(html); - assert_eq!(formats.len(), 3); - assert!(formats[0].url.contains("a_720p"), "best quality first"); - assert!(formats[2].url.contains("a_360")); - assert_eq!(tags, vec!["3D", "Cyberpunk 2077", "bondage", "panam palmer (cyberpunk)"]); - } - - #[test] - fn detail_without_flashvars_yields_nothing() { - let (formats, tags) = Rule34videoProvider::parse_detail(""); - assert!(formats.is_empty() && tags.is_empty()); - } - #[test] fn parses_prefixed_queries() { assert_eq!( @@ -878,19 +692,12 @@ mod tests { } #[test] - fn parses_tag_suggestions_and_artists() { + fn parses_tag_suggestions() { let suggest = r#"
"#; assert_eq!( Rule34videoProvider::parse_tag_suggestions(suggest), vec!["samus aran (metroid)".to_string()] ); - let page = r#""#; - assert_eq!( - Rule34videoProvider::parse_artists(page), - vec![("JOIGang (VA)".to_string(), "joigang-va".to_string())] - ); } #[test] diff --git a/src/proxies/mod.rs b/src/proxies/mod.rs index 616758c..c0303dd 100644 --- a/src/proxies/mod.rs +++ b/src/proxies/mod.rs @@ -37,6 +37,7 @@ pub mod javtiful; pub mod lulustream; pub mod noodlemagazine; pub mod pimpbunny; +pub mod rule34video; pub mod porndish; pub mod porndishthumb; pub mod pornhd3x; diff --git a/src/proxies/rule34video.rs b/src/proxies/rule34video.rs new file mode 100644 index 0000000..e005cbe --- /dev/null +++ b/src/proxies/rule34video.rs @@ -0,0 +1,28 @@ +use ntex::http::Method; +use ntex::http::header::LOCATION; +use ntex::web::{self, HttpRequest}; + +use crate::util::requester::Requester; +use crate::util::rule34video_resolve::{page_url, resolve_media_url}; + +/// The channel lists `/proxy/rule34video/{id}/{slug}` instead of resolving every +/// video page while listing (which ran into the site's 429 limit): the page is +/// read when the video is played, and the player is sent to its signed mp4. +pub async fn redirect_to_media( + req: HttpRequest, + requester: web::types::State, +) -> impl web::Responder { + let endpoint = req.match_info().query("endpoint").to_string(); + let Some(page_url) = page_url(&endpoint) else { + return web::HttpResponse::BadRequest().finish(); + }; + if req.method() == Method::HEAD { + return web::HttpResponse::Ok().finish(); + } + + let mut requester = requester.get_ref().clone(); + match resolve_media_url(&mut requester, &page_url, None).await { + Some(url) => web::HttpResponse::Found().header(LOCATION, url).finish(), + None => web::HttpResponse::BadGateway().finish(), + } +} diff --git a/src/proxy.rs b/src/proxy.rs index 47b2bee..04b5670 100644 --- a/src/proxy.rs +++ b/src/proxy.rs @@ -157,6 +157,11 @@ pub fn config(cfg: &mut web::ServiceConfig) { .route(web::post().to(crate::proxies::fikfapthumb::get_image)) .route(web::get().to(crate::proxies::fikfapthumb::get_image)), ) + .service( + web::resource("/rule34video/{endpoint}*") + .route(web::get().to(crate::proxies::rule34video::redirect_to_media)) + .route(web::head().to(crate::proxies::rule34video::redirect_to_media)), + ) .service( web::resource("/hanime/{endpoint}*") .route(web::get().to(crate::proxies::hanime::redirect_to_media)) diff --git a/src/util/mod.rs b/src/util/mod.rs index 8d56d39..8e80313 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -10,6 +10,7 @@ pub mod hanime_resolve; pub mod hoster_proxy; pub mod proxy; pub mod requester; +pub mod rule34video_resolve; pub mod time; pub mod browser; pub mod dean_edwards; diff --git a/src/util/rule34video_resolve.rs b/src/util/rule34video_resolve.rs new file mode 100644 index 0000000..89bc137 --- /dev/null +++ b/src/util/rule34video_resolve.rs @@ -0,0 +1,117 @@ +//! rule34video.com video-page resolution for the `/proxy/rule34video/` redirect. +//! +//! Lives here (always compiled) because `HOT_TUB_PROVIDER=` single-provider +//! builds only compile the selected provider, while the proxy routes are +//! always registered. +//! +//! yt-dlp's Rule34Video extractor no longer resolves the page URL ("No video +//! formats found"), so the signed mp4 URLs (360p/480p/720p, valid ~12h, no +//! cookie/Referer needed) come from the page's `var flashvars`. + +use crate::util::requester::Requester; + +const BASE_URL: &str = "https://rule34video.com"; + +/// Reads a single-quoted JS string value (`key: '...'`) from the +/// `flashvars` object, honouring `\'` escapes. +fn flashvar(block: &str, key: &str) -> Option { + let marker = format!("{}: '", key); + let start = block.find(&marker)? + marker.len(); + let mut out = String::new(); + let mut chars = block[start..].chars(); + while let Some(c) = chars.next() { + match c { + '\\' => out.push(chars.next()?), + '\'' => return Some(out), + other => out.push(other), + } + } + None +} + +/// `(height, url)` for every quality on the page, best first. +pub fn parse_formats(html: &str) -> Vec<(u32, String)> { + let Some(start) = html.find("var flashvars") else { + return vec![]; + }; + let block = &html[start..]; + let block = &block[..block.len().min(6000)]; + + let mut formats: Vec<(u32, String)> = Vec::new(); + for (url_key, text_key) in [ + ("video_url", "video_url_text"), + ("video_alt_url", "video_alt_url_text"), + ("video_alt_url2", "video_alt_url2_text"), + ("video_alt_url3", "video_alt_url3_text"), + ] { + let Some(url) = flashvar(block, url_key).filter(|url| url.starts_with("http")) else { + continue; + }; + let height = flashvar(block, text_key) + .and_then(|quality| quality.trim_end_matches('p').parse::().ok()) + .unwrap_or(0); + formats.push((height, url)); + } + formats.sort_by(|a, b| b.0.cmp(&a.0)); + formats +} + +/// `{id}/{slug}[/]` -> the video page URL. Only digits, letters, `-` and `_` +/// are accepted so the endpoint cannot steer the request elsewhere. +pub fn page_url(endpoint: &str) -> Option { + let mut parts = endpoint.trim_matches('/').split('/'); + let id = parts.next().filter(|id| !id.is_empty() && id.chars().all(|c| c.is_ascii_digit()))?; + let slug = parts.next().filter(|slug| { + !slug.is_empty() && slug.chars().all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_') + })?; + match parts.next() { + None => Some(format!("{BASE_URL}/video/{id}/{slug}/")), + Some(_) => None, + } +} + +/// The best signed mp4 URL (or the one of `height`) for a video page. +pub async fn resolve_media_url( + requester: &mut Requester, + page_url: &str, + height: Option, +) -> Option { + let headers = vec![("Referer".to_string(), format!("{BASE_URL}/"))]; + let html = requester.get_probe(page_url, headers).await.ok().flatten()?; + let formats = parse_formats(&html); + height + .and_then(|h| formats.iter().find(|(height, _)| *height == h)) + .or_else(|| formats.first()) + .map(|(_, url)| url.clone()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_flashvars_formats_best_first() { + let html = r#"var flashvars = { video_id: '1', video_url: 'https://rule34video.com/get_file/a_360.mp4/?v-acctoken=1', video_url_text: '360p', video_alt_url: 'https://rule34video.com/get_file/a_480p.mp4/?v-acctoken=2', video_alt_url_text: '480p', video_alt_url2: 'https://rule34video.com/get_file/a_720p.mp4/?v-acctoken=3', video_alt_url2_text: '720p', video_title: 'It\'s' };"#; + let formats = parse_formats(html); + assert_eq!(formats.len(), 3); + assert_eq!(formats[0].0, 720); + assert!(formats[0].1.contains("a_720p")); + assert!(formats[2].1.contains("a_360")); + } + + #[test] + fn page_without_flashvars_yields_nothing() { + assert!(parse_formats("").is_empty()); + } + + #[test] + fn builds_page_url_from_safe_endpoints_only() { + assert_eq!( + page_url("123/some-slug/").as_deref(), + Some("https://rule34video.com/video/123/some-slug/") + ); + assert_eq!(page_url("123/../x"), None); + assert_eq!(page_url("abc/slug"), None); + assert_eq!(page_url("123/slug/extra"), None); + } +}