rule34video: drop per-video enrichment; add on-demand page resolver proxy (off)

Listings no longer fetch every video page for formats/tags/artist, which
ran into the site's 429 limit. /proxy/rule34video/{id}/{slug} resolves the
page at play time and 302s to the signed mp4; the resolver lives in
util/rule34video_resolve.rs. Handing out proxy URLs is gated behind
USE_MEDIA_PROXY, currently false.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01AEp724zDpCWNaSeCrMSvjx

src/providers/rule34video.rs

@@ -5,25 +5,21 @@ use crate::status::*;
 use crate::util::cache::VideoCache;
 use crate::util::discord::send_discord_error_report;
 use crate::util::parse_abbreviated_number;
-use crate::util::requester::Requester;
 use crate::util::time::parse_time_to_seconds;
-use crate::videos::{ServerOptions, VideoFormat, VideoItem};
+use crate::videos::{ServerOptions, VideoItem};
 use async_trait::async_trait;
 use error_chain::error_chain;
-use futures::stream::{self, StreamExt};
 use htmlentity::entity::{ICodedDataTrait, decode};
 use scraper::{Html, Selector};
 use std::collections::HashMap;
-use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, Mutex};
 use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
 use percent_encoding::{NON_ALPHANUMERIC, utf8_percent_encode};
 use std::vec;
 
-const ENRICH_CONCURRENCY: usize = 4;
+/// Hands out `/proxy/rule34video/` URLs instead of the page URLs. Switched off;
-/// The signed `v-acctoken` format URLs last ~12h; keep details well inside that.
+/// the proxy and resolver stay in place (`proxies/rule34video.rs`).
-const DETAIL_TTL: Duration = Duration::from_secs(2 * 60 * 60);
+const USE_MEDIA_PROXY: bool = false;
-const DETAIL_CACHE_SIZE: usize = 5000;
 const TARGET_TTL: Duration = Duration::from_secs(6 * 60 * 60);
 const TARGET_CACHE_SIZE: usize = 2000;
 /// A probed archive smaller than this loses to keyword search: "elf girl" is a
@@ -75,21 +71,14 @@ impl Target {
 #[derive(Clone)]
 pub struct Rule34videoProvider {
     url: String,
-    /// Enriched items (formats, tags, artist) by video URL. Every listing costs
-    /// ~23 detail pages, again for the prefetched next page; refetching them
-    /// on each request is what runs into the site's 429 rate limit.
-    details: VideoCache,
     /// Bare query -> resolved target, so later pages skip the archive probes.
     targets: Arc<Mutex<HashMap<String, (Instant, Target)>>>,
 }
 
 impl Rule34videoProvider {
     pub fn new() -> Self {
-        let mut details = VideoCache::new();
-        details.max_size(DETAIL_CACHE_SIZE);
         Rule34videoProvider {
             url: "https://rule34video.com".to_string(),
-            details,
             targets: Arc::new(Mutex::new(HashMap::new())),
         }
     }
@@ -325,7 +314,7 @@ impl Rule34videoProvider {
             }
         };
         let video_items = self.get_video_items_from_html(text);
-        let video_items = self.enrich_items(video_items, requester).await;
+        let video_items = Self::proxy_items(video_items, &options);
         if video_items.is_empty() {
             return Ok(stale());
         }
@@ -406,7 +395,7 @@ impl Rule34videoProvider {
             "".to_string()
         });
         let video_items = self.get_video_items_from_html(text);
-        let video_items = self.enrich_items(video_items, requester).await;
+        let video_items = Self::proxy_items(video_items, &options);
 
         if !video_items.is_empty() {
             cache.insert(index, video_items.clone());
@@ -479,7 +468,7 @@ impl Rule34videoProvider {
             "".to_string()
         });
         let video_items = self.get_video_items_from_html(text);
-        let video_items = self.enrich_items(video_items, requester).await;
+        let video_items = Self::proxy_items(video_items, &options);
 
         if !video_items.is_empty() {
             cache.insert(index, video_items.clone());
@@ -492,177 +481,18 @@ impl Rule34videoProvider {
         }
     }
 
-    /// Reads a single-quoted JS string value (`key: '...'`) from the
+    /// Playback resolves the video page on demand through `/proxy/rule34video/`,
-    /// `flashvars` object, honouring `\'` escapes.
+    /// instead of fetching every listed page up front (which ran into 429s).
-    fn flashvar(block: &str, key: &str) -> Option<String> {
+    fn proxy_items(mut items: Vec<VideoItem>, options: &ServerOptions) -> Vec<VideoItem> {
-        let marker = format!("{}: '", key);
+        if !USE_MEDIA_PROXY {
-        let start = block.find(&marker)? + marker.len();
+            return items;
-        let mut out = String::new();
+        }
-        let mut chars = block[start..].chars();
+        for item in &mut items {
-        while let Some(c) = chars.next() {
+            if let Some(path) = item.url.split("/video/").nth(1).filter(|p| !p.is_empty()) {
-            match c {
+                item.url = crate::providers::build_proxy_url(options, "rule34video", path);
-                '\\' => out.push(chars.next()?),
-                '\'' => return Some(out),
-                other => out.push(other),
             }
         }
-        None
+        items
-    }
-
-    /// The page URL is no longer resolvable by yt-dlp's Rule34Video
-    /// extractor ("No video formats found"), so pull the signed mp4 URLs
-    /// (360p/480p/720p, valid ~12h, no cookie/Referer needed) and the tags
-    /// straight from the detail page's `flashvars`.
-    fn parse_detail(html: &str) -> (Vec<VideoFormat>, Vec<String>) {
-        let Some(start) = html.find("var flashvars") else {
-            return (vec![], vec![]);
-        };
-        let block = &html[start..];
-        let block = &block[..block.len().min(6000)];
-
-        let mut formats: Vec<(u32, VideoFormat)> = Vec::new();
-        for (url_key, text_key) in [
-            ("video_url", "video_url_text"),
-            ("video_alt_url", "video_alt_url_text"),
-            ("video_alt_url2", "video_alt_url2_text"),
-            ("video_alt_url3", "video_alt_url3_text"),
-        ] {
-            let Some(url) = Self::flashvar(block, url_key) else {
-                continue;
-            };
-            if !url.starts_with("http") {
-                continue;
-            }
-            let quality = Self::flashvar(block, text_key).unwrap_or_else(|| "auto".to_string());
-            let height = quality.trim_end_matches('p').parse::<u32>().unwrap_or(0);
-            let mut format = VideoFormat::new(url, quality.clone(), "mp4".to_string())
-                .format_id(format!("mp4-{}", quality));
-            if height > 0 {
-                format = format.height(height);
-            }
-            formats.push((height, format));
-        }
-        formats.sort_by(|a, b| b.0.cmp(&a.0));
-
-        let mut tags: Vec<String> = Vec::new();
-        for key in ["video_categories", "video_tags"] {
-            for tag in Self::flashvar(block, key).unwrap_or_default().split(',') {
-                let tag = tag.trim();
-                if !tag.is_empty() && !tags.iter().any(|t| t.eq_ignore_ascii_case(tag)) {
-                    tags.push(tag.to_string());
-                }
-            }
-        }
-
-        (formats.into_iter().map(|(_, f)| f).collect(), tags)
-    }
-
-    /// Artists credited on a video page: `(name, slug)` from the "Artist" group.
-    fn parse_artists(html: &str) -> Vec<(String, String)> {
-        let document = Html::parse_document(html);
-        let Ok(group_sel) = Selector::parse("[data-suggest-type=\"model\"] a[href*=\"/models/\"]")
-        else {
-            return vec![];
-        };
-        let name_sel = Selector::parse(".name").ok();
-        let mut artists: Vec<(String, String)> = Vec::new();
-        for link in document.select(&group_sel) {
-            let href = link.value().attr("href").unwrap_or_default();
-            let slug = href
-                .split("/models/")
-                .nth(1)
-                .unwrap_or_default()
-                .trim_matches('/')
-                .to_string();
-            let name = name_sel
-                .as_ref()
-                .and_then(|sel| link.select(sel).next())
-                .map(|n| n.text().collect::<String>())
-                .unwrap_or_else(|| link.text().collect::<String>());
-            let name = name.split_whitespace().collect::<Vec<_>>().join(" ");
-            if slug.is_empty() || name.is_empty() || artists.iter().any(|(_, s)| *s == slug) {
-                continue;
-            }
-            artists.push((name, slug));
-        }
-        artists
-    }
-
-    fn cached_detail(&self, item: &mut VideoItem) -> bool {
-        let Some((time, cached)) = self.details.get(&item.url) else {
-            return false;
-        };
-        let Some(cached) = cached.into_iter().next() else {
-            return false;
-        };
-        if time.elapsed().unwrap_or(DETAIL_TTL) >= DETAIL_TTL {
-            return false;
-        }
-        item.formats = cached.formats;
-        item.tags = cached.tags;
-        item.uploader = cached.uploader;
-        item.uploaderUrl = cached.uploaderUrl;
-        item.uploaderId = cached.uploaderId;
-        true
-    }
-
-    async fn enrich_items(&self, items: Vec<VideoItem>, requester: Requester) -> Vec<VideoItem> {
-        let referer = format!("{}/", self.url);
-        let base = self.url.clone();
-        // Once the site rate-limits a detail page, the rest of the batch keeps
-        // the listing data instead of digging the hole deeper.
-        let throttled = Arc::new(AtomicBool::new(false));
-        stream::iter(items.into_iter().map(|mut item| {
-            let mut requester = requester.clone();
-            let referer = referer.clone();
-            let base = base.clone();
-            let throttled = throttled.clone();
-            async move {
-                if item.url.is_empty() || self.cached_detail(&mut item) {
-                    return item;
-                }
-                if throttled.load(Ordering::Relaxed) {
-                    return item;
-                }
-                // `get_probe` hands challenges to the full fallback chain but
-                // reports a lasting 429 as an error rather than escalating every
-                // detail page to FlareSolverr.
-                let headers = vec![("Referer".to_string(), referer)];
-                let html = match requester.get_probe(&item.url, headers).await {
-                    Ok(Some(html)) => html,
-                    Ok(None) => return item,
-                    Err(_) => {
-                        throttled.store(true, Ordering::Relaxed);
-                        return item;
-                    }
-                };
-                let (formats, mut tags) = Self::parse_detail(&html);
-                if !formats.is_empty() {
-                    item.formats = Some(formats);
-                }
-                let artists = Self::parse_artists(&html);
-                if let Some((name, slug)) = artists.first() {
-                    item.uploader = Some(name.clone());
-                    item.uploaderUrl = Some(format!("{base}/models/{slug}/"));
-                    item.uploaderId = Some(format!("rule34video:{slug}"));
-                }
-                for (name, _) in &artists {
-                    if !tags.iter().any(|t| t.eq_ignore_ascii_case(name)) {
-                        tags.push(name.clone());
-                    }
-                }
-                if !tags.is_empty() {
-                    item.tags = Some(tags);
-                }
-                if item.formats.is_some() {
-                    self.details.insert(item.url.clone(), vec![item.clone()]);
-                }
-                item
-            }
-        }))
-        .buffered(ENRICH_CONCURRENCY)
-        .collect::<Vec<_>>()
-        .await
     }
 
     fn get_video_items_from_html(&self, html: String) -> Vec<VideoItem> {
@@ -830,22 +660,6 @@ mod tests {
         assert_eq!(items[0].duration, 88);
     }
 
-    #[test]
-    fn parses_flashvars_formats_and_tags() {
-        let html = r#"var flashvars = { video_id: '1', video_categories: '3D, Cyberpunk 2077', video_tags: 'bondage, panam palmer (cyberpunk), 3d', video_url: 'https://rule34video.com/get_file/a_360.mp4/?v-acctoken=1', video_url_text: '360p', video_alt_url: 'https://rule34video.com/get_file/a_480p.mp4/?v-acctoken=2', video_alt_url_text: '480p', video_alt_url2: 'https://rule34video.com/get_file/a_720p.mp4/?v-acctoken=3', video_alt_url2_text: '720p', video_title: 'It\'s' };"#;
-        let (formats, tags) = Rule34videoProvider::parse_detail(html);
-        assert_eq!(formats.len(), 3);
-        assert!(formats[0].url.contains("a_720p"), "best quality first");
-        assert!(formats[2].url.contains("a_360"));
-        assert_eq!(tags, vec!["3D", "Cyberpunk 2077", "bondage", "panam palmer (cyberpunk)"]);
-    }
-
-    #[test]
-    fn detail_without_flashvars_yields_nothing() {
-        let (formats, tags) = Rule34videoProvider::parse_detail("<html></html>");
-        assert!(formats.is_empty() && tags.is_empty());
-    }
-
     #[test]
     fn parses_prefixed_queries() {
         assert_eq!(
@@ -878,19 +692,12 @@ mod tests {
     }
 
     #[test]
-    fn parses_tag_suggestions_and_artists() {
+    fn parses_tag_suggestions() {
         let suggest = r#"<div class="item checkbox-container"><input id="tag_143" type="checkbox" value="143" data-title="samus aran (metroid)"/></div>"#;
         assert_eq!(
             Rule34videoProvider::parse_tag_suggestions(suggest),
             vec!["samus aran (metroid)".to_string()]
         );
-        let page = r#"<div class="col vp-group" data-suggest-type="model"><div class="label">Artist</div>
-            <span class="chip"><a class="item btn_link" href="https://rule34video.com/models/joigang-va/">
-            <span class="name">JOIGang (VA)</span></a></span></div>"#;
-        assert_eq!(
-            Rule34videoProvider::parse_artists(page),
-            vec![("JOIGang (VA)".to_string(), "joigang-va".to_string())]
-        );
     }
 
     #[test]

src/proxies/mod.rs

@@ -37,6 +37,7 @@ pub mod javtiful;
 pub mod lulustream;
 pub mod noodlemagazine;
 pub mod pimpbunny;
+pub mod rule34video;
 pub mod porndish;
 pub mod porndishthumb;
 pub mod pornhd3x;

src/proxies/rule34video.rs

@@ -0,0 +1,28 @@
+use ntex::http::Method;
+use ntex::http::header::LOCATION;
+use ntex::web::{self, HttpRequest};
+
+use crate::util::requester::Requester;
+use crate::util::rule34video_resolve::{page_url, resolve_media_url};
+
+/// The channel lists `/proxy/rule34video/{id}/{slug}` instead of resolving every
+/// video page while listing (which ran into the site's 429 limit): the page is
+/// read when the video is played, and the player is sent to its signed mp4.
+pub async fn redirect_to_media(
+    req: HttpRequest,
+    requester: web::types::State<Requester>,
+) -> impl web::Responder {
+    let endpoint = req.match_info().query("endpoint").to_string();
+    let Some(page_url) = page_url(&endpoint) else {
+        return web::HttpResponse::BadRequest().finish();
+    };
+    if req.method() == Method::HEAD {
+        return web::HttpResponse::Ok().finish();
+    }
+
+    let mut requester = requester.get_ref().clone();
+    match resolve_media_url(&mut requester, &page_url, None).await {
+        Some(url) => web::HttpResponse::Found().header(LOCATION, url).finish(),
+        None => web::HttpResponse::BadGateway().finish(),
+    }
+}

src/proxy.rs

@@ -157,6 +157,11 @@ pub fn config(cfg: &mut web::ServiceConfig) {
             .route(web::post().to(crate::proxies::fikfapthumb::get_image))
             .route(web::get().to(crate::proxies::fikfapthumb::get_image)),
     )
+    .service(
+        web::resource("/rule34video/{endpoint}*")
+            .route(web::get().to(crate::proxies::rule34video::redirect_to_media))
+            .route(web::head().to(crate::proxies::rule34video::redirect_to_media)),
+    )
     .service(
         web::resource("/hanime/{endpoint}*")
             .route(web::get().to(crate::proxies::hanime::redirect_to_media))

src/util/mod.rs

@@ -10,6 +10,7 @@ pub mod hanime_resolve;
 pub mod hoster_proxy;
 pub mod proxy;
 pub mod requester;
+pub mod rule34video_resolve;
 pub mod time;
 pub mod browser;
 pub mod dean_edwards;

src/util/rule34video_resolve.rs

@@ -0,0 +1,117 @@
+//! rule34video.com video-page resolution for the `/proxy/rule34video/` redirect.
+//!
+//! Lives here (always compiled) because `HOT_TUB_PROVIDER=<id>` single-provider
+//! builds only compile the selected provider, while the proxy routes are
+//! always registered.
+//!
+//! yt-dlp's Rule34Video extractor no longer resolves the page URL ("No video
+//! formats found"), so the signed mp4 URLs (360p/480p/720p, valid ~12h, no
+//! cookie/Referer needed) come from the page's `var flashvars`.
+
+use crate::util::requester::Requester;
+
+const BASE_URL: &str = "https://rule34video.com";
+
+/// Reads a single-quoted JS string value (`key: '...'`) from the
+/// `flashvars` object, honouring `\'` escapes.
+fn flashvar(block: &str, key: &str) -> Option<String> {
+    let marker = format!("{}: '", key);
+    let start = block.find(&marker)? + marker.len();
+    let mut out = String::new();
+    let mut chars = block[start..].chars();
+    while let Some(c) = chars.next() {
+        match c {
+            '\\' => out.push(chars.next()?),
+            '\'' => return Some(out),
+            other => out.push(other),
+        }
+    }
+    None
+}
+
+/// `(height, url)` for every quality on the page, best first.
+pub fn parse_formats(html: &str) -> Vec<(u32, String)> {
+    let Some(start) = html.find("var flashvars") else {
+        return vec![];
+    };
+    let block = &html[start..];
+    let block = &block[..block.len().min(6000)];
+
+    let mut formats: Vec<(u32, String)> = Vec::new();
+    for (url_key, text_key) in [
+        ("video_url", "video_url_text"),
+        ("video_alt_url", "video_alt_url_text"),
+        ("video_alt_url2", "video_alt_url2_text"),
+        ("video_alt_url3", "video_alt_url3_text"),
+    ] {
+        let Some(url) = flashvar(block, url_key).filter(|url| url.starts_with("http")) else {
+            continue;
+        };
+        let height = flashvar(block, text_key)
+            .and_then(|quality| quality.trim_end_matches('p').parse::<u32>().ok())
+            .unwrap_or(0);
+        formats.push((height, url));
+    }
+    formats.sort_by(|a, b| b.0.cmp(&a.0));
+    formats
+}
+
+/// `{id}/{slug}[/]` -> the video page URL. Only digits, letters, `-` and `_`
+/// are accepted so the endpoint cannot steer the request elsewhere.
+pub fn page_url(endpoint: &str) -> Option<String> {
+    let mut parts = endpoint.trim_matches('/').split('/');
+    let id = parts.next().filter(|id| !id.is_empty() && id.chars().all(|c| c.is_ascii_digit()))?;
+    let slug = parts.next().filter(|slug| {
+        !slug.is_empty() && slug.chars().all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_')
+    })?;
+    match parts.next() {
+        None => Some(format!("{BASE_URL}/video/{id}/{slug}/")),
+        Some(_) => None,
+    }
+}
+
+/// The best signed mp4 URL (or the one of `height`) for a video page.
+pub async fn resolve_media_url(
+    requester: &mut Requester,
+    page_url: &str,
+    height: Option<u32>,
+) -> Option<String> {
+    let headers = vec![("Referer".to_string(), format!("{BASE_URL}/"))];
+    let html = requester.get_probe(page_url, headers).await.ok().flatten()?;
+    let formats = parse_formats(&html);
+    height
+        .and_then(|h| formats.iter().find(|(height, _)| *height == h))
+        .or_else(|| formats.first())
+        .map(|(_, url)| url.clone())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn parses_flashvars_formats_best_first() {
+        let html = r#"var flashvars = { video_id: '1', video_url: 'https://rule34video.com/get_file/a_360.mp4/?v-acctoken=1', video_url_text: '360p', video_alt_url: 'https://rule34video.com/get_file/a_480p.mp4/?v-acctoken=2', video_alt_url_text: '480p', video_alt_url2: 'https://rule34video.com/get_file/a_720p.mp4/?v-acctoken=3', video_alt_url2_text: '720p', video_title: 'It\'s' };"#;
+        let formats = parse_formats(html);
+        assert_eq!(formats.len(), 3);
+        assert_eq!(formats[0].0, 720);
+        assert!(formats[0].1.contains("a_720p"));
+        assert!(formats[2].1.contains("a_360"));
+    }
+
+    #[test]
+    fn page_without_flashvars_yields_nothing() {
+        assert!(parse_formats("<html></html>").is_empty());
+    }
+
+    #[test]
+    fn builds_page_url_from_safe_endpoints_only() {
+        assert_eq!(
+            page_url("123/some-slug/").as_deref(),
+            Some("https://rule34video.com/video/123/some-slug/")
+        );
+        assert_eq!(page_url("123/../x"), None);
+        assert_eq!(page_url("abc/slug"), None);
+        assert_eq!(page_url("123/slug/extra"), None);
+    }
+}