simon/hottub
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rule34video: drop per-video enrichment; add on-demand page resolver proxy (off)

Browse Source 
Listings no longer fetch every video page for formats/tags/artist, which
ran into the site's 429 limit. /proxy/rule34video/{id}/{slug} resolves the
page at play time and 302s to the signed mp4; the resolver lives in
util/rule34video_resolve.rs. Handing out proxy URLs is gated behind
USE_MEDIA_PROXY, currently false.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01AEp724zDpCWNaSeCrMSvjx
This commit is contained in:
Simon
2026-11-05 20:26:50 +00:00
parent cf0ae15f93
commit 4f83dae3fa
6 changed files with 170 additions and 211 deletions
Download Patch File Download Diff File Expand all files Collapse all files

229
src/providers/rule34video.rs
View File

@@ -5,25 +5,21 @@ use crate::status::*;
use crate::util::cache::VideoCache;
use crate::util::discord::send_discord_error_report;
use crate::util::parse_abbreviated_number;
use crate::util::requester::Requester;
use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoFormat, VideoItem};
use crate::videos::{ServerOptions, VideoItem};
use async_trait::async_trait;
use error_chain::error_chain;
use futures::stream::{self, StreamExt};
use htmlentity::entity::{ICodedDataTrait, decode};
use scraper::{Html, Selector};
use std::collections::HashMap;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use percent_encoding::{NON_ALPHANUMERIC, utf8_percent_encode};
use std::vec;
const ENRICH_CONCURRENCY: usize = 4;
/// The signed `v-acctoken` format URLs last ~12h; keep details well inside that.
const DETAIL_TTL: Duration = Duration::from_secs(2 * 60 * 60);
const DETAIL_CACHE_SIZE: usize = 5000;
/// Hands out `/proxy/rule34video/` URLs instead of the page URLs. Switched off;
/// the proxy and resolver stay in place (`proxies/rule34video.rs`).
const USE_MEDIA_PROXY: bool = false;
const TARGET_TTL: Duration = Duration::from_secs(6 * 60 * 60);
const TARGET_CACHE_SIZE: usize = 2000;
/// A probed archive smaller than this loses to keyword search: "elf girl" is a
@@ -75,21 +71,14 @@ impl Target {
#[derive(Clone)]
pub struct Rule34videoProvider {
url: String,
/// Enriched items (formats, tags, artist) by video URL. Every listing costs
/// ~23 detail pages, again for the prefetched next page; refetching them
/// on each request is what runs into the site's 429 rate limit.
details: VideoCache,
/// Bare query -> resolved target, so later pages skip the archive probes.
targets: Arc<Mutex<HashMap<String, (Instant, Target)>>>,
}
impl Rule34videoProvider {
pub fn new() -> Self {
let mut details = VideoCache::new();
details.max_size(DETAIL_CACHE_SIZE);
Rule34videoProvider {
url: "https://rule34video.com".to_string(),
details,
targets: Arc::new(Mutex::new(HashMap::new())),
}
}
@@ -325,7 +314,7 @@ impl Rule34videoProvider {
}
};
let video_items = self.get_video_items_from_html(text);
let video_items = self.enrich_items(video_items, requester).await;
let video_items = Self::proxy_items(video_items, &options);
if video_items.is_empty() {
return Ok(stale());
}
@@ -406,7 +395,7 @@ impl Rule34videoProvider {
"".to_string()
});
let video_items = self.get_video_items_from_html(text);
let video_items = self.enrich_items(video_items, requester).await;
let video_items = Self::proxy_items(video_items, &options);
if !video_items.is_empty() {
cache.insert(index, video_items.clone());
@@ -479,7 +468,7 @@ impl Rule34videoProvider {
"".to_string()
});
let video_items = self.get_video_items_from_html(text);
let video_items = self.enrich_items(video_items, requester).await;
let video_items = Self::proxy_items(video_items, &options);
if !video_items.is_empty() {
cache.insert(index, video_items.clone());
@@ -492,177 +481,18 @@ impl Rule34videoProvider {
}
}
/// Reads a single-quoted JS string value (`key: '...'`) from the
/// `flashvars` object, honouring `\'` escapes.
fn flashvar(block: &str, key: &str) -> Option<String> {
let marker = format!("{}: '", key);
let start = block.find(&marker)? + marker.len();
let mut out = String::new();
let mut chars = block[start..].chars();
while let Some(c) = chars.next() {
match c {
'\\' => out.push(chars.next()?),
'\'' => return Some(out),
other => out.push(other),
/// Playback resolves the video page on demand through `/proxy/rule34video/`,
/// instead of fetching every listed page up front (which ran into 429s).
fn proxy_items(mut items: Vec<VideoItem>, options: &ServerOptions) -> Vec<VideoItem> {
if !USE_MEDIA_PROXY {
return items;
}
for item in &mut items {
if let Some(path) = item.url.split("/video/").nth(1).filter(|p| !p.is_empty()) {
item.url = crate::providers::build_proxy_url(options, "rule34video", path);
}
}
None
}
/// The page URL is no longer resolvable by yt-dlp's Rule34Video
/// extractor ("No video formats found"), so pull the signed mp4 URLs
/// (360p/480p/720p, valid ~12h, no cookie/Referer needed) and the tags
/// straight from the detail page's `flashvars`.
fn parse_detail(html: &str) -> (Vec<VideoFormat>, Vec<String>) {
let Some(start) = html.find("var flashvars") else {
return (vec![], vec![]);
};
let block = &html[start..];
let block = &block[..block.len().min(6000)];
let mut formats: Vec<(u32, VideoFormat)> = Vec::new();
for (url_key, text_key) in [
("video_url", "video_url_text"),
("video_alt_url", "video_alt_url_text"),
("video_alt_url2", "video_alt_url2_text"),
("video_alt_url3", "video_alt_url3_text"),
] {
let Some(url) = Self::flashvar(block, url_key) else {
continue;
};
if !url.starts_with("http") {
continue;
}
let quality = Self::flashvar(block, text_key).unwrap_or_else(|| "auto".to_string());
let height = quality.trim_end_matches('p').parse::<u32>().unwrap_or(0);
let mut format = VideoFormat::new(url, quality.clone(), "mp4".to_string())
.format_id(format!("mp4-{}", quality));
if height > 0 {
format = format.height(height);
}
formats.push((height, format));
}
formats.sort_by(|a, b| b.0.cmp(&a.0));
let mut tags: Vec<String> = Vec::new();
for key in ["video_categories", "video_tags"] {
for tag in Self::flashvar(block, key).unwrap_or_default().split(',') {
let tag = tag.trim();
if !tag.is_empty() && !tags.iter().any(|t| t.eq_ignore_ascii_case(tag)) {
tags.push(tag.to_string());
}
}
}
(formats.into_iter().map(|(_, f)| f).collect(), tags)
}
/// Artists credited on a video page: `(name, slug)` from the "Artist" group.
fn parse_artists(html: &str) -> Vec<(String, String)> {
let document = Html::parse_document(html);
let Ok(group_sel) = Selector::parse("[data-suggest-type=\"model\"] a[href*=\"/models/\"]")
else {
return vec![];
};
let name_sel = Selector::parse(".name").ok();
let mut artists: Vec<(String, String)> = Vec::new();
for link in document.select(&group_sel) {
let href = link.value().attr("href").unwrap_or_default();
let slug = href
.split("/models/")
.nth(1)
.unwrap_or_default()
.trim_matches('/')
.to_string();
let name = name_sel
.as_ref()
.and_then(|sel| link.select(sel).next())
.map(|n| n.text().collect::<String>())
.unwrap_or_else(|| link.text().collect::<String>());
let name = name.split_whitespace().collect::<Vec<_>>().join(" ");
if slug.is_empty() || name.is_empty() || artists.iter().any(|(_, s)| *s == slug) {
continue;
}
artists.push((name, slug));
}
artists
}
fn cached_detail(&self, item: &mut VideoItem) -> bool {
let Some((time, cached)) = self.details.get(&item.url) else {
return false;
};
let Some(cached) = cached.into_iter().next() else {
return false;
};
if time.elapsed().unwrap_or(DETAIL_TTL) >= DETAIL_TTL {
return false;
}
item.formats = cached.formats;
item.tags = cached.tags;
item.uploader = cached.uploader;
item.uploaderUrl = cached.uploaderUrl;
item.uploaderId = cached.uploaderId;
true
}
async fn enrich_items(&self, items: Vec<VideoItem>, requester: Requester) -> Vec<VideoItem> {
let referer = format!("{}/", self.url);
let base = self.url.clone();
// Once the site rate-limits a detail page, the rest of the batch keeps
// the listing data instead of digging the hole deeper.
let throttled = Arc::new(AtomicBool::new(false));
stream::iter(items.into_iter().map(|mut item| {
let mut requester = requester.clone();
let referer = referer.clone();
let base = base.clone();
let throttled = throttled.clone();
async move {
if item.url.is_empty() || self.cached_detail(&mut item) {
return item;
}
if throttled.load(Ordering::Relaxed) {
return item;
}
// `get_probe` hands challenges to the full fallback chain but
// reports a lasting 429 as an error rather than escalating every
// detail page to FlareSolverr.
let headers = vec![("Referer".to_string(), referer)];
let html = match requester.get_probe(&item.url, headers).await {
Ok(Some(html)) => html,
Ok(None) => return item,
Err(_) => {
throttled.store(true, Ordering::Relaxed);
return item;
}
};
let (formats, mut tags) = Self::parse_detail(&html);
if !formats.is_empty() {
item.formats = Some(formats);
}
let artists = Self::parse_artists(&html);
if let Some((name, slug)) = artists.first() {
item.uploader = Some(name.clone());
item.uploaderUrl = Some(format!("{base}/models/{slug}/"));
item.uploaderId = Some(format!("rule34video:{slug}"));
}
for (name, _) in &artists {
if !tags.iter().any(|t| t.eq_ignore_ascii_case(name)) {
tags.push(name.clone());
}
}
if !tags.is_empty() {
item.tags = Some(tags);
}
if item.formats.is_some() {
self.details.insert(item.url.clone(), vec![item.clone()]);
}
item
}
}))
.buffered(ENRICH_CONCURRENCY)
.collect::<Vec<_>>()
.await
items
}
fn get_video_items_from_html(&self, html: String) -> Vec<VideoItem> {
@@ -830,22 +660,6 @@ mod tests {
assert_eq!(items[0].duration, 88);
}
#[test]
fn parses_flashvars_formats_and_tags() {
let html = r#"var flashvars = { video_id: '1', video_categories: '3D, Cyberpunk 2077', video_tags: 'bondage, panam palmer (cyberpunk), 3d', video_url: 'https://rule34video.com/get_file/a_360.mp4/?v-acctoken=1', video_url_text: '360p', video_alt_url: 'https://rule34video.com/get_file/a_480p.mp4/?v-acctoken=2', video_alt_url_text: '480p', video_alt_url2: 'https://rule34video.com/get_file/a_720p.mp4/?v-acctoken=3', video_alt_url2_text: '720p', video_title: 'It\'s' };"#;
let (formats, tags) = Rule34videoProvider::parse_detail(html);
assert_eq!(formats.len(), 3);
assert!(formats[0].url.contains("a_720p"), "best quality first");
assert!(formats[2].url.contains("a_360"));
assert_eq!(tags, vec!["3D", "Cyberpunk 2077", "bondage", "panam palmer (cyberpunk)"]);
}
#[test]
fn detail_without_flashvars_yields_nothing() {
let (formats, tags) = Rule34videoProvider::parse_detail("<html></html>");
assert!(formats.is_empty() && tags.is_empty());
}
#[test]
fn parses_prefixed_queries() {
assert_eq!(
@@ -878,19 +692,12 @@ mod tests {
}
#[test]
fn parses_tag_suggestions_and_artists() {
fn parses_tag_suggestions() {
let suggest = r#"<div class="item checkbox-container"><input id="tag_143" type="checkbox" value="143" data-title="samus aran (metroid)"/></div>"#;
assert_eq!(
Rule34videoProvider::parse_tag_suggestions(suggest),
vec!["samus aran (metroid)".to_string()]
);
let page = r#"<div class="col vp-group" data-suggest-type="model"><div class="label">Artist</div>
<span class="chip"><a class="item btn_link" href="https://rule34video.com/models/joigang-va/">
<span class="name">JOIGang (VA)</span></a></span></div>"#;
assert_eq!(
Rule34videoProvider::parse_artists(page),
vec![("JOIGang (VA)".to_string(), "joigang-va".to_string())]
);
}
#[test]