simon/hottub
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix rule34video: thumbs, views and playback

Browse Source 
Thumbs moved to data-rd-src, views to title="Views: ..", and yt-dlp no
longer resolves the page URL, so formats and tags come from the detail
page flashvars.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MUYdCh2nCVsQYgJAT7gwSi
This commit is contained in:
Simon
2026-10-09 22:27:29 +00:00
parent 7fbf6cfd15
commit 316c317c81
1 changed files with 191 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

203
src/providers/rule34video.rs
View File

@@ -5,14 +5,18 @@ use crate::status::*;
use crate::util::cache::VideoCache;
use crate::util::discord::send_discord_error_report;
use crate::util::parse_abbreviated_number;
use crate::util::requester::Requester;
use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoItem};
use crate::videos::{ServerOptions, VideoFormat, VideoItem};
use async_trait::async_trait;
use error_chain::error_chain;
use futures::stream::{self, StreamExt};
use htmlentity::entity::{ICodedDataTrait, decode};
use std::time::{SystemTime, UNIX_EPOCH};
use std::vec;
const ENRICH_CONCURRENCY: usize = 6;
pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata =
crate::providers::ProviderChannelMetadata {
group_id: "hentai-animation",
@@ -152,6 +156,7 @@ impl Rule34videoProvider {
"".to_string()
});
let video_items = self.get_video_items_from_html(text);
let video_items = self.enrich_items(video_items, requester).await;
if !video_items.is_empty() {
cache.insert(index, video_items.clone());
@@ -224,6 +229,7 @@ impl Rule34videoProvider {
"".to_string()
});
let video_items = self.get_video_items_from_html(text);
let video_items = self.enrich_items(video_items, requester).await;
if !video_items.is_empty() {
cache.insert(index, video_items.clone());
@@ -236,6 +242,103 @@ impl Rule34videoProvider {
}
}
/// Reads a single-quoted JS string value (`key: '...'`) from the
/// `flashvars` object, honouring `\'` escapes.
fn flashvar(block: &str, key: &str) -> Option<String> {
let marker = format!("{}: '", key);
let start = block.find(&marker)? + marker.len();
let mut out = String::new();
let mut chars = block[start..].chars();
while let Some(c) = chars.next() {
match c {
'\\' => out.push(chars.next()?),
'\'' => return Some(out),
other => out.push(other),
}
}
None
}
/// The page URL is no longer resolvable by yt-dlp's Rule34Video
/// extractor ("No video formats found"), so pull the signed mp4 URLs
/// (360p/480p/720p, valid ~12h, no cookie/Referer needed) and the tags
/// straight from the detail page's `flashvars`.
fn parse_detail(html: &str) -> (Vec<VideoFormat>, Vec<String>) {
let Some(start) = html.find("var flashvars") else {
return (vec![], vec![]);
};
let block = &html[start..];
let block = &block[..block.len().min(6000)];
let mut formats: Vec<(u32, VideoFormat)> = Vec::new();
for (url_key, text_key) in [
("video_url", "video_url_text"),
("video_alt_url", "video_alt_url_text"),
("video_alt_url2", "video_alt_url2_text"),
("video_alt_url3", "video_alt_url3_text"),
] {
let Some(url) = Self::flashvar(block, url_key) else {
continue;
};
if !url.starts_with("http") {
continue;
}
let quality = Self::flashvar(block, text_key).unwrap_or_else(|| "auto".to_string());
let height = quality.trim_end_matches('p').parse::<u32>().unwrap_or(0);
let mut format = VideoFormat::new(url, quality.clone(), "mp4".to_string())
.format_id(format!("mp4-{}", quality));
if height > 0 {
format = format.height(height);
}
formats.push((height, format));
}
formats.sort_by(|a, b| b.0.cmp(&a.0));
let mut tags: Vec<String> = Vec::new();
for key in ["video_categories", "video_tags"] {
for tag in Self::flashvar(block, key).unwrap_or_default().split(',') {
let tag = tag.trim();
if !tag.is_empty() && !tags.iter().any(|t| t.eq_ignore_ascii_case(tag)) {
tags.push(tag.to_string());
}
}
}
(formats.into_iter().map(|(_, f)| f).collect(), tags)
}
async fn enrich_items(&self, items: Vec<VideoItem>, requester: Requester) -> Vec<VideoItem> {
let referer = format!("{}/", self.url);
stream::iter(items.into_iter().map(|mut item| {
let mut requester = requester.clone();
let referer = referer.clone();
async move {
if item.url.is_empty() {
return item;
}
let html = requester
.get_with_headers(
&item.url,
vec![("Referer".to_string(), referer)],
None,
)
.await
.unwrap_or_default();
let (formats, tags) = Self::parse_detail(&html);
if !formats.is_empty() {
item.formats = Some(formats);
}
if !tags.is_empty() {
item.tags = Some(tags);
}
item
}
}))
.buffered(ENRICH_CONCURRENCY)
.collect::<Vec<_>>()
.await
}
fn get_video_items_from_html(&self, html: String) -> Vec<VideoItem> {
if html.is_empty() {
return vec![];
@@ -280,15 +383,37 @@ impl Rule34videoProvider {
Self::extract_between(video_segment, "<div class=\"time\">", "<").unwrap_or("0:00");
let duration = parse_time_to_seconds(raw_duration).unwrap_or(0) as u32;
// Views extraction
let views_segment = Self::extract_between(video_segment, "<div class=\"views\">", "<");
let views_count_str = views_segment
.and_then(|s| s.split("</svg>").nth(1))
.unwrap_or("0");
let views = parse_abbreviated_number(views_count_str.trim()).unwrap_or(0);
// Views extraction: `<div class="views video-views-count ..."
// title="Views: 13K">`. The sibling `views` div is the comment
// count, so match the title rather than the bare class.
let views = Self::extract_between(video_segment, "title=\"Views: ", "\"")
.and_then(|v| parse_abbreviated_number(v.trim()))
.or_else(|| {
Self::extract_between(video_segment, "<div class=\"views\">", "</div>")
.map(|s| s.rsplit("</svg>").next().unwrap_or(s))
.and_then(|v| parse_abbreviated_number(v.trim()))
})
.unwrap_or(0);
// Thumbnail extraction
let thumb = Self::extract_between(video_segment, "data-original=\"", "\"")
// Rating extraction: `72% (47)` inside the rating div.
let rating = Self::extract_between(video_segment, "custom-thumbs-up", "</div>")
.and_then(|s| s.rsplit("</svg>").next())
.and_then(|s| s.trim().split('%').next())
.and_then(|s| s.trim().parse::<f32>().ok());
// Thumbnail extraction: the site dropped `data-original`. Lazy
// cards are `<img class="thumb rd-lz" src="data:image/gif..."
// data-rd-src="...">`; eager ones carry the real `src`.
let thumb = Self::extract_between(video_segment, "<img class=\"thumb", ">")
.and_then(|tag| {
Self::extract_between(tag, "data-rd-src=\"", "\"")
.or_else(|| {
Self::extract_between(tag, " src=\"", "\"")
.filter(|src| src.starts_with("http"))
})
.or_else(|| Self::extract_between(tag, "data-rd-jpg=\"", "\""))
})
.or_else(|| Self::extract_between(video_segment, "data-original=\"", "\""))
.unwrap_or("")
.to_string();
@@ -298,10 +423,13 @@ impl Rule34videoProvider {
.unwrap_or("")
.to_string();
items.push(
let mut item =
VideoItem::new(id, title, url, "Rule34video".to_string(), thumb, duration)
.views(views),
);
.views(views);
if let Some(rating) = rating {
item = item.rating(rating);
}
items.push(item);
}
items
}
@@ -339,3 +467,54 @@ impl Provider for Rule34videoProvider {
Some(self.build_channel(clientversion))
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parses_card_thumb_from_img_src_and_skips_ads() {
let html = r#"<div id="custom_list_videos_x"><div class="item thumb video_1 " data-video-card-id="42">
<a class="th js-open-popup" href="https://rule34video.com/video/42/some-title/" title="Some Title">
<div class="img wrap_image"><img class="thumb" src="https://rule34video.com/contents/videos_screenshots/0/42/336x189/3.jpg" data-rd-jpg="x" alt="a">
<div class="time">1:28</div></div><div class="thumb_title">Some Title</div>
<div class="rating"><svg class="custom-svg custom-thumbs-up"></svg> 72% (47) </div>
<div class="views video-comments-count" title="Comments: 0"><svg></svg> 0 </div>
<div class="views video-views-count" title="Views: 6.9K"><svg></svg> 6.9K </div></a></div>
<div class="item thumb video_3"><a class="th js-open-popup" href="https://rule34video.com/video/43/lazy/" title="Lazy">
<div class="img wrap_image"><img class="thumb rd-lz" src="data:image/gif;base64,R0lG" data-rd-src="https://rule34video.com/contents/videos_screenshots/0/43/336x189/3.jpg" alt="b">
<div class="time">0:30</div></div><div class="thumb_title">Lazy</div></a></div>
<div class="item thumb video_2"><a class="th" href="https://ads" title="Advertisement"></a></div>
<div class="pagination" id="p">"#;
let items = Rule34videoProvider::new().get_video_items_from_html(html.to_string());
assert_eq!(items.len(), 2);
assert_eq!(items[0].id, "42");
assert_eq!(items[0].views, Some(6900));
assert_eq!(items[0].rating, Some(72.0));
assert_eq!(
items[1].thumb,
"https://rule34video.com/contents/videos_screenshots/0/43/336x189/3.jpg"
);
assert_eq!(
items[0].thumb,
"https://rule34video.com/contents/videos_screenshots/0/42/336x189/3.jpg"
);
assert_eq!(items[0].duration, 88);
}
#[test]
fn parses_flashvars_formats_and_tags() {
let html = r#"var flashvars = { video_id: '1', video_categories: '3D, Cyberpunk 2077', video_tags: 'bondage, panam palmer (cyberpunk), 3d', video_url: 'https://rule34video.com/get_file/a_360.mp4/?v-acctoken=1', video_url_text: '360p', video_alt_url: 'https://rule34video.com/get_file/a_480p.mp4/?v-acctoken=2', video_alt_url_text: '480p', video_alt_url2: 'https://rule34video.com/get_file/a_720p.mp4/?v-acctoken=3', video_alt_url2_text: '720p', video_title: 'It\'s' };"#;
let (formats, tags) = Rule34videoProvider::parse_detail(html);
assert_eq!(formats.len(), 3);
assert!(formats[0].url.contains("a_720p"), "best quality first");
assert!(formats[2].url.contains("a_360"));
assert_eq!(tags, vec!["3D", "Cyberpunk 2077", "bondage", "panam palmer (cyberpunk)"]);
}
#[test]
fn detail_without_flashvars_yields_nothing() {
let (formats, tags) = Rule34videoProvider::parse_detail("<html></html>");
assert!(formats.is_empty() && tags.is_empty());
}
}