Fix rule34video: thumbs, views and playback

Thumbs moved to data-rd-src, views to title="Views: ..", and yt-dlp no
longer resolves the page URL, so formats and tags come from the detail
page flashvars.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MUYdCh2nCVsQYgJAT7gwSi

src/providers/rule34video.rs

@@ -5,14 +5,18 @@ use crate::status::*;
 use crate::util::cache::VideoCache;
 use crate::util::discord::send_discord_error_report;
 use crate::util::parse_abbreviated_number;
+use crate::util::requester::Requester;
 use crate::util::time::parse_time_to_seconds;
-use crate::videos::{ServerOptions, VideoItem};
+use crate::videos::{ServerOptions, VideoFormat, VideoItem};
 use async_trait::async_trait;
 use error_chain::error_chain;
+use futures::stream::{self, StreamExt};
 use htmlentity::entity::{ICodedDataTrait, decode};
 use std::time::{SystemTime, UNIX_EPOCH};
 use std::vec;
 
+const ENRICH_CONCURRENCY: usize = 6;
+
 pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata =
     crate::providers::ProviderChannelMetadata {
         group_id: "hentai-animation",
@@ -152,6 +156,7 @@ impl Rule34videoProvider {
             "".to_string()
         });
         let video_items = self.get_video_items_from_html(text);
+        let video_items = self.enrich_items(video_items, requester).await;
 
         if !video_items.is_empty() {
             cache.insert(index, video_items.clone());
@@ -224,6 +229,7 @@ impl Rule34videoProvider {
             "".to_string()
         });
         let video_items = self.get_video_items_from_html(text);
+        let video_items = self.enrich_items(video_items, requester).await;
 
         if !video_items.is_empty() {
             cache.insert(index, video_items.clone());
@@ -236,6 +242,103 @@ impl Rule34videoProvider {
         }
     }
 
+    /// Reads a single-quoted JS string value (`key: '...'`) from the
+    /// `flashvars` object, honouring `\'` escapes.
+    fn flashvar(block: &str, key: &str) -> Option<String> {
+        let marker = format!("{}: '", key);
+        let start = block.find(&marker)? + marker.len();
+        let mut out = String::new();
+        let mut chars = block[start..].chars();
+        while let Some(c) = chars.next() {
+            match c {
+                '\\' => out.push(chars.next()?),
+                '\'' => return Some(out),
+                other => out.push(other),
+            }
+        }
+        None
+    }
+
+    /// The page URL is no longer resolvable by yt-dlp's Rule34Video
+    /// extractor ("No video formats found"), so pull the signed mp4 URLs
+    /// (360p/480p/720p, valid ~12h, no cookie/Referer needed) and the tags
+    /// straight from the detail page's `flashvars`.
+    fn parse_detail(html: &str) -> (Vec<VideoFormat>, Vec<String>) {
+        let Some(start) = html.find("var flashvars") else {
+            return (vec![], vec![]);
+        };
+        let block = &html[start..];
+        let block = &block[..block.len().min(6000)];
+
+        let mut formats: Vec<(u32, VideoFormat)> = Vec::new();
+        for (url_key, text_key) in [
+            ("video_url", "video_url_text"),
+            ("video_alt_url", "video_alt_url_text"),
+            ("video_alt_url2", "video_alt_url2_text"),
+            ("video_alt_url3", "video_alt_url3_text"),
+        ] {
+            let Some(url) = Self::flashvar(block, url_key) else {
+                continue;
+            };
+            if !url.starts_with("http") {
+                continue;
+            }
+            let quality = Self::flashvar(block, text_key).unwrap_or_else(|| "auto".to_string());
+            let height = quality.trim_end_matches('p').parse::<u32>().unwrap_or(0);
+            let mut format = VideoFormat::new(url, quality.clone(), "mp4".to_string())
+                .format_id(format!("mp4-{}", quality));
+            if height > 0 {
+                format = format.height(height);
+            }
+            formats.push((height, format));
+        }
+        formats.sort_by(|a, b| b.0.cmp(&a.0));
+
+        let mut tags: Vec<String> = Vec::new();
+        for key in ["video_categories", "video_tags"] {
+            for tag in Self::flashvar(block, key).unwrap_or_default().split(',') {
+                let tag = tag.trim();
+                if !tag.is_empty() && !tags.iter().any(|t| t.eq_ignore_ascii_case(tag)) {
+                    tags.push(tag.to_string());
+                }
+            }
+        }
+
+        (formats.into_iter().map(|(_, f)| f).collect(), tags)
+    }
+
+    async fn enrich_items(&self, items: Vec<VideoItem>, requester: Requester) -> Vec<VideoItem> {
+        let referer = format!("{}/", self.url);
+        stream::iter(items.into_iter().map(|mut item| {
+            let mut requester = requester.clone();
+            let referer = referer.clone();
+            async move {
+                if item.url.is_empty() {
+                    return item;
+                }
+                let html = requester
+                    .get_with_headers(
+                        &item.url,
+                        vec![("Referer".to_string(), referer)],
+                        None,
+                    )
+                    .await
+                    .unwrap_or_default();
+                let (formats, tags) = Self::parse_detail(&html);
+                if !formats.is_empty() {
+                    item.formats = Some(formats);
+                }
+                if !tags.is_empty() {
+                    item.tags = Some(tags);
+                }
+                item
+            }
+        }))
+        .buffered(ENRICH_CONCURRENCY)
+        .collect::<Vec<_>>()
+        .await
+    }
+
     fn get_video_items_from_html(&self, html: String) -> Vec<VideoItem> {
         if html.is_empty() {
             return vec![];
@@ -280,15 +383,37 @@ impl Rule34videoProvider {
                 Self::extract_between(video_segment, "<div class=\"time\">", "<").unwrap_or("0:00");
             let duration = parse_time_to_seconds(raw_duration).unwrap_or(0) as u32;
 
-            // Views extraction
+            // Views extraction: `<div class="views video-views-count ..."
-            let views_segment = Self::extract_between(video_segment, "<div class=\"views\">", "<");
+            // title="Views: 13K">`. The sibling `views` div is the comment
-            let views_count_str = views_segment
+            // count, so match the title rather than the bare class.
-                .and_then(|s| s.split("</svg>").nth(1))
+            let views = Self::extract_between(video_segment, "title=\"Views: ", "\"")
-                .unwrap_or("0");
+                .and_then(|v| parse_abbreviated_number(v.trim()))
-            let views = parse_abbreviated_number(views_count_str.trim()).unwrap_or(0);
+                .or_else(|| {
+                    Self::extract_between(video_segment, "<div class=\"views\">", "</div>")
+                        .map(|s| s.rsplit("</svg>").next().unwrap_or(s))
+                        .and_then(|v| parse_abbreviated_number(v.trim()))
+                })
+                .unwrap_or(0);
 
-            // Thumbnail extraction
+            // Rating extraction: `72% (47)` inside the rating div.
-            let thumb = Self::extract_between(video_segment, "data-original=\"", "\"")
+            let rating = Self::extract_between(video_segment, "custom-thumbs-up", "</div>")
+                .and_then(|s| s.rsplit("</svg>").next())
+                .and_then(|s| s.trim().split('%').next())
+                .and_then(|s| s.trim().parse::<f32>().ok());
+
+            // Thumbnail extraction: the site dropped `data-original`. Lazy
+            // cards are `<img class="thumb rd-lz" src="data:image/gif..."
+            // data-rd-src="...">`; eager ones carry the real `src`.
+            let thumb = Self::extract_between(video_segment, "<img class=\"thumb", ">")
+                .and_then(|tag| {
+                    Self::extract_between(tag, "data-rd-src=\"", "\"")
+                        .or_else(|| {
+                            Self::extract_between(tag, " src=\"", "\"")
+                                .filter(|src| src.starts_with("http"))
+                        })
+                        .or_else(|| Self::extract_between(tag, "data-rd-jpg=\"", "\""))
+                })
+                .or_else(|| Self::extract_between(video_segment, "data-original=\"", "\""))
                 .unwrap_or("")
                 .to_string();
 
@@ -298,10 +423,13 @@ impl Rule34videoProvider {
                     .unwrap_or("")
                     .to_string();
 
-            items.push(
+            let mut item =
                 VideoItem::new(id, title, url, "Rule34video".to_string(), thumb, duration)
-                    .views(views),
+                    .views(views);
-            );
+            if let Some(rating) = rating {
+                item = item.rating(rating);
+            }
+            items.push(item);
         }
         items
     }
@@ -339,3 +467,54 @@ impl Provider for Rule34videoProvider {
         Some(self.build_channel(clientversion))
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn parses_card_thumb_from_img_src_and_skips_ads() {
+        let html = r#"<div id="custom_list_videos_x"><div class="item thumb video_1   " data-video-card-id="42">
+<a class="th js-open-popup" href="https://rule34video.com/video/42/some-title/"  title="Some Title">
+<div class="img wrap_image"><img class="thumb" src="https://rule34video.com/contents/videos_screenshots/0/42/336x189/3.jpg" data-rd-jpg="x" alt="a">
+<div class="time">1:28</div></div><div class="thumb_title">Some Title</div>
+<div class="rating"><svg class="custom-svg custom-thumbs-up"></svg> 72% (47) </div>
+<div class="views video-comments-count" title="Comments: 0"><svg></svg> 0 </div>
+<div class="views video-views-count" title="Views: 6.9K"><svg></svg> 6.9K </div></a></div>
+<div class="item thumb video_3"><a class="th js-open-popup" href="https://rule34video.com/video/43/lazy/" title="Lazy">
+<div class="img wrap_image"><img class="thumb rd-lz" src="data:image/gif;base64,R0lG" data-rd-src="https://rule34video.com/contents/videos_screenshots/0/43/336x189/3.jpg" alt="b">
+<div class="time">0:30</div></div><div class="thumb_title">Lazy</div></a></div>
+<div class="item thumb video_2"><a class="th" href="https://ads" title="Advertisement"></a></div>
+<div class="pagination" id="p">"#;
+        let items = Rule34videoProvider::new().get_video_items_from_html(html.to_string());
+        assert_eq!(items.len(), 2);
+        assert_eq!(items[0].id, "42");
+        assert_eq!(items[0].views, Some(6900));
+        assert_eq!(items[0].rating, Some(72.0));
+        assert_eq!(
+            items[1].thumb,
+            "https://rule34video.com/contents/videos_screenshots/0/43/336x189/3.jpg"
+        );
+        assert_eq!(
+            items[0].thumb,
+            "https://rule34video.com/contents/videos_screenshots/0/42/336x189/3.jpg"
+        );
+        assert_eq!(items[0].duration, 88);
+    }
+
+    #[test]
+    fn parses_flashvars_formats_and_tags() {
+        let html = r#"var flashvars = { video_id: '1', video_categories: '3D, Cyberpunk 2077', video_tags: 'bondage, panam palmer (cyberpunk), 3d', video_url: 'https://rule34video.com/get_file/a_360.mp4/?v-acctoken=1', video_url_text: '360p', video_alt_url: 'https://rule34video.com/get_file/a_480p.mp4/?v-acctoken=2', video_alt_url_text: '480p', video_alt_url2: 'https://rule34video.com/get_file/a_720p.mp4/?v-acctoken=3', video_alt_url2_text: '720p', video_title: 'It\'s' };"#;
+        let (formats, tags) = Rule34videoProvider::parse_detail(html);
+        assert_eq!(formats.len(), 3);
+        assert!(formats[0].url.contains("a_720p"), "best quality first");
+        assert!(formats[2].url.contains("a_360"));
+        assert_eq!(tags, vec!["3D", "Cyberpunk 2077", "bondage", "panam palmer (cyberpunk)"]);
+    }
+
+    #[test]
+    fn detail_without_flashvars_yields_nothing() {
+        let (formats, tags) = Rule34videoProvider::parse_detail("<html></html>");
+        assert!(formats.is_empty() && tags.is_empty());
+    }
+}