rule34video: artist, category and tag archives; resolve bare queries by probing; credit artists as uploader

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MUYdCh2nCVsQYgJAT7gwSi

src/providers/rule34video.rs

@@ -1,6 +1,6 @@
 use crate::DbPool;
 use crate::api::ClientVersion;
-use crate::providers::Provider;
+use crate::providers::{Provider, probe_first_archive, requester_or_default, slug_dashed};
 use crate::status::*;
 use crate::util::cache::VideoCache;
 use crate::util::discord::send_discord_error_report;
@@ -12,7 +12,9 @@ use async_trait::async_trait;
 use error_chain::error_chain;
 use futures::stream::{self, StreamExt};
 use htmlentity::entity::{ICodedDataTrait, decode};
+use scraper::{Html, Selector};
 use std::time::{SystemTime, UNIX_EPOCH};
+use percent_encoding::{NON_ALPHANUMERIC, utf8_percent_encode};
 use std::vec;
 
 const ENRICH_CONCURRENCY: usize = 6;
@@ -36,6 +38,29 @@ error_chain! {
     }
 }
 
+/// What a request resolves to. Artists live under `/models/`, franchises and
+/// genres under `/categories/`, characters and everything else under `/tags/`.
+#[derive(Debug, Clone, PartialEq, Eq)]
+enum Target {
+    Latest,
+    Search(String),
+    Artist(String),
+    Category(String),
+    Tag(String),
+}
+
+impl Target {
+    /// Archive path (`models/{slug}`, ...) for the archive targets.
+    fn archive_path(&self) -> Option<String> {
+        match self {
+            Target::Artist(slug) => Some(format!("models/{slug}")),
+            Target::Category(slug) => Some(format!("categories/{slug}")),
+            Target::Tag(slug) => Some(format!("tags/{slug}")),
+            _ => None,
+        }
+    }
+}
+
 #[derive(Debug, Clone)]
 pub struct Rule34videoProvider {
     url: String,
@@ -93,6 +118,156 @@ impl Rule34videoProvider {
         }
     }
 
+    /// Explicit `artist:` / `category:` / `tag:` style prefixes (the server has
+    /// already stripped a leading `#`).
+    fn parse_prefixed(query: &str) -> Option<Target> {
+        let (kind, value) = query.trim().split_once(':')?;
+        let slug = slug_dashed(value);
+        if slug.is_empty() {
+            return None;
+        }
+        match kind.trim().to_ascii_lowercase().as_str() {
+            "artist" | "artists" | "model" | "models" | "author" | "creator" | "uploader" => {
+                Some(Target::Artist(slug))
+            }
+            "cat" | "category" | "categories" | "franchise" | "series" | "game" => {
+                Some(Target::Category(slug))
+            }
+            "tag" | "tags" | "character" | "char" => Some(Target::Tag(slug)),
+            _ => None,
+        }
+    }
+
+    /// Titles of the tags offered by `/search_ajax.php?tag=`.
+    fn parse_tag_suggestions(html: &str) -> Vec<String> {
+        let document = Html::parse_fragment(html);
+        let Ok(selector) = Selector::parse("input[data-title]") else {
+            return vec![];
+        };
+        document
+            .select(&selector)
+            .filter_map(|input| input.value().attr("data-title"))
+            .map(|title| title.trim().to_string())
+            .filter(|title| !title.is_empty())
+            .collect()
+    }
+
+    /// The suggested tag whose name (ignoring a trailing "(franchise)") equals
+    /// the query, e.g. "samus aran" -> "samus aran (metroid)".
+    fn best_tag_title(titles: &[String], query: &str) -> Option<String> {
+        let wanted = slug_dashed(query);
+        if wanted.is_empty() {
+            return None;
+        }
+        let mut exact = titles.iter().filter(|title| {
+            let base = title.split('(').next().unwrap_or(title);
+            slug_dashed(base) == wanted || slug_dashed(title) == wanted
+        });
+        let first = exact.next()?;
+        // Several franchises share a character name: ambiguous, leave it to search.
+        exact.next().is_none().then(|| first.clone())
+    }
+
+    /// A free-form query that names an artist, category or tag goes to that
+    /// archive (probed on the site; unknown names 404), else keyword search.
+    async fn resolve_keyword(&self, query: &str, options: &ServerOptions) -> Target {
+        let slug = slug_dashed(query);
+        if slug.is_empty() {
+            return Target::Search(query.trim().to_string());
+        }
+        let candidates = [
+            Target::Artist(slug.clone()),
+            Target::Category(slug.clone()),
+            Target::Tag(slug),
+        ];
+        let urls: Vec<String> = candidates
+            .iter()
+            .filter_map(|t| t.archive_path())
+            .map(|path| format!("{}/{path}/", self.url))
+            .collect();
+        let mut requester = requester_or_default(options, "rule34video", "resolve_keyword");
+        let referer = format!("{}/", self.url);
+        if let Some((index, _)) = probe_first_archive(&mut requester, &urls, &referer, |body| {
+            body.contains("item thumb video_")
+        })
+        .await
+        {
+            return candidates[index].clone();
+        }
+
+        // Characters are tagged "name (franchise)": ask the site's own tag search.
+        let suggest_url = format!(
+            "{}/search_ajax.php?tag={}",
+            self.url,
+            utf8_percent_encode(query.trim(), NON_ALPHANUMERIC)
+        );
+        let headers = vec![
+            ("Referer".to_string(), referer),
+            ("X-Requested-With".to_string(), "XMLHttpRequest".to_string()),
+        ];
+        if let Ok(Some(body)) = requester.get_probe(&suggest_url, headers).await {
+            let titles = Self::parse_tag_suggestions(&body);
+            if let Some(title) = Self::best_tag_title(&titles, query) {
+                return Target::Tag(slug_dashed(&title));
+            }
+        }
+        Target::Search(query.trim().to_string())
+    }
+
+    async fn resolve_target(&self, query: Option<&str>, options: &ServerOptions) -> Target {
+        let Some(query) = query.map(str::trim).filter(|q| !q.is_empty()) else {
+            return Target::Latest;
+        };
+        if let Some(target) = Self::parse_prefixed(query) {
+            return target;
+        }
+        self.resolve_keyword(query, options).await
+    }
+
+    async fn archive(
+        &self,
+        cache: VideoCache,
+        target: &Target,
+        page: u8,
+        sort: &str,
+        options: ServerOptions,
+    ) -> Result<Vec<VideoItem>> {
+        let Some(path) = target.archive_path() else {
+            return Ok(vec![]);
+        };
+        let sort_val = match sort {
+            "post_date" | "video_viewed" | "rating" | "duration" | "pseudo_random" => sort,
+            _ => "post_date",
+        };
+        let index = format!("rule34video:{path}:{page}:{sort_val}");
+        if sort_val != "pseudo_random" {
+            if let Some((time, items)) = cache.get(&index) {
+                if time.elapsed().unwrap_or_default().as_secs() < 300 {
+                    return Ok(items.clone());
+                }
+            }
+        }
+
+        let mut requester = options.requester.clone().ok_or("Requester missing")?;
+        let url = format!(
+            "{}/{path}/?mode=async&function=get_block&block_id=custom_list_videos_common_videos&sort_by={sort_val}&from={page}",
+            self.url
+        );
+        let headers = vec![("Referer".to_string(), format!("{}/{path}/", self.url))];
+        let text = match requester.get_probe(&url, headers).await {
+            Ok(Some(text)) => text,
+            // A 404 is an unknown archive (or a page past the end).
+            Ok(None) => return Ok(vec![]),
+            Err(e) => return Err(format!("archive fetch failed for {url}: {e}").into()),
+        };
+        let video_items = self.get_video_items_from_html(text);
+        let video_items = self.enrich_items(video_items, requester).await;
+        if !video_items.is_empty() {
+            cache.insert(index, video_items.clone());
+        }
+        Ok(video_items)
+    }
+
     /// Helper to safely extract a string between two delimiters
     fn extract_between<'a>(content: &'a str, start_pat: &str, end_pat: &str) -> Option<&'a str> {
         let start_idx = content.find(start_pat)? + start_pat.len();
@@ -307,11 +482,44 @@ impl Rule34videoProvider {
         (formats.into_iter().map(|(_, f)| f).collect(), tags)
     }
 
+    /// Artists credited on a video page: `(name, slug)` from the "Artist" group.
+    fn parse_artists(html: &str) -> Vec<(String, String)> {
+        let document = Html::parse_document(html);
+        let Ok(group_sel) = Selector::parse("[data-suggest-type=\"model\"] a[href*=\"/models/\"]")
+        else {
+            return vec![];
+        };
+        let name_sel = Selector::parse(".name").ok();
+        let mut artists: Vec<(String, String)> = Vec::new();
+        for link in document.select(&group_sel) {
+            let href = link.value().attr("href").unwrap_or_default();
+            let slug = href
+                .split("/models/")
+                .nth(1)
+                .unwrap_or_default()
+                .trim_matches('/')
+                .to_string();
+            let name = name_sel
+                .as_ref()
+                .and_then(|sel| link.select(sel).next())
+                .map(|n| n.text().collect::<String>())
+                .unwrap_or_else(|| link.text().collect::<String>());
+            let name = name.split_whitespace().collect::<Vec<_>>().join(" ");
+            if slug.is_empty() || name.is_empty() || artists.iter().any(|(_, s)| *s == slug) {
+                continue;
+            }
+            artists.push((name, slug));
+        }
+        artists
+    }
+
     async fn enrich_items(&self, items: Vec<VideoItem>, requester: Requester) -> Vec<VideoItem> {
         let referer = format!("{}/", self.url);
+        let base = self.url.clone();
         stream::iter(items.into_iter().map(|mut item| {
             let mut requester = requester.clone();
             let referer = referer.clone();
+            let base = base.clone();
             async move {
                 if item.url.is_empty() {
                     return item;
@@ -324,10 +532,21 @@ impl Rule34videoProvider {
                     )
                     .await
                     .unwrap_or_default();
-                let (formats, tags) = Self::parse_detail(&html);
+                let (formats, mut tags) = Self::parse_detail(&html);
                 if !formats.is_empty() {
                     item.formats = Some(formats);
                 }
+                let artists = Self::parse_artists(&html);
+                if let Some((name, slug)) = artists.first() {
+                    item.uploader = Some(name.clone());
+                    item.uploaderUrl = Some(format!("{base}/models/{slug}/"));
+                    item.uploaderId = Some(format!("rule34video:{slug}"));
+                }
+                for (name, _) in &artists {
+                    if !tags.iter().any(|t| t.eq_ignore_ascii_case(name)) {
+                        tags.push(name.clone());
+                    }
+                }
                 if !tags.is_empty() {
                     item.tags = Some(tags);
                 }
@@ -449,9 +668,11 @@ impl Provider for Rule34videoProvider {
     ) -> Vec<VideoItem> {
         let page_num = page.parse::<u8>().unwrap_or(1);
 
-        let result = match query {
-            Some(q) => self.query(cache, page_num, &q, &sort, options).await,
-            None => self.get(cache, page_num, &sort, options).await,
+        let target = self.resolve_target(query.as_deref(), &options).await;
+        let result = match &target {
+            Target::Latest => self.get(cache, page_num, &sort, options).await,
+            Target::Search(q) => self.query(cache, page_num, q, &sort, options).await,
+            archive => self.archive(cache, archive, page_num, &sort, options).await,
         };
 
         match result {
@@ -517,4 +738,51 @@ mod tests {
         let (formats, tags) = Rule34videoProvider::parse_detail("<html></html>");
         assert!(formats.is_empty() && tags.is_empty());
     }
+
+    #[test]
+    fn parses_prefixed_queries() {
+        assert_eq!(
+            Rule34videoProvider::parse_prefixed("artist:General Butch"),
+            Some(Target::Artist("general-butch".into()))
+        );
+        assert_eq!(
+            Rule34videoProvider::parse_prefixed("category:Super Smash Bros"),
+            Some(Target::Category("super-smash-bros".into()))
+        );
+        assert_eq!(
+            Rule34videoProvider::parse_prefixed("tag:black widow (marvel)"),
+            Some(Target::Tag("black-widow-marvel".into()))
+        );
+        assert_eq!(Rule34videoProvider::parse_prefixed("samus aran"), None);
+    }
+
+    #[test]
+    fn picks_the_unambiguous_character_tag() {
+        let titles = vec![
+            "samus aran (metroid)".to_string(),
+            "samus aran's mother".to_string(),
+        ];
+        assert_eq!(
+            Rule34videoProvider::best_tag_title(&titles, "samus aran").as_deref(),
+            Some("samus aran (metroid)")
+        );
+        let ambiguous = vec!["peach (mario)".to_string(), "peach (pokemon)".to_string()];
+        assert_eq!(Rule34videoProvider::best_tag_title(&ambiguous, "peach"), None);
+    }
+
+    #[test]
+    fn parses_tag_suggestions_and_artists() {
+        let suggest = r#"<div class="item checkbox-container"><input id="tag_143" type="checkbox" value="143" data-title="samus aran (metroid)"/></div>"#;
+        assert_eq!(
+            Rule34videoProvider::parse_tag_suggestions(suggest),
+            vec!["samus aran (metroid)".to_string()]
+        );
+        let page = r#"<div class="col vp-group" data-suggest-type="model"><div class="label">Artist</div>
+            <span class="chip"><a class="item btn_link" href="https://rule34video.com/models/joigang-va/">
+            <span class="name">JOIGang (VA)</span></a></span></div>"#;
+        assert_eq!(
+            Rule34videoProvider::parse_artists(page),
+            vec![("JOIGang (VA)".to_string(), "joigang-va".to_string())]
+        );
+    }
 }