From 1f9977062c6a6cc5da665b53bd20bc32e38bd3fd Mon Sep 17 00:00:00 2001 From: Simon Date: Mon, 6 Jul 2026 11:51:52 +0000 Subject: [PATCH] redgifs --- build.rs | 5 + docs/provider-catalog.md | 1 + src/providers/redgifs.rs | 672 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 678 insertions(+) create mode 100644 src/providers/redgifs.rs diff --git a/build.rs b/build.rs index 9be97a5..9df09e0 100644 --- a/build.rs +++ b/build.rs @@ -396,6 +396,11 @@ const PROVIDERS: &[ProviderDef] = &[ module: "hotbunny", ty: "HotbunnyProvider", }, + ProviderDef { + id: "redgifs", + module: "redgifs", + ty: "RedgifsProvider", + }, ]; fn main() { diff --git a/docs/provider-catalog.md b/docs/provider-catalog.md index f29d239..5ab1e42 100644 --- a/docs/provider-catalog.md +++ b/docs/provider-catalog.md @@ -49,6 +49,7 @@ This is the current implementation inventory as of this snapshot of the repo. Us | `pornmz` | `mainstream-tube` | no | no | Mainstream archive. | | `pornzog` | `mainstream-tube` | no | no | Basic list/detail scraper. | | `porntrex` | `mainstream-tube` | no | no | KVS-style HTML archive with direct MP4 formats and tag-aware search shortcuts. | +| `redgifs` | `amateur-homemade` | yes | no | Direct integration against the public RedGifs v2 API (same backend `xxxtik` proxies through, but consumed natively here). Auth: anonymous bearer token from `GET api.redgifs.com/v2/auth/temporary`, cached in an `Arc>>` with double-checked locking, refreshed once on a 401 via a centralized `authed_get` helper. Default feed: `GET /v2/feeds/trending/{popular,established}?page=&count=` (honors `count` exactly). Free-text query and `tag:`/`category:`/`cat:` shortcuts hit `GET /v2/gifs/search?query=|tags=&order=&page=&count=`; `user:`/`uploader:`/`creator:` shortcuts hit `GET /v2/users/{username}/search?page=&count=`. Uploader profile (`GET /v1/users/{username}`) backs `/api/uploaders`, with `profileContent:true` pulling the creator's own gif listing for `videoCount`/`totalViews`. Listing JSON already carries duration/poster/tags/uploader per item, so no per-item enrichment call is needed (`build_video_item` is a plain sync function, unlike `xxxtik`'s `buffer_unordered` resolve step). `video.url` is the yt-dlp-native `redgifs.com/watch/{id}` page; `formats` intentionally left unset. Thumbnails (`urls.poster`/`urls.thumbnail`) are directly hotlinkable, no proxy needed. 20 curated tags exposed via `categories`. **Known upstream quirk:** `/v2/gifs/search` (query/tag targets only — trending and creator listings are unaffected) returns noticeably fewer items than the requested `count` on some pages (e.g. `count=20` → ~14); not compensated for by over-fetching, since inflating `count` while keeping the same `page` would desync the API's own offset-based pagination cursor (`page × count`) and risk skipped/duplicated items across pages. | | `redtube` | `mainstream-tube` | no | no | Mainstream archive. | | `rule34gen` | `ai` | no | no | AI group example. | | `rule34video` | `hentai-animation` | no | no | Hentai group example. | diff --git a/src/providers/redgifs.rs b/src/providers/redgifs.rs new file mode 100644 index 0000000..ea60f32 --- /dev/null +++ b/src/providers/redgifs.rs @@ -0,0 +1,672 @@ +use crate::DbPool; +use crate::api::ClientVersion; +use crate::providers::{Provider, report_provider_error, requester_or_default}; +use crate::status::*; +use crate::uploaders::{UploaderChannelStat, UploaderLayoutRow, UploaderProfile, UploaderVideoRef}; +use crate::util::cache::VideoCache; +use crate::videos::{ServerOptions, VideoItem}; +use async_trait::async_trait; +use error_chain::error_chain; +use percent_encoding::{NON_ALPHANUMERIC, utf8_percent_encode}; +use serde::Deserialize; +use std::sync::Arc; +use tokio::sync::RwLock; + +pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata = + crate::providers::ProviderChannelMetadata { + group_id: "amateur-homemade", + tags: &["amateur", "gifs", "creators"], + }; + +const BASE_URL: &str = "https://www.redgifs.com"; +const API_ROOT: &str = "https://api.redgifs.com"; +const CHANNEL_ID: &str = "redgifs"; +const DEFAULT_PER_PAGE: usize = 20; +const MAX_PER_PAGE: usize = 30; + +// RedGifs' own trending feed only exposes two curated variations (matching the +// site's own "Trending" toggle). Its keyword/tag search endpoint separately +// takes an `order` enum (top, top7, top28, latest, score, trending); we reuse +// the same two-way sort selector for both surfaces rather than exposing every +// combination, mapping each to the closest search order semantically. +const CURATED_CATEGORIES: &[&str] = &[ + "Amateur", + "Homemade", + "Solo", + "Lesbian", + "Anal", + "Blowjob", + "Cumshot", + "Big Tits", + "Teen (18+)", + "MILF", + "Asian", + "Ebony", + "Latina", + "Redhead", + "Public", + "Threesome", + "Squirting", + "Creampie", + "Pussy", + "Ass", +]; + +error_chain! { + foreign_links { + Json(serde_json::Error); + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Sort { + Popular, + Established, +} + +impl Sort { + fn from_sort_id(value: &str) -> Self { + match value.trim().to_ascii_lowercase().as_str() { + "established" => Sort::Established, + _ => Sort::Popular, + } + } + + fn trending_variation(self) -> &'static str { + match self { + Sort::Popular => "popular", + Sort::Established => "established", + } + } + + fn search_order(self) -> &'static str { + match self { + Sort::Popular => "trending", + Sort::Established => "top", + } + } +} + +#[derive(Debug, Clone)] +enum Target { + Trending(Sort), + Tag(String, Sort), + Query(String, Sort), + Creator(String), +} + +#[derive(Debug, Deserialize, Clone, Default)] +#[serde(rename_all = "camelCase")] +struct Gif { + #[serde(default)] + id: String, + #[serde(default)] + create_date: i64, + #[serde(default)] + duration: f64, + #[serde(default)] + views: u32, + #[serde(default)] + width: u32, + #[serde(default)] + height: u32, + #[serde(default)] + tags: Vec, + #[serde(default)] + user_name: String, + #[serde(default, rename = "type")] + kind: i32, + #[serde(default)] + description: Option, + #[serde(default)] + urls: GifUrls, +} + +#[derive(Debug, Deserialize, Clone, Default)] +struct GifUrls { + #[serde(default)] + poster: Option, + #[serde(default)] + thumbnail: Option, +} + +#[derive(Debug, Deserialize, Clone, Default)] +struct SearchResponse { + #[serde(default)] + gifs: Vec, +} + +#[derive(Debug, Deserialize, Clone, Default)] +struct TokenResponse { + #[serde(default)] + token: String, +} + +#[derive(Debug, Deserialize, Clone, Default)] +#[serde(rename_all = "camelCase")] +struct UserProfileResp { + #[serde(default)] + username: String, + #[serde(default)] + description: Option, + #[serde(default)] + followers: u64, + #[serde(default)] + gifs: u64, + #[serde(default)] + profile_image_url: Option, + #[serde(default)] + verified: bool, +} + +#[derive(Debug, Clone)] +pub struct RedgifsProvider { + token: Arc>>, +} + +impl RedgifsProvider { + pub fn new() -> Self { + Self { + token: Arc::new(RwLock::new(None)), + } + } + + fn build_channel(&self, _clientversion: ClientVersion) -> Channel { + let category_titles = CURATED_CATEGORIES.iter().map(|name| name.to_string()).collect::>(); + let category_options = CURATED_CATEGORIES + .iter() + .map(|name| FilterOption { + id: name.to_string(), + title: name.to_string(), + }) + .collect::>(); + + Channel { + id: CHANNEL_ID.to_string(), + name: "RedGifs".to_string(), + description: "RedGifs amateur clip feed with trending/established sort, tag and creator browsing, and keyword search.".to_string(), + premium: false, + favicon: "https://www.google.com/s2/favicons?sz=64&domain=redgifs.com".to_string(), + status: "active".to_string(), + categories: category_titles, + options: vec![ + ChannelOption { + id: "sort".to_string(), + title: "Sort".to_string(), + description: "Browse RedGifs' trending feed, popular or established creators.".to_string(), + systemImage: "arrow.up.arrow.down".to_string(), + colorName: "blue".to_string(), + options: vec![ + FilterOption { + id: "popular".to_string(), + title: "Popular".to_string(), + }, + FilterOption { + id: "established".to_string(), + title: "Established Creators".to_string(), + }, + ], + multiSelect: false, + }, + ChannelOption { + id: "categories".to_string(), + title: "Tags".to_string(), + description: "Open a popular RedGifs tag feed directly.".to_string(), + systemImage: "tag".to_string(), + colorName: "orange".to_string(), + options: category_options, + multiSelect: false, + }, + ], + nsfw: true, + cacheDuration: Some(1800), + } + } + + fn resolve_query_target(query: &str, sort: Sort) -> Option { + let trimmed = query.trim(); + let (kind, value) = trimmed.split_once(':')?; + let value = value.trim(); + if value.is_empty() { + return None; + } + match kind.trim().to_ascii_lowercase().as_str() { + "tag" | "category" | "cat" => Some(Target::Tag(value.to_string(), sort)), + "user" | "uploader" | "creator" => Some(Target::Creator(value.to_string())), + _ => None, + } + } + + fn pick_target(query: Option<&str>, sort: Sort, options: &ServerOptions) -> Target { + if let Some(query) = query { + if !query.trim().is_empty() { + if let Some(target) = Self::resolve_query_target(query, sort) { + return target; + } + return Target::Query(query.trim().to_string(), sort); + } + } + + if let Some(category) = options.categories.as_deref() { + let category = category.trim(); + if !category.is_empty() && category != "all" { + return Target::Tag(category.to_string(), sort); + } + } + + Target::Trending(sort) + } + + async fn redgifs_token(&self, options: &ServerOptions, force_refresh: bool) -> Result { + if !force_refresh { + if let Some(token) = self.token.read().await.clone() { + return Ok(token); + } + } + + let mut guard = self.token.write().await; + if !force_refresh { + if let Some(token) = guard.clone() { + return Ok(token); + } + } + + let url = format!("{API_ROOT}/v2/auth/temporary"); + let mut requester = requester_or_default(options, CHANNEL_ID, "redgifs_token"); + let text = requester + .get_with_headers(&url, vec![("Referer".to_string(), format!("{BASE_URL}/"))], None) + .await + .map_err(|error| Error::from(format!("request failed for {url}: {error}")))?; + let parsed: TokenResponse = serde_json::from_str(&text)?; + if parsed.token.is_empty() { + return Err(Error::from(format!("empty redgifs token response for {url}"))); + } + *guard = Some(parsed.token.clone()); + Ok(parsed.token) + } + + async fn authed_get(&self, url: &str, options: &ServerOptions) -> Result { + for attempt in 0..2 { + let force_refresh = attempt > 0; + let token = self.redgifs_token(options, force_refresh).await?; + let mut requester = requester_or_default(options, CHANNEL_ID, "authed_get"); + let response = requester + .get_raw_with_headers( + url, + vec![ + ("Authorization".to_string(), format!("Bearer {token}")), + ("Referer".to_string(), format!("{BASE_URL}/")), + ], + ) + .await + .map_err(|error| Error::from(format!("request failed for {url}: {error}")))?; + + if response.status().as_u16() == 401 && attempt == 0 { + continue; + } + if !response.status().is_success() { + return Err(Error::from(format!("redgifs request failed for {url}: status={}", response.status()))); + } + return response + .text() + .await + .map_err(|error| Error::from(format!("response read failed for {url}: {error}"))); + } + Err(Error::from(format!("redgifs request failed for {url}: exhausted retries"))) + } + + async fn fetch_trending(&self, sort: Sort, page: u16, per_page: usize, options: &ServerOptions) -> Result> { + let url = format!( + "{API_ROOT}/v2/feeds/trending/{}?page={}&count={}", + sort.trending_variation(), + page, + per_page + ); + let text = self.authed_get(&url, options).await?; + let parsed: SearchResponse = serde_json::from_str(&text)?; + Ok(parsed.gifs) + } + + async fn fetch_tag(&self, tag: &str, sort: Sort, page: u16, per_page: usize, options: &ServerOptions) -> Result> { + let encoded = utf8_percent_encode(tag, NON_ALPHANUMERIC).to_string(); + let url = format!( + "{API_ROOT}/v2/gifs/search?tags={encoded}&order={}&page={page}&count={per_page}", + sort.search_order() + ); + let text = self.authed_get(&url, options).await?; + let parsed: SearchResponse = serde_json::from_str(&text)?; + Ok(parsed.gifs) + } + + async fn fetch_query(&self, text_query: &str, sort: Sort, page: u16, per_page: usize, options: &ServerOptions) -> Result> { + let encoded = utf8_percent_encode(text_query, NON_ALPHANUMERIC).to_string(); + let url = format!( + "{API_ROOT}/v2/gifs/search?query={encoded}&order={}&page={page}&count={per_page}", + sort.search_order() + ); + let text = self.authed_get(&url, options).await?; + let parsed: SearchResponse = serde_json::from_str(&text)?; + Ok(parsed.gifs) + } + + async fn fetch_creator(&self, username: &str, page: u16, per_page: usize, options: &ServerOptions) -> Result> { + let encoded = utf8_percent_encode(username, NON_ALPHANUMERIC).to_string(); + let url = format!("{API_ROOT}/v2/users/{encoded}/search?page={page}&count={per_page}"); + let text = self.authed_get(&url, options).await?; + let parsed: SearchResponse = serde_json::from_str(&text)?; + Ok(parsed.gifs) + } + + async fn fetch_target_gifs(&self, target: &Target, page: u16, per_page: usize, options: &ServerOptions) -> Result> { + match target { + Target::Trending(sort) => self.fetch_trending(*sort, page, per_page, options).await, + Target::Tag(name, sort) => self.fetch_tag(name, *sort, page, per_page, options).await, + Target::Query(text, sort) => self.fetch_query(text, *sort, page, per_page, options).await, + Target::Creator(name) => self.fetch_creator(name, page, per_page, options).await, + } + } + + // RedGifs' listing payloads already carry duration, poster, tags, and + // uploader for every item, so unlike xxxtik (which reposts links and must + // resolve each one against the RedGifs API individually) no per-item + // network call or bounded-concurrency enrichment step is needed here. + fn build_video_item(gif: Gif) -> Option { + // type 2 is a static image/gallery post with no video source. + if gif.kind != 1 || gif.id.is_empty() { + return None; + } + + let thumb = gif.urls.poster.clone().or_else(|| gif.urls.thumbnail.clone())?; + + let title = gif + .description + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(ToOwned::to_owned) + .unwrap_or_else(|| { + if !gif.user_name.trim().is_empty() { + format!("RedGifs clip by {}", gif.user_name.trim()) + } else { + format!("RedGifs clip {}", gif.id) + } + }); + + let url = format!("{BASE_URL}/watch/{}", gif.id); + let duration = gif.duration.round().max(0.0) as u32; + + let mut item = VideoItem::new(gif.id.clone(), title, url, CHANNEL_ID.to_string(), thumb, duration); + item.views = Some(gif.views); + item.uploadedAt = (gif.create_date > 0).then_some(gif.create_date as u64); + + if !gif.user_name.trim().is_empty() { + let username = gif.user_name.trim().to_string(); + item.uploader = Some(username.clone()); + item.uploaderUrl = Some(format!("{BASE_URL}/users/{username}")); + item.uploaderId = Some(format!("{CHANNEL_ID}:{username}")); + } + + let tags = gif + .tags + .iter() + .map(|tag| tag.trim().to_string()) + .filter(|tag| !tag.is_empty()) + .collect::>(); + item.tags = (!tags.is_empty()).then_some(tags); + + if gif.height > 0 { + item.aspectRatio = Some(gif.width as f32 / gif.height as f32); + } + + // video.url is the redgifs.com watch page, which yt-dlp resolves + // natively; formats is intentionally left unset. + Some(item) + } + + async fn fetch_target_items(&self, target: Target, page: u16, per_page: usize, options: &ServerOptions) -> Result> { + let gifs = self.fetch_target_gifs(&target, page, per_page, options).await?; + Ok(gifs.into_iter().filter_map(Self::build_video_item).collect()) + } + + async fn fetch_user_profile(&self, username: &str, options: &ServerOptions) -> Result> { + let encoded = utf8_percent_encode(username, NON_ALPHANUMERIC).to_string(); + let url = format!("{API_ROOT}/v1/users/{encoded}"); + let text = match self.authed_get(&url, options).await { + Ok(text) => text, + Err(_) => return Ok(None), + }; + match serde_json::from_str::(&text) { + Ok(profile) if !profile.username.trim().is_empty() => Ok(Some(profile)), + _ => Ok(None), + } + } + + async fn build_uploader_profile( + &self, + username: &str, + query: Option<&str>, + profile_content: bool, + options: &ServerOptions, + ) -> Result> { + let Some(profile) = self.fetch_user_profile(username, options).await? else { + return Ok(None); + }; + + let canonical_id = format!("{CHANNEL_ID}:{}", profile.username); + let video_count = profile.gifs; + let mut videos = None; + let mut total_views: u64 = 0; + + if profile_content { + let items = self + .fetch_target_items(Target::Creator(profile.username.clone()), 1, 24, options) + .await?; + + let filtered_items = if let Some(query) = query.filter(|value| !value.trim().is_empty()) { + let normalized_query = query.to_ascii_lowercase(); + items + .into_iter() + .filter(|item| { + let haystack = format!( + "{} {}", + item.title, + item.tags.as_ref().map(|values| values.join(" ")).unwrap_or_default() + ) + .to_ascii_lowercase(); + haystack.contains(&normalized_query) + }) + .collect::>() + } else { + items + }; + + total_views = filtered_items.iter().filter_map(|item| item.views).map(|v| v as u64).sum(); + + let refs = filtered_items + .iter() + .map(|item| UploaderVideoRef::from_video_item(item, &profile.username, &canonical_id)) + .collect::>(); + videos = Some(refs); + } + + Ok(Some(UploaderProfile { + id: canonical_id, + name: profile.username.clone(), + url: Some(format!("{BASE_URL}/users/{}", profile.username)), + channel: Some(CHANNEL_ID.to_string()), + verified: profile.verified, + videoCount: video_count, + totalViews: total_views, + channels: Some(vec![UploaderChannelStat { + channel: CHANNEL_ID.to_string(), + videoCount: video_count, + firstSeenAt: None, + lastSeenAt: None, + }]), + avatar: profile.profile_image_url, + description: profile.description.clone(), + bio: profile.description, + videos, + tapes: Some(vec![]), + playlists: Some(vec![]), + layout: Some(vec![UploaderLayoutRow::videos(Some("Gifs".to_string()))]), + })) + } +} + +#[async_trait] +impl Provider for RedgifsProvider { + async fn get_videos( + &self, + cache: VideoCache, + pool: DbPool, + sort: String, + query: Option, + page: String, + per_page: String, + options: ServerOptions, + ) -> Vec { + let _ = cache; + let _ = pool; + + let page = page.parse::().unwrap_or(1).max(1); + let per_page = per_page + .parse::() + .unwrap_or(DEFAULT_PER_PAGE) + .clamp(1, MAX_PER_PAGE); + let normalized_query = query + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()); + + let sort = Sort::from_sort_id(&sort); + let target = Self::pick_target(normalized_query, sort, &options); + + match self.fetch_target_items(target, page, per_page, &options).await { + Ok(items) => items, + Err(error) => { + report_provider_error(CHANNEL_ID, "get_videos", &error.to_string()).await; + vec![] + } + } + } + + fn get_channel(&self, clientversion: ClientVersion) -> Option { + Some(self.build_channel(clientversion)) + } + + async fn get_uploader( + &self, + cache: VideoCache, + pool: DbPool, + uploader_id: Option, + uploader_name: Option, + query: Option, + profile_content: bool, + options: ServerOptions, + ) -> std::result::Result, String> { + let _ = cache; + let _ = pool; + + let username = uploader_id + .as_deref() + .and_then(|id| id.strip_prefix(&format!("{CHANNEL_ID}:"))) + .map(ToOwned::to_owned) + .or(uploader_name.clone()) + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()); + + let Some(username) = username else { + return Ok(None); + }; + + self.build_uploader_profile(&username, query.as_deref(), profile_content, &options) + .await + .map_err(|error| error.to_string()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn resolves_query_shortcuts() { + match RedgifsProvider::resolve_query_target("tag:Big Tits", Sort::Popular) { + Some(Target::Tag(name, _)) => assert_eq!(name, "Big Tits"), + other => panic!("expected tag target, got {other:?}"), + } + + match RedgifsProvider::resolve_query_target("user:SomeCreator", Sort::Popular) { + Some(Target::Creator(name)) => assert_eq!(name, "SomeCreator"), + other => panic!("expected creator target, got {other:?}"), + } + + assert!(RedgifsProvider::resolve_query_target("blonde teen", Sort::Popular).is_none()); + } + + fn empty_options() -> ServerOptions { + ServerOptions { + featured: None, + category: None, + sites: None, + filter: None, + language: None, + public_url_base: None, + requester: None, + network: None, + stars: None, + categories: None, + duration: None, + sort: None, + sexuality: None, + } + } + + #[test] + fn falls_back_to_free_text_query() { + match RedgifsProvider::pick_target(Some("blonde teen"), Sort::Popular, &empty_options()) { + Target::Query(text, _) => assert_eq!(text, "blonde teen"), + other => panic!("expected query target, got {other:?}"), + } + } + + #[test] + fn maps_sort_ids() { + assert_eq!(Sort::from_sort_id("popular"), Sort::Popular); + assert_eq!(Sort::from_sort_id("established"), Sort::Established); + assert_eq!(Sort::from_sort_id("unknown"), Sort::Popular); + } + + #[test] + fn skips_image_gallery_items() { + let json = r#"{"id":"abc","type":2,"urls":{"poster":"https://media.redgifs.com/abc.jpg"}}"#; + let gif: Gif = serde_json::from_str(json).expect("parses"); + assert!(RedgifsProvider::build_video_item(gif).is_none()); + } + + #[test] + fn parses_gif_payload_and_builds_item() { + let json = r#"{ + "id": "someid", + "createDate": 1700000000, + "duration": 12.5, + "views": 42, + "width": 1080, + "height": 1920, + "tags": ["Amateur", "Solo"], + "userName": "creator1", + "type": 1, + "description": "", + "urls": {"poster": "https://media.redgifs.com/someid-poster.jpg", "thumbnail": "https://media.redgifs.com/someid-mobile.jpg"} + }"#; + + let gif: Gif = serde_json::from_str(json).expect("parses"); + let item = RedgifsProvider::build_video_item(gif).expect("builds item"); + assert_eq!(item.id, "someid"); + assert_eq!(item.url, "https://www.redgifs.com/watch/someid"); + assert!(item.formats.is_none()); + assert_eq!(item.uploaderId.as_deref(), Some("redgifs:creator1")); + assert_eq!(item.tags, Some(vec!["Amateur".to_string(), "Solo".to_string()])); + } +}