use crate::DbPool; use crate::api::ClientVersion; use crate::providers::{Provider, report_provider_error, report_provider_error_background, requester_or_default}; use crate::status::*; use crate::uploaders::{UploaderChannelStat, UploaderLayoutRow, UploaderProfile, UploaderVideoRef}; use crate::util::cache::VideoCache; use crate::videos::{ServerOptions, VideoFormat, VideoItem}; use async_trait::async_trait; use chrono::DateTime; use error_chain::error_chain; use futures::stream::{self, StreamExt}; use percent_encoding::{NON_ALPHANUMERIC, utf8_percent_encode}; use serde::Deserialize; use std::sync::Arc; use tokio::sync::RwLock; pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata = crate::providers::ProviderChannelMetadata { group_id: "tiktok", tags: &["shortform", "redgifs", "amateur", "swipe"], }; const BASE_URL: &str = "https://xxxtik.com"; const API_BASE: &str = "https://xxxtik-api-iw98m.ondigitalocean.app"; const REDGIFS_API: &str = "https://api.redgifs.com/v2"; const CHANNEL_ID: &str = "xxxtik"; const DEFAULT_PER_PAGE: usize = 20; const MAX_PER_PAGE: usize = 30; const RESOLVE_CONCURRENCY: usize = 8; // xxxtik's listing endpoints are cursor-based (the numeric `id` of the last item // returned), not page-number based, so reaching page N requires walking N // sequential requests from the start of the feed. const MAX_REFILL_FETCHES: usize = 4; const MAX_PAGE_WALK: u16 = 25; // xxxtik exposes ~60k tags through /tag/options, far too many to background-load // as a filter catalog. This is a small curated set of popular tags for the // `categories` shortcut option; any other tag still works via a `tag:`/`cat:` // query prefix or the live `/search` lookup. const CURATED_CATEGORIES: &[&str] = &[ "amateur", "asian", "anal", "ass", "big tits", "blowjob", "brunette", "cumshot", "hardcore", "lesbian", "milf", "pussy", "redhead", "teen (18+)", "public sex", "latina", "ebony", "squirting", "creampie", "threesome", ]; error_chain! { foreign_links { Json(serde_json::Error); } errors { Request(msg: String) { description("request error") display("request error: {}", msg) } NotFound(msg: String) { description("not found") display("not found: {}", msg) } } } #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum Sort { New, TopWeek, TopMonth, TopYear, TopAll, } impl Sort { fn from_sort_id(value: &str) -> Self { match value.trim().to_ascii_lowercase().as_str() { "top" | "top_week" | "week" => Sort::TopWeek, "top_month" | "month" => Sort::TopMonth, "top_year" | "year" => Sort::TopYear, "top_all" | "all" | "alltime" | "all_time" => Sort::TopAll, _ => Sort::New, } } fn list_path(self) -> &'static str { match self { Sort::New => "post/new", Sort::TopWeek => "post/top/week", Sort::TopMonth => "post/top/month", Sort::TopYear => "post/top/year", Sort::TopAll => "post/top/all", } } } #[derive(Debug, Clone)] enum Target { Feed(Sort), Tag(String), Creator(String), Empty, } #[derive(Debug, Deserialize, Clone, Default)] #[serde(rename_all = "camelCase")] struct Post { id: i64, #[serde(default)] uuid: String, #[serde(default)] description: Option, #[serde(default)] source: Option, #[serde(default)] status: Option, #[serde(default)] created_at: String, #[serde(default)] author: PostAuthor, #[serde(default)] tags: Vec, } #[derive(Debug, Deserialize, Clone, Default)] struct PostAuthor { #[serde(default)] name: String, } #[derive(Debug, Deserialize, Clone, Default)] struct PostTag { #[serde(default)] name: String, } #[derive(Debug, Deserialize, Clone, Default)] struct SearchResult { #[serde(default)] name: String, #[serde(default, rename = "type")] kind: String, } #[derive(Debug, Deserialize, Clone, Default)] struct UserProfileResp { #[serde(default)] name: String, #[serde(default)] description: Option, #[serde(default, rename = "_count")] count: Option, } #[derive(Debug, Deserialize, Clone, Default)] struct UserCount { #[serde(default)] posts: u64, } #[derive(Debug, Deserialize, Clone, Default)] struct TokenResponse { #[serde(default)] token: String, } #[derive(Debug, Deserialize, Clone, Default)] struct RedgifsResponse { #[serde(default)] gif: RedgifsGif, } #[derive(Debug, Deserialize, Clone, Default)] struct RedgifsGif { #[serde(default)] duration: f64, #[serde(default)] views: u32, #[serde(default)] width: u32, #[serde(default)] height: u32, #[serde(default)] tags: Vec, #[serde(default)] #[serde(rename = "userName")] user_name: String, #[serde(default)] urls: RedgifsUrls, } #[derive(Debug, Deserialize, Clone, Default)] struct RedgifsUrls { #[serde(default)] hd: Option, #[serde(default)] sd: Option, #[serde(default)] poster: Option, #[serde(default)] thumbnail: Option, } #[derive(Debug, Clone)] pub struct XxxtikProvider { redgifs_token: Arc>>, } impl XxxtikProvider { pub fn new() -> Self { Self { redgifs_token: Arc::new(RwLock::new(None)), } } fn title_case(value: &str) -> String { value .split(' ') .map(|word| { let mut chars = word.chars(); match chars.next() { Some(first) => first.to_uppercase().collect::() + chars.as_str(), None => String::new(), } }) .collect::>() .join(" ") } fn normalize_tag(value: &str) -> String { value.trim().to_ascii_lowercase() } fn build_channel(&self, _clientversion: ClientVersion) -> Channel { let category_titles = CURATED_CATEGORIES .iter() .map(|name| Self::title_case(name)) .collect::>(); let category_options = CURATED_CATEGORIES .iter() .map(|name| FilterOption { id: name.to_string(), title: Self::title_case(name), }) .collect::>(); Channel { id: CHANNEL_ID.to_string(), name: "xxxtik".to_string(), description: "xxxtik short-form TikTok-style clips, an aggregator of moderated RedGifs reposts with tag and creator browsing.".to_string(), premium: false, favicon: "https://www.google.com/s2/favicons?sz=64&domain=xxxtik.com".to_string(), status: "active".to_string(), categories: category_titles, options: vec![ ChannelOption { id: "sort".to_string(), title: "Sort".to_string(), description: "Browse xxxtik by latest or top-rated over a time period.".to_string(), systemImage: "arrow.up.arrow.down".to_string(), colorName: "blue".to_string(), options: vec![ FilterOption { id: "new".to_string(), title: "Latest".to_string(), }, FilterOption { id: "top_week".to_string(), title: "Top This Week".to_string(), }, FilterOption { id: "top_month".to_string(), title: "Top This Month".to_string(), }, FilterOption { id: "top_year".to_string(), title: "Top This Year".to_string(), }, FilterOption { id: "top_all".to_string(), title: "Top All Time".to_string(), }, ], multiSelect: false, }, ChannelOption { id: "categories".to_string(), title: "Tags".to_string(), description: "Open a popular xxxtik tag feed directly.".to_string(), systemImage: "tag".to_string(), colorName: "orange".to_string(), options: category_options, multiSelect: false, }, ], nsfw: true, cacheDuration: Some(1800), ytdlpCommand: Some("yt-dlp --impersonate chrome-120".to_string()) } } fn resolve_query_target(query: &str) -> Option { let trimmed = query.trim(); let (kind, value) = trimmed.split_once(':')?; let value = value.trim(); if value.is_empty() { return None; } match kind.trim().to_ascii_lowercase().as_str() { "tag" | "category" | "cat" => Some(Target::Tag(Self::normalize_tag(value))), "user" | "uploader" | "creator" => Some(Target::Creator(value.to_string())), _ => None, } } async fn search_target(&self, query: &str, options: &ServerOptions) -> Result { let encoded = utf8_percent_encode(query.trim(), NON_ALPHANUMERIC).to_string(); let url = format!("{API_BASE}/search?query={encoded}"); let mut requester = requester_or_default(options, CHANNEL_ID, "search_target"); let text = requester .get_with_headers(&url, vec![("Referer".to_string(), format!("{BASE_URL}/"))], None) .await .map_err(|error| Error::from(format!("request failed for {url}: {error}")))?; let results: Vec = serde_json::from_str(&text)?; let normalized_query = query.trim().to_ascii_lowercase(); if let Some(exact_tag) = results .iter() .find(|item| item.kind == "tag" && item.name.to_ascii_lowercase() == normalized_query) { return Ok(Target::Tag(Self::normalize_tag(&exact_tag.name))); } if let Some(exact_profile) = results .iter() .find(|item| item.kind == "profile" && item.name.to_ascii_lowercase() == normalized_query) { return Ok(Target::Creator(exact_profile.name.clone())); } if let Some(first_tag) = results.iter().find(|item| item.kind == "tag") { return Ok(Target::Tag(Self::normalize_tag(&first_tag.name))); } if let Some(first_profile) = results.iter().find(|item| item.kind == "profile") { return Ok(Target::Creator(first_profile.name.clone())); } Ok(Target::Empty) } async fn pick_target(&self, query: Option<&str>, sort: &str, options: &ServerOptions) -> Result { if let Some(query) = query { if !query.trim().is_empty() { if let Some(target) = Self::resolve_query_target(query) { return Ok(target); } return self.search_target(query, options).await; } } if let Some(category) = options.categories.as_deref() { let category = category.trim(); if !category.is_empty() && category != "all" { return Ok(Target::Tag(Self::normalize_tag(category))); } } Ok(Target::Feed(Sort::from_sort_id(sort))) } fn parse_timestamp(value: &str) -> Option { let value = value.trim(); if value.is_empty() { return None; } DateTime::parse_from_rfc3339(value) .ok() .map(|parsed| parsed.timestamp().max(0) as u64) } fn extract_gif_id(source: &str) -> Option { let without_query = source.split(['?', '#']).next().unwrap_or(source); let id = without_query.trim_end_matches('/').rsplit('/').next()?; if id.is_empty() { None } else { Some(id.to_string()) } } async fn fetch_one_page( &self, path: &str, cursor: i64, per_page: usize, options: &ServerOptions, ) -> Result> { let url = format!("{API_BASE}/{path}?limit={per_page}&cursor={cursor}"); let mut requester = requester_or_default(options, CHANNEL_ID, "fetch_one_page"); let text = requester .get_with_headers(&url, vec![("Referer".to_string(), format!("{BASE_URL}/"))], None) .await .map_err(|error| Error::from(format!("request failed for {url}: {error}")))?; Ok(serde_json::from_str(&text)?) } async fn fetch_cursor_page( &self, path: &str, page: u16, per_page: usize, options: &ServerOptions, ) -> Result> { let page = page.max(1).min(MAX_PAGE_WALK); let mut cursor: i64 = 0; let mut items = Vec::new(); for _ in 0..page { // Tag and creator pages mix in posts whose source is not RedGifs // (erome links, empty sources); those cannot be resolved here and // used to leave whole pages empty ("girl" returned nothing). Keep // pulling raw batches, bounded, until a page has enough // resolvable posts. let mut collected: Vec = Vec::new(); let mut exhausted = false; for _ in 0..MAX_REFILL_FETCHES { let batch = self.fetch_one_page(path, cursor, per_page, options).await?; if batch.is_empty() { exhausted = true; break; } cursor = batch.last().map(|post| post.id).unwrap_or(cursor); let short_batch = batch.len() < per_page; collected.extend(batch.into_iter().filter(Self::is_redgifs_post)); if collected.len() >= per_page || short_batch { break; } } if exhausted && collected.is_empty() { items = Vec::new(); break; } items = collected; } Ok(items) } fn is_redgifs_post(post: &Post) -> bool { post.source .as_deref() .map(|source| source.contains("redgifs.com/")) .unwrap_or(false) } async fn fetch_target_posts( &self, target: &Target, page: u16, per_page: usize, options: &ServerOptions, ) -> Result> { match target { Target::Feed(sort) => self.fetch_cursor_page(sort.list_path(), page, per_page, options).await, Target::Tag(name) => { let encoded = utf8_percent_encode(name, NON_ALPHANUMERIC).to_string(); let path = format!("post/tag/{encoded}"); self.fetch_cursor_page(&path, page, per_page, options).await } Target::Creator(name) => { let encoded = utf8_percent_encode(name, NON_ALPHANUMERIC).to_string(); let path = format!("post/creator/{encoded}"); self.fetch_cursor_page(&path, page, per_page, options).await } Target::Empty => Ok(Vec::new()), } } async fn redgifs_token(&self, options: &ServerOptions, force_refresh: bool) -> Result { if !force_refresh { if let Some(token) = self.redgifs_token.read().await.clone() { return Ok(token); } } let mut guard = self.redgifs_token.write().await; if !force_refresh { if let Some(token) = guard.clone() { return Ok(token); } } let url = format!("{REDGIFS_API}/auth/temporary"); let mut requester = requester_or_default(options, CHANNEL_ID, "redgifs_token"); let text = requester .get_with_headers(&url, vec![("Referer".to_string(), format!("{BASE_URL}/"))], None) .await .map_err(|error| Error::from(format!("request failed for {url}: {error}")))?; let parsed: TokenResponse = serde_json::from_str(&text)?; if parsed.token.is_empty() { return Err(Error::from(format!("empty redgifs token response for {url}"))); } *guard = Some(parsed.token.clone()); Ok(parsed.token) } async fn resolve_redgifs(&self, gif_id: &str, options: &ServerOptions) -> Result { for attempt in 0..2 { let force_refresh = attempt > 0; let token = self.redgifs_token(options, force_refresh).await?; let url = format!("{REDGIFS_API}/gifs/{gif_id}"); let mut requester = requester_or_default(options, CHANNEL_ID, "resolve_redgifs"); let response = requester .get_raw_with_headers( &url, vec![ ("Authorization".to_string(), format!("Bearer {token}")), ("Referer".to_string(), "https://www.redgifs.com/".to_string()), ], ) .await .map_err(|error| Error::from(format!("request failed for {url}: {error}")))?; if response.status().as_u16() == 401 && attempt == 0 { continue; } if !response.status().is_success() { return Err(Error::from(format!( "redgifs resolve failed for {gif_id}: status={}", response.status() ))); } let text = response .text() .await .map_err(|error| Error::from(format!("response read failed for {url}: {error}")))?; let parsed: RedgifsResponse = serde_json::from_str(&text)?; return Ok(parsed.gif); } Err(Error::from(format!("redgifs resolve failed for {gif_id}: exhausted retries"))) } async fn build_video_item(&self, post: Post, options: &ServerOptions) -> Option { if let Some(status) = post.status.as_deref() { let status = status.trim().to_ascii_lowercase(); if !status.is_empty() && status != "approved" { return None; } } if post.uuid.is_empty() { return None; } let source = post.source.as_deref()?; let gif_id = Self::extract_gif_id(source)?; let gif = match self.resolve_redgifs(&gif_id, options).await { Ok(gif) => gif, Err(error) => { report_provider_error_background(CHANNEL_ID, "build_video_item.resolve_redgifs", &error.to_string()); return None; } }; let media_url = gif.urls.hd.clone().or_else(|| gif.urls.sd.clone())?; let thumb = gif.urls.poster.clone().or_else(|| gif.urls.thumbnail.clone())?; let title = post .description .as_deref() .map(str::trim) .filter(|value| !value.is_empty()) .map(ToOwned::to_owned) .unwrap_or_else(|| { if !post.author.name.trim().is_empty() { format!("xxxtik clip by {}", post.author.name.trim()) } else { format!("xxxtik clip {}", post.uuid) } }); let url = format!("{BASE_URL}/feed/{}", post.uuid); let duration = gif.duration.round().max(0.0) as u32; let mut item = VideoItem::new(post.uuid.clone(), title, url, CHANNEL_ID.to_string(), thumb, duration); item.views = Some(gif.views); item.uploadedAt = Self::parse_timestamp(&post.created_at); let uploader_name = if !post.author.name.trim().is_empty() { post.author.name.trim().to_string() } else { gif.user_name.trim().to_string() }; if !uploader_name.is_empty() { item.uploader = Some(uploader_name.clone()); item.uploaderUrl = Some(format!("{BASE_URL}/profile/{uploader_name}")); item.uploaderId = Some(format!("{CHANNEL_ID}:{uploader_name}")); } let mut tags = post .tags .iter() .map(|tag| tag.name.clone()) .filter(|name| !name.trim().is_empty()) .collect::>(); for tag in &gif.tags { let lower = tag.to_ascii_lowercase(); if !tags.iter().any(|existing| existing.to_ascii_lowercase() == lower) { tags.push(tag.clone()); } } item.tags = (!tags.is_empty()).then_some(tags); if gif.height > 0 { item.aspectRatio = Some(gif.width as f32 / gif.height as f32); } item.formats = Some(vec![VideoFormat::new(media_url, "hd".to_string(), "mp4".to_string())]); Some(item) } async fn fetch_target_items( &self, target: Target, page: u16, per_page: usize, options: &ServerOptions, ) -> Result> { let posts = self.fetch_target_posts(&target, page, per_page, options).await?; let items = stream::iter(posts.into_iter().map(|post| { let options = options.clone(); async move { self.build_video_item(post, &options).await } })) .buffer_unordered(RESOLVE_CONCURRENCY) .filter_map(async move |v| v) .collect::>() .await; Ok(items) } async fn fetch_user_profile(&self, username: &str, options: &ServerOptions) -> Result> { let encoded = utf8_percent_encode(username, NON_ALPHANUMERIC).to_string(); let url = format!("{API_BASE}/user/by-username/{encoded}"); let mut requester = requester_or_default(options, CHANNEL_ID, "fetch_user_profile"); let text = match requester .get_with_headers(&url, vec![("Referer".to_string(), format!("{BASE_URL}/"))], None) .await { Ok(text) => text, Err(_) => return Ok(None), }; match serde_json::from_str::(&text) { Ok(profile) if !profile.name.trim().is_empty() => Ok(Some(profile)), _ => Ok(None), } } async fn build_uploader_profile( &self, username: &str, query: Option<&str>, profile_content: bool, options: &ServerOptions, ) -> Result> { let Some(profile) = self.fetch_user_profile(username, options).await? else { return Ok(None); }; let canonical_id = format!("{CHANNEL_ID}:{}", profile.name); let video_count = profile.count.as_ref().map(|count| count.posts).unwrap_or(0); let mut videos = None; let mut total_views: u64 = 0; if profile_content { let items = self .fetch_target_items(Target::Creator(profile.name.clone()), 1, 24, options) .await?; let filtered_items = if let Some(query) = query.filter(|value| !value.trim().is_empty()) { let normalized_query = query.to_ascii_lowercase(); items .into_iter() .filter(|item| { let haystack = format!( "{} {}", item.title, item.tags.as_ref().map(|values| values.join(" ")).unwrap_or_default() ) .to_ascii_lowercase(); haystack.contains(&normalized_query) }) .collect::>() } else { items }; total_views = filtered_items.iter().filter_map(|item| item.views).map(|v| v as u64).sum(); let refs = filtered_items .iter() .map(|item| UploaderVideoRef::from_video_item(item, &profile.name, &canonical_id)) .collect::>(); videos = Some(refs); } Ok(Some(UploaderProfile { id: canonical_id, name: profile.name.clone(), url: Some(format!("{BASE_URL}/profile/{}", profile.name)), channel: Some(CHANNEL_ID.to_string()), verified: false, videoCount: video_count, totalViews: total_views, channels: Some(vec![UploaderChannelStat { channel: CHANNEL_ID.to_string(), videoCount: video_count, firstSeenAt: None, lastSeenAt: None, }]), avatar: None, description: profile.description.clone(), bio: profile.description, videos, tapes: Some(vec![]), playlists: Some(vec![]), layout: Some(vec![UploaderLayoutRow::videos(Some("Posts".to_string()))]), })) } } #[async_trait] impl Provider for XxxtikProvider { async fn get_videos( &self, cache: VideoCache, pool: DbPool, sort: String, query: Option, page: String, per_page: String, options: ServerOptions, ) -> Vec { let _ = cache; let _ = pool; let page = page.parse::().unwrap_or(1).max(1); let per_page = per_page .parse::() .unwrap_or(DEFAULT_PER_PAGE) .clamp(1, MAX_PER_PAGE); let normalized_query = query .as_deref() .map(str::trim) .filter(|value| !value.is_empty()); let target = match self.pick_target(normalized_query, &sort, &options).await { Ok(target) => target, Err(error) => { report_provider_error(CHANNEL_ID, "get_videos.pick_target", &error.to_string()).await; return vec![]; } }; match self.fetch_target_items(target, page, per_page, &options).await { Ok(items) => items, Err(error) => { report_provider_error(CHANNEL_ID, "get_videos", &error.to_string()).await; vec![] } } } fn get_channel(&self, clientversion: ClientVersion) -> Option { Some(self.build_channel(clientversion)) } async fn get_uploader( &self, cache: VideoCache, pool: DbPool, uploader_id: Option, uploader_name: Option, query: Option, profile_content: bool, options: ServerOptions, ) -> std::result::Result, String> { let _ = cache; let _ = pool; let username = uploader_id .as_deref() .and_then(|id| id.strip_prefix(&format!("{CHANNEL_ID}:"))) .map(ToOwned::to_owned) .or(uploader_name.clone()) .map(|value| value.trim().to_string()) .filter(|value| !value.is_empty()); let Some(username) = username else { return Ok(None); }; self.build_uploader_profile(&username, query.as_deref(), profile_content, &options) .await .map_err(|error| error.to_string()) } } #[cfg(test)] mod tests { #[test] fn only_redgifs_sources_are_resolvable() { let mk = |source: Option<&str>| super::Post { id: 1, uuid: "u".to_string(), description: None, source: source.map(str::to_string), status: None, created_at: String::new(), author: Default::default(), tags: vec![], }; assert!(XxxtikProvider::is_redgifs_post(&mk(Some("https://www.redgifs.com/watch/abc")))); assert!(XxxtikProvider::is_redgifs_post(&mk(Some("https://redgifs.com/watch/abc")))); assert!(!XxxtikProvider::is_redgifs_post(&mk(Some("https://www.erome.com/a/CE5z46ne#39")))); assert!(!XxxtikProvider::is_redgifs_post(&mk(Some("")))); assert!(!XxxtikProvider::is_redgifs_post(&mk(None))); } use super::*; #[test] fn resolves_query_shortcuts() { match XxxtikProvider::resolve_query_target("tag:Big Tits") { Some(Target::Tag(name)) => assert_eq!(name, "big tits"), other => panic!("expected tag target, got {other:?}"), } match XxxtikProvider::resolve_query_target("user:SomeCreator") { Some(Target::Creator(name)) => assert_eq!(name, "SomeCreator"), other => panic!("expected creator target, got {other:?}"), } assert!(XxxtikProvider::resolve_query_target("blonde teen").is_none()); } #[test] fn maps_sort_ids() { assert_eq!(Sort::from_sort_id("new"), Sort::New); assert_eq!(Sort::from_sort_id("top_week"), Sort::TopWeek); assert_eq!(Sort::from_sort_id("month"), Sort::TopMonth); assert_eq!(Sort::from_sort_id("unknown"), Sort::New); } #[test] fn extracts_gif_id_from_source() { assert_eq!( XxxtikProvider::extract_gif_id("https://www.redgifs.com/watch/fastlightsteelbluexenotarsosaurus"), Some("fastlightsteelbluexenotarsosaurus".to_string()) ); assert_eq!( XxxtikProvider::extract_gif_id("https://v3.redgifs.com/watch/790203707073529740"), Some("790203707073529740".to_string()) ); assert_eq!( XxxtikProvider::extract_gif_id("https://www.redgifs.com/ifr/lastgrimycrossbill"), Some("lastgrimycrossbill".to_string()) ); } #[test] fn parses_post_payload() { let json = r#"{ "id": 123, "uuid": "abc-uuid", "description": "Test post", "source": "https://www.redgifs.com/watch/someid", "status": "approved", "createdAt": "2026-01-01T00:00:00.000Z", "author": {"name": "creator"}, "tags": [{"name": "asian"}] }"#; let post: Post = serde_json::from_str(json).expect("parses"); assert_eq!(post.id, 123); assert_eq!(post.author.name, "creator"); assert_eq!(post.tags.len(), 1); } }