diff --git a/build.rs b/build.rs index a318035..cba9fb3 100644 --- a/build.rs +++ b/build.rs @@ -386,6 +386,11 @@ const PROVIDERS: &[ProviderDef] = &[ module: "xxxtik", ty: "XxxtikProvider", }, + ProviderDef { + id: "kwiky", + module: "kwiky", + ty: "KwikyProvider", + }, ]; fn main() { diff --git a/docs/provider-catalog.md b/docs/provider-catalog.md index 6d7dede..4677f54 100644 --- a/docs/provider-catalog.md +++ b/docs/provider-catalog.md @@ -70,6 +70,7 @@ This is the current implementation inventory as of this snapshot of the repo. Us | `youporn` | `mainstream-tube` | no | no | Pornhub-network HTML provider with watch-page playback URLs and tag/channel/pornstar shortcuts. | | `tube8` | `mainstream-tube` | no | yes | Aylo/MindGeek platform scraper; redirect proxy fetches signed `/media/hls/?s=TOKEN` endpoint and returns highest-quality CDN HLS URL; supports tag/category/channel/pornstar shortcut queries. | | `jable` | `jav` | no | yes | HTML JAV archive scraper; extracts `var hlsUrl` from detail pages; m3u8 format requires Referer + browser User-Agent; proxy route handles HEAD (200 OK) and GET (redirect to watch page) since yt-dlp blocks jable.tv; tag/category/model shortcut queries. | +| `kwiky` | `tiktok` | no | no | Short-form cam clip provider for kwiky.com (ICF/StreamMate platform). Two-step fetch: ID list then metadata batch. Auth: XSRF token extracted from `meta.content = ''` injected by `/api/state/v1/preload`, cached in an `Arc>>` with double-checked locking (same pattern as `xxxtik` bearer token); all API calls also require `x-gateway: 76d59e04-97fa-4ced-aa99-b86ffaf756a2` (platform-identifying fixed UUID observed in browser JS, not stored in cookies or localStorage). Default feed: `GET /api/v1/gateway/v1/quickies/recommended?source=` (cursor-paged, walks up to 5 cursors for higher page numbers). All tag/keyword queries go through `keywordSearch?keyword=...` (offset-paged: `pageToken=(page-1)*perPage`) — the recommended endpoint's `&tag=` param does not reliably filter, so even curated-tag requests use keywordSearch. Media: `media.url` = direct `media.icfcdn.com/*.mp4` (publicly accessible, no Referer/auth), `media.thumbUrl` thumbnail (same CDN, direct), `media.previewUrl` preview clip; both kwiky page URLs (`kwiky.com/quickies/{id}`) and the media CDN URLs pass direct `curl -I` health checks but the page URL is NOT yt-dlp-resolvable (React SPA, generic extractor fails), so `formats` are populated with the direct MP4 URL and `video.url` is the page. Aspect ratio set from `media.width`/`media.height`; `uploadedAt` from RFC3339 `created` field; uploader name/URL/ID from `creator.name` and `creator.id`. 48 curated tags from `/api/v1/gateway/v1/keywords/populartags?gender=f` exposed via `categories` filter option. No proxy needed. | | `fullporner` | `mainstream-tube` | no | no | HTML scraper for fullporner.com; thumbnail IDs derived from `/thumb/{id}.jpg` URLs and used to build direct `xiaoshenke.net/vid/{id}/720` media redirect URLs (Referer + User-Agent headers required); supports cat:/category:/pornstar:/star: shortcut queries; no proxy needed. | | `thepornbunny` | `mainstream-tube` | no | yes | KVS-style HTML scraper for thepornbunny.com; 24 items per site page; thumbnails at `https://www.thepornbunny.com/images/thumb/{id}.webp` from `data-original` attribute (no proxy needed); studio exposed as uploader; pornstar names in tags; `/proxy/thepornbunny/{slug}` fetches the video page, extracts `generate_mp4(enc_data, key, rnd, video_id)` args, decrypts `enc_data` via PBKDF2-HMAC-SHA512+AES-256-CBC to get an OK.ru session key, calls `api.ok.ru/fb.do?method=video.get&session_key=KEY&vids=RND` to get signed CDN URLs, and returns 302 to the best-quality okcdn.ru/vkuser.net MP4 URL (no special client headers needed); supports sort: new/popular/rated, 20 hardcoded categories via `categories` option, and tag:/category:/studio:/pornstar: query shortcuts. | | `eporner` | `mainstream-tube` | no | no | HTML scraper for eporner.com (5M+ videos); card selector `div.mb[data-id]` with inline duration/rating/views/uploader; thumbnails at `static-eu-cdn.eporner.com` (no proxy needed); pagination uses `/{N}/` suffix (page 1 = no suffix, page 2 = `/2/`); search queries map to `/tag/{slug}/` (eporner redirects all keyword searches to tag pages — 404 tag pages still return related content); supports sort: new/popular/rated/best; 65 hardcoded categories via `cat:`, `tag:`, `pornstar:`, `uploader:` query shortcuts; background-loads pornstar name→URL map from `/pornstar-list/`; yt-dlp resolves `video.url` natively (Eporner extractor); no proxy needed. | diff --git a/src/providers/kwiky.rs b/src/providers/kwiky.rs new file mode 100644 index 0000000..bbceb1c --- /dev/null +++ b/src/providers/kwiky.rs @@ -0,0 +1,453 @@ +use std::collections::HashMap; +use std::sync::Arc; +use async_trait::async_trait; +use chrono::DateTime; +use percent_encoding::{NON_ALPHANUMERIC, utf8_percent_encode}; +use serde::Deserialize; +use tokio::sync::RwLock; + +use crate::DbPool; +use crate::api::ClientVersion; +use crate::providers::{Provider, report_provider_error, requester_or_default}; +use crate::status::*; +use crate::util::cache::VideoCache; +use crate::videos::{ServerOptions, VideoFormat, VideoItem}; + +pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata = + crate::providers::ProviderChannelMetadata { + group_id: "tiktok", + tags: &["shortform", "cam", "amateur", "swipe"], + }; + +const CHANNEL_ID: &str = "kwiky"; +const BASE_URL: &str = "https://kwiky.com"; +const API_GW: &str = "https://kwiky.com/api/v1/gateway/v1"; +// Observed fixed gateway UUID from browser — identifies the Kwiky/ICF platform gateway. +const X_GATEWAY: &str = "76d59e04-97fa-4ced-aa99-b86ffaf756a2"; +// base64("https://recommendedquickies-webservice.icfsys.com/v2/KW/hotlist") +const SOURCE_HOTLIST: &str = "aHR0cHM6Ly9yZWNvbW1lbmRlZHF1aWNraWVzLXdlYnNlcnZpY2UuaWNmc3lzLmNvbS92Mi9LVy9ob3RsaXN0"; +const DEFAULT_PER_PAGE: usize = 25; +const MAX_PAGE_WALK: u16 = 5; + +// Curated tags from /api/v1/gateway/v1/keywords/populartags?gender=f +const CURATED_TAGS: &[&str] = &[ + "anal", "ass", "asian", "babe", "bbc", "bbw", "bdsm", "bigass", "bigtits", + "blowjob", "brunette", "cameltoe", "clit", "cosplay", "couples", "cowgirl", + "cum", "curvy", "deepthroat", "dildo", "ebony", "feet", "fingering", "fishnets", + "fuckme", "hugetits", "latina", "leather", "lesbian", "masturbation", "mature", + "milf", "panties", "petite", "pov", "pussy", "redhead", "ride", "shower", + "smalltits", "spank", "sucking", "tattoos", "tease", "tits", "tongue", + "toys", "twerk", +]; + +// --------------------------------------------------------------------------- +// Target routing +// --------------------------------------------------------------------------- + +#[derive(Debug, Clone)] +enum Target { + Recommended, + Search(String), +} + +// --------------------------------------------------------------------------- +// API response types +// --------------------------------------------------------------------------- + +#[derive(Debug, Deserialize)] +struct IdListResponse { + recommended: Vec, + #[serde(rename = "pageToken", default)] + page_token: Option, +} + +#[derive(Debug, Deserialize)] +struct IdEntry { + id: String, +} + +type MetadataMap = HashMap; + +#[derive(Debug, Deserialize, Default)] +struct QuickyEntry { + media: Option, + creator: Option, +} + +#[derive(Debug, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +struct QuickyMedia { + #[serde(default)] + description: String, + #[serde(default)] + thumb_url: String, + #[serde(default)] + preview_url: String, + #[serde(default)] + url: String, + #[serde(default)] + duration: u32, + #[serde(default)] + created: String, + #[serde(default)] + width: u32, + #[serde(default)] + height: u32, +} + +#[derive(Debug, Deserialize, Default)] +struct QuickyCreator { + #[serde(default)] + id: u64, + #[serde(default)] + name: String, +} + +// --------------------------------------------------------------------------- +// Provider +// --------------------------------------------------------------------------- + +pub struct KwikyProvider { + xsrf: Arc>>, +} + +impl KwikyProvider { + pub fn new() -> Self { + Self { + xsrf: Arc::new(RwLock::new(None)), + } + } + + fn api_headers(xsrf: &str) -> Vec<(String, String)> { + vec![ + ("Referer".to_string(), format!("{BASE_URL}/")), + ("x-xsrf-token".to_string(), xsrf.to_string()), + ("x-gateway".to_string(), X_GATEWAY.to_string()), + ("Accept".to_string(), "application/json, text/plain, */*".to_string()), + ] + } + + fn parse_xsrf(body: &str) -> Option { + let needle = "meta.content = '"; + let start = body.find(needle)?; + let after = &body[start + needle.len()..]; + let end = after.find('\'')?; + let token = after[..end].trim().to_string(); + if token.is_empty() { None } else { Some(token) } + } + + async fn get_xsrf(&self, options: &ServerOptions) -> Option { + { + let guard = self.xsrf.read().await; + if let Some(ref token) = *guard { + return Some(token.clone()); + } + } + + let mut guard = self.xsrf.write().await; + // Double-check after acquiring write lock + if let Some(ref token) = *guard { + return Some(token.clone()); + } + + let mut requester = requester_or_default(options, CHANNEL_ID, "get_xsrf"); + + // Fetch main page first to establish session cookies, then get XSRF token + let _ = requester.get(BASE_URL, None).await; + + let preload_url = format!("{BASE_URL}/api/state/v1/preload"); + let body = match requester.get(&preload_url, None).await { + Ok(body) => body, + Err(e) => { + report_provider_error(CHANNEL_ID, "get_xsrf", &format!("preload failed: {e}")).await; + return None; + } + }; + + let token = Self::parse_xsrf(&body); + if token.is_none() { + report_provider_error(CHANNEL_ID, "get_xsrf", "xsrf token not found in preload").await; + } + *guard = token.clone(); + token + } + + fn invalidate_xsrf(&self) { + if let Ok(mut guard) = self.xsrf.try_write() { + *guard = None; + } + } + + async fn fetch_id_list(&self, url: &str, xsrf: &str, options: &ServerOptions) -> Option { + let mut requester = requester_or_default(options, CHANNEL_ID, "fetch_id_list"); + let text = match requester.get_with_headers(url, Self::api_headers(xsrf), None).await { + Ok(text) => text, + Err(e) => { + let msg = e.to_string(); + if msg.contains("401") || msg.contains("Unauthorized") { + self.invalidate_xsrf(); + } + report_provider_error(CHANNEL_ID, "fetch_id_list", &format!("request failed for {url}: {e}")).await; + return None; + } + }; + + match serde_json::from_str::(&text) { + Ok(resp) => Some(resp), + Err(e) => { + report_provider_error(CHANNEL_ID, "fetch_id_list", &format!("parse failed: {e}")).await; + None + } + } + } + + async fn fetch_metadata(&self, ids: &[String], xsrf: &str, options: &ServerOptions) -> MetadataMap { + if ids.is_empty() { + return MetadataMap::new(); + } + let ids_param = ids.join(","); + let url = format!("{API_GW}/quickies/metadata?quickieIds={ids_param}"); + + let mut requester = requester_or_default(options, CHANNEL_ID, "fetch_metadata"); + let text = match requester.get_with_headers(&url, Self::api_headers(xsrf), None).await { + Ok(text) => text, + Err(e) => { + report_provider_error(CHANNEL_ID, "fetch_metadata", &format!("request failed: {e}")).await; + return MetadataMap::new(); + } + }; + + serde_json::from_str::(&text).unwrap_or_default() + } + + async fn recommended_ids( + &self, + page: u16, + per_page: usize, + xsrf: &str, + options: &ServerOptions, + ) -> Vec { + let mut cursor: Option = None; + let mut ids = Vec::new(); + + let walk = page.min(MAX_PAGE_WALK); + for _ in 0..walk { + let mut url = format!( + "{API_GW}/quickies/recommended?pageSize={per_page}&source={SOURCE_HOTLIST}&gender=f" + ); + if let Some(ref token) = cursor { + url.push_str(&format!("&pageToken={token}")); + } + + let resp = match self.fetch_id_list(&url, xsrf, options).await { + Some(resp) => resp, + None => return vec![], + }; + + ids = resp.recommended.into_iter().map(|e| e.id).collect(); + cursor = resp.page_token.filter(|t| !t.trim().is_empty()); + + if ids.is_empty() || cursor.is_none() { + break; + } + } + + ids + } + + async fn search_ids( + &self, + keyword: &str, + page: u16, + per_page: usize, + xsrf: &str, + options: &ServerOptions, + ) -> Vec { + let page_offset = (page as usize - 1) * per_page; + let encoded = utf8_percent_encode(keyword, NON_ALPHANUMERIC).to_string(); + let mut url = format!( + "{API_GW}/quickies/keywordSearch?pageSize={per_page}&keyword={encoded}&gender=f" + ); + if page_offset > 0 { + url.push_str(&format!("&pageToken={page_offset}")); + } + + match self.fetch_id_list(&url, xsrf, options).await { + Some(resp) => resp.recommended.into_iter().map(|e| e.id).collect(), + None => vec![], + } + } + + fn pick_target(query: Option<&str>, options: &ServerOptions) -> Target { + // Any query (raw keyword, tag: prefix, or #tag) routes to keywordSearch. + // The recommended endpoint's &tag= param does not reliably filter; + // keywordSearch handles both keywords and tag names. + if let Some(q) = query.map(str::trim).filter(|q| !q.is_empty()) { + let keyword = if let Some((kind, value)) = q.split_once(':') { + match kind.trim().to_ascii_lowercase().as_str() { + "tag" | "category" | "cat" | "genre" => value.trim().to_string(), + _ => q.to_string(), + } + } else if let Some(tag) = q.strip_prefix('#') { + tag.trim().to_string() + } else { + q.to_string() + }; + return Target::Search(keyword); + } + + if let Some(cat) = options.categories.as_deref().filter(|c| *c != "all" && !c.trim().is_empty()) { + return Target::Search(cat.to_string()); + } + + Target::Recommended + } + + fn build_video_item(id: &str, entry: QuickyEntry) -> Option { + let media = entry.media?; + if media.url.trim().is_empty() { + return None; + } + + let title = if media.description.trim().is_empty() { + format!("Kwiky clip {id}") + } else { + media.description.trim().to_string() + }; + + let page_url = format!("{BASE_URL}/quickies/{id}"); + let thumb = media.thumb_url.clone(); + let duration = media.duration; + + let mut item = VideoItem::new(id.to_string(), title, page_url, CHANNEL_ID.to_string(), thumb, duration); + + // Direct MP4 playback via formats (page URL is not yt-dlp-resolvable) + let format = VideoFormat::new(media.url, "auto".to_string(), "mp4".to_string()); + item.formats = Some(vec![format]); + + if !media.preview_url.trim().is_empty() { + item.preview = Some(media.preview_url); + } + + if media.width > 0 && media.height > 0 { + item.aspectRatio = Some(media.width as f32 / media.height as f32); + } + + if !media.created.trim().is_empty() { + if let Ok(dt) = DateTime::parse_from_rfc3339(&media.created) { + item.uploadedAt = Some(dt.timestamp().max(0) as u64); + } + } + + if let Some(creator) = entry.creator { + if !creator.name.trim().is_empty() { + let name = creator.name.trim().to_string(); + item.uploader = Some(name.clone()); + item.uploaderUrl = Some(format!("{BASE_URL}/profile/{name}")); + if creator.id > 0 { + item.uploaderId = Some(format!("{CHANNEL_ID}:{}", creator.id)); + } + } + } + + Some(item) + } + + fn build_channel(&self, _cv: ClientVersion) -> Channel { + let category_options: Vec = CURATED_TAGS + .iter() + .map(|tag| FilterOption { + id: tag.to_string(), + title: capitalize_first(tag), + }) + .collect(); + + Channel { + id: CHANNEL_ID.to_string(), + name: "Kwiky".to_string(), + description: "Kwiky short-form cam clips — amateur performer quickies with direct MP4 playback and tag browsing.".to_string(), + premium: false, + favicon: "https://www.google.com/s2/favicons?sz=64&domain=kwiky.com".to_string(), + status: "active".to_string(), + categories: CURATED_TAGS.iter().map(|t| capitalize_first(t)).collect(), + options: vec![ + ChannelOption { + id: "categories".to_string(), + title: "Tags".to_string(), + description: "Browse Kwiky clips by tag".to_string(), + systemImage: "tag".to_string(), + colorName: "orange".to_string(), + options: category_options, + multiSelect: false, + }, + ], + nsfw: true, + cacheDuration: Some(1800), + } + } +} + +fn capitalize_first(s: &str) -> String { + let mut chars = s.chars(); + match chars.next() { + Some(first) => first.to_uppercase().collect::() + chars.as_str(), + None => String::new(), + } +} + +#[async_trait] +impl Provider for KwikyProvider { + async fn get_videos( + &self, + cache: VideoCache, + pool: DbPool, + sort: String, + query: Option, + page: String, + per_page: String, + options: ServerOptions, + ) -> Vec { + let _ = cache; + let _ = pool; + let _ = sort; + + let page = page.parse::().unwrap_or(1).max(1); + let per_page = per_page + .parse::() + .unwrap_or(DEFAULT_PER_PAGE) + .clamp(1, 50); + + let normalized_query = query.as_deref().map(str::trim).filter(|q| !q.is_empty()); + let target = Self::pick_target(normalized_query, &options); + + let xsrf = match self.get_xsrf(&options).await { + Some(xsrf) => xsrf, + None => return vec![], + }; + + let ids = match &target { + Target::Recommended => { + self.recommended_ids(page, per_page, &xsrf, &options).await + } + Target::Search(keyword) => { + self.search_ids(keyword, page, per_page, &xsrf, &options).await + } + }; + + if ids.is_empty() { + return vec![]; + } + + let mut metadata = self.fetch_metadata(&ids, &xsrf, &options).await; + + ids.into_iter() + .filter_map(|id| { + metadata.remove(&id).and_then(|entry| Self::build_video_item(&id, entry)) + }) + .collect() + } + + fn get_channel(&self, clientversion: ClientVersion) -> Option { + Some(self.build_channel(clientversion)) + } +}