From a20a4ab64293d39e4084e3fa168af94a02f14f3e Mon Sep 17 00:00:00 2001 From: Simon Date: Sat, 10 Oct 2026 08:04:00 +0000 Subject: [PATCH] Fix coomer: offset pagination, per-query cache key, video-only posts The API ignores page= (every page 2+ repeated page 1), searches shared a single cache entry, image-only posts were listed as unplayable videos, and gzip/br bodies were not decoded because wreq had no decompression features. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01MUYdCh2nCVsQYgJAT7gwSi --- Cargo.toml | 2 +- docs/provider-fix-todo.md | 2 +- src/providers/coomer.rs | 164 ++++++++++++++++++++------------------ 3 files changed, 89 insertions(+), 79 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 8f9ec4a..8f55c69 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,7 +20,7 @@ ntex-files = "3" serde = "1.0.228" serde_json = "1.0.145" tokio = { version = "1.49", features = ["full"] } -wreq = { version = "6.0.0-rc.31", features = ["cookies", "multipart", "json"] } +wreq = { version = "6.0.0-rc.31", features = ["cookies", "multipart", "json", "gzip", "brotli", "deflate", "zstd"] } wreq-util = "3.0.0-rc.14" percent-encoding = "2.3.2" capitalize = "0.3.4" diff --git a/docs/provider-fix-todo.md b/docs/provider-fix-todo.md index ce255f0..88b483b 100644 --- a/docs/provider-fix-todo.md +++ b/docs/provider-fix-todo.md @@ -23,7 +23,7 @@ Status: `[ ]` open, `[~]` in progress, `[x]` fixed and verified, `[-]` not repro |---|---|---|---| | [x] | `thaiporntv` | page URL "Unsupported URL", no `formats` | player attribute renamed `data-enc` -> `data-video-sources` (reversed base64 JSON `{quality,url}`); decoder handles both. 3 formats/item, yt-dlp OK, check.py passes | | [x] | `pornmz` | page URL "Unsupported URL", no `formats` | site changed `itemprop="contentURL"` -> `contentUrl`; selector now case-insensitive. Enrichment concurrency 10 + 25s per-item timeout. Origin is slow (15s+/detail) so first feed 25-50s. check.py passes | -| [ ] | `coomer` | page URL "Unsupported URL"; media host CF-gated here | check whether `formats` are populated for videos | +| [x] | `coomer` | page URL "Unsupported URL"; media host CF-gated here | image-only posts were listed as videos with no playable URL: now dropped, and a video in `attachments` is picked up. Also fixed: `page=` is ignored by the API (every page 2+ was a copy of page 1; now `o=` offset), search results shared one cache key, and compressed API bodies were not decoded (enabled wreq gzip/br/deflate/zstd). n*.coomer.st CDN connect-times-out from this host, and the API 429s this IP after bursts, so playback/"girl" search could not be verified end to end | | [ ] | `tube8` | page URL 404; `/proxy/tube8/{id}` 502 | | | [ ] | `hypnotube` | page URLs 404 on all sampled items | URL construction | | [ ] | `fullpornerorg` | `lulustream` proxy 403 / 502 | | diff --git a/src/providers/coomer.rs b/src/providers/coomer.rs index 4941f06..d84b39d 100644 --- a/src/providers/coomer.rs +++ b/src/providers/coomer.rs @@ -37,6 +37,8 @@ const REFERER: &str = "https://coomer.st/"; const JSON_ACCEPT: &str = "text/css,*/*;q=0.1"; const CACHE_TTL_SECS: u64 = 60 * 15; const MAX_PAGE: u16 = 50; +/// The API pages by offset (`o`), in blocks of this many posts. `page=` is ignored. +const API_PAGE_SIZE: u16 = 50; const DEFAULT_PER_PAGE: usize = 30; const USER_POSTS_FETCH_CONCURRENCY: usize = 4; @@ -239,19 +241,18 @@ impl CoomerProvider { } } + fn offset_for(page: u16) -> u32 { + (page.max(1).min(MAX_PAGE) as u32 - 1) * API_PAGE_SIZE as u32 + } + fn build_main_url(target: &Target, page: u16) -> String { - let page = page.max(1).min(MAX_PAGE); + let offset = Self::offset_for(page); match target { - Target::Latest => format!("{}/api/v1/posts?limit=50&page={page}", BASE_URL), - Target::Popular => format!("{}/api/v1/posts/popular?limit=50&page={page}", BASE_URL), - // Search is non-paginated: ?page=N is ignored by the server for - // q=... requests, but we still emit a `page=1` for consistency. + Target::Latest => format!("{}/api/v1/posts?o={offset}", BASE_URL), + Target::Popular => format!("{}/api/v1/posts/popular?o={offset}", BASE_URL), Target::Search { query } => { let encoded = percent_encode(query); - format!( - "{}/api/v1/posts?q={encoded}&limit=50&page=1", - BASE_URL - ) + format!("{}/api/v1/posts?q={encoded}&o={offset}", BASE_URL) } // Uploader URL is built separately. Target::Uploader { .. } => String::new(), @@ -259,14 +260,12 @@ impl CoomerProvider { } fn build_uploader_url(service: &Service, username: &str, page: u16) -> String { - let page = page.max(1).min(MAX_PAGE); - // The user-posts endpoint is also non-paginated, but we send page=1 for - // clarity. Service comes from the enum so the path is stable. format!( - "{}/api/v1/{}/user/{}/posts?limit=50&page={page}", + "{}/api/v1/{}/user/{}/posts?o={}", BASE_URL, service.as_str(), percent_encode(username), + Self::offset_for(page), ) } @@ -308,25 +307,21 @@ impl CoomerProvider { per_page_limit: usize, options: &ServerOptions, ) -> Result> { - let cache_key = match &target { - Target::Uploader { service, username } => { - format!("uploader::{}::{}::{}", service.as_str(), username, page) - } - _ => format!("main::{}::{}", target_label(&target), page), - }; - - if let Some((time, items)) = cache.get(&cache_key) { - if time.elapsed().unwrap_or_default().as_secs() < CACHE_TTL_SECS { - return Ok(items.clone()); - } - } - let url = match &target { Target::Uploader { service, username } => { Self::build_uploader_url(service, username, page) } _ => Self::build_main_url(&target, page), }; + // Key on the full request URL: the search query must be part of it, + // otherwise every search shares one cached page. + let cache_key = format!("coomer::{url}"); + + if let Some((time, items)) = cache.get(&cache_key) { + if time.elapsed().unwrap_or_default().as_secs() < CACHE_TTL_SECS { + return Ok(items.clone()); + } + } let posts = self.fetch_json(options, "coomer.fetch_items", &url).await?; eprintln!("[coomer] fetched {} posts from {}", posts.len(), url); @@ -389,47 +384,46 @@ impl CoomerProvider { }); let page_url = format!("{}/{}/user/{}/post/{}", BASE_URL, service, user, id); - // Coomer returns the file list in `attachments`; the main image/video - // lives in `file` for the first attachment. The `file.path` is the - // data-path (without the /data/ prefix); we have to prepend it. - let preview_file = post.file.as_ref(); - let preview_url: Option = preview_file + // Coomer returns the main file in `file` and the rest in `attachments`. + // Pick the first *video* across both: a video can sit in an attachment + // while `file` is a cover image. Image-only posts have nothing + // playable (and yt-dlp can't resolve the post page), so they are + // dropped from this video channel. + let is_video_ext = + |ext: &str| matches!(ext.to_ascii_lowercase().as_str(), "mp4" | "webm" | "mov" | "mkv" | "m4v"); + let mut candidates: Vec<&ApiFile> = Vec::new(); + if let Some(file) = post.file.as_ref() { + candidates.push(file); + } + if let Some(attachments) = post.attachments.as_ref() { + candidates.extend(attachments.iter()); + } + let video_file = candidates.into_iter().find(|file| { + file.path.as_deref().is_some_and(|path| !path.is_empty()) + && file + .name + .as_deref() + .and_then(extension_from_name) + .or_else(|| file.path.as_deref().and_then(extension_from_name)) + .is_some_and(|ext| is_video_ext(ext)) + })?; + + let preview_url: Option = post + .file + .as_ref() .and_then(|file| file.path.as_deref()) + .filter(|path| !path.is_empty()) .map(|path| format!("{}/data/{}", BASE_URL, path.trim_start_matches('/'))); - let mut format_url: Option = None; - let mut ext_hint: Option = None; - - if let Some(file) = preview_file { - if let Some(name) = file.name.as_deref() { - if let Some(ext) = extension_from_name(name) { - ext_hint = Some(ext.to_string()); - } - } - // The main file is the playable media. For images it's still - // served as the media URL; for videos it's the actual .mp4. - if let Some(path) = file.path.as_deref() { - format_url = Some(format!("{}/data/{}", BASE_URL, path.trim_start_matches('/'))); - } - } - - if format_url.is_none() { - if let Some(attach) = post.attachments.as_ref().and_then(|a| a.first()) { - if let Some(path) = attach.path.as_deref() { - format_url = Some(format!( - "{}/data/{}", - BASE_URL, - path.trim_start_matches('/') - )); - } - if ext_hint.is_none() { - if let Some(name) = attach.name.as_deref() { - if let Some(ext) = extension_from_name(name) { - ext_hint = Some(ext.to_string()); - } - } - } - } - } + let format_url: Option = video_file + .path + .as_deref() + .map(|path| format!("{}/data/{}", BASE_URL, path.trim_start_matches('/'))); + let ext_hint: Option = video_file + .name + .as_deref() + .and_then(extension_from_name) + .or_else(|| video_file.path.as_deref().and_then(extension_from_name)) + .map(|ext| ext.to_ascii_lowercase()); // Final fallback for the thumbnail: the preview file's data URL, or // the format URL if the file itself is the only image. @@ -461,7 +455,7 @@ impl CoomerProvider { if let Some(ref url) = format_url { let ext = ext_hint.unwrap_or_else(|| "mp4".to_string()); - let is_video = matches!(ext.as_str(), "mp4" | "webm" | "mov" | "mkv"); + let is_video = is_video_ext(&ext); // An image-only post (very common - many creators post photos, // not just videos) has nothing playable; emitting a "format" // pointing at a static jpg/png just gives players a broken @@ -660,11 +654,11 @@ mod tests { fn builds_latest_url() { assert_eq!( CoomerProvider::build_main_url(&Target::Latest, 1), - "https://coomer.st/api/v1/posts?limit=50&page=1" + "https://coomer.st/api/v1/posts?o=0" ); assert_eq!( CoomerProvider::build_main_url(&Target::Latest, 3), - "https://coomer.st/api/v1/posts?limit=50&page=3" + "https://coomer.st/api/v1/posts?o=100" ); } @@ -672,7 +666,7 @@ mod tests { fn builds_popular_url() { assert_eq!( CoomerProvider::build_main_url(&Target::Popular, 1), - "https://coomer.st/api/v1/posts/popular?limit=50&page=1" + "https://coomer.st/api/v1/posts/popular?o=0" ); } @@ -683,22 +677,24 @@ mod tests { }; let url = CoomerProvider::build_main_url(&target, 1); assert!(url.contains("q=milf%20%2B%20solo")); - assert!(url.contains("limit=50")); + assert!(url.ends_with("&o=0")); + let page_three = CoomerProvider::build_main_url(&target, 3); + assert!(page_three.ends_with("&o=100")); } #[test] fn builds_uploader_url_per_service() { assert_eq!( CoomerProvider::build_uploader_url(&Service::Onlyfans, "tabycatxoxo", 1), - "https://coomer.st/api/v1/onlyfans/user/tabycatxoxo/posts?limit=50&page=1" + "https://coomer.st/api/v1/onlyfans/user/tabycatxoxo/posts?o=0" ); assert_eq!( CoomerProvider::build_uploader_url(&Service::Fansly, "ruby", 2), - "https://coomer.st/api/v1/fansly/user/ruby/posts?limit=50&page=2" + "https://coomer.st/api/v1/fansly/user/ruby/posts?o=50" ); assert_eq!( CoomerProvider::build_uploader_url(&Service::Candfans, "model", 1), - "https://coomer.st/api/v1/candfans/user/model/posts?limit=50&page=1" + "https://coomer.st/api/v1/candfans/user/model/posts?o=0" ); } @@ -835,11 +831,25 @@ mod tests { "file": {"name": "p.jpg", "path": "/x/p.jpg"} }"#; let post: ApiPost = serde_json::from_str(json).expect("parse"); + // Image-only posts have nothing playable and are dropped. + assert!(CoomerProvider::build_video_item(post).is_none()); + } + + #[test] + fn build_video_item_uses_video_attachment_when_file_is_image() { + let json = r#"{ + "id": "43", + "user": "ruby", + "service": "fansly", + "title": "clip", + "file": {"name": "cover.jpg", "path": "/c/cover.jpg"}, + "attachments": [{"name": "a.jpg", "path": "/a/a.jpg"}, {"name": "v.MP4", "path": "/v/v.mp4"}] + }"#; + let post: ApiPost = serde_json::from_str(json).expect("parse"); let item = CoomerProvider::build_video_item(post).expect("item"); - assert_eq!(item.title, "Hello there friends"); - assert_eq!(item.thumb, "https://coomer.st/data/x/p.jpg"); - // For an image attachment, no formats are emitted (preview is enough). - assert!(item.formats.is_none()); + let formats = item.formats.expect("video attachment must yield a format"); + assert_eq!(formats[0].url, "https://coomer.st/data/v/v.mp4"); + assert_eq!(item.thumb, "https://coomer.st/data/c/cover.jpg"); } #[test]