simon/hottub
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix coomer: offset pagination, per-query cache key, video-only posts

Browse Source 
The API ignores page= (every page 2+ repeated page 1), searches shared a
single cache entry, image-only posts were listed as unplayable videos, and
gzip/br bodies were not decoded because wreq had no decompression features.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MUYdCh2nCVsQYgJAT7gwSi
This commit is contained in:
Simon
2026-10-10 08:04:00 +00:00
parent 5098d142f9
commit a20a4ab642
3 changed files with 89 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Cargo.toml
View File

@@ -20,7 +20,7 @@ ntex-files = "3"
serde = "1.0.228"
serde_json = "1.0.145"
tokio = { version = "1.49", features = ["full"] }
wreq = { version = "6.0.0-rc.31", features = ["cookies", "multipart", "json"] }
wreq = { version = "6.0.0-rc.31", features = ["cookies", "multipart", "json", "gzip", "brotli", "deflate", "zstd"] }
wreq-util = "3.0.0-rc.14"
percent-encoding = "2.3.2"
capitalize = "0.3.4"

2
docs/provider-fix-todo.md
View File

@@ -23,7 +23,7 @@ Status: `[ ]` open, `[~]` in progress, `[x]` fixed and verified, `[-]` not repro
|---|---|---|---|
| [x] | `thaiporntv` | page URL "Unsupported URL", no `formats` | player attribute renamed `data-enc` -> `data-video-sources` (reversed base64 JSON `{quality,url}`); decoder handles both. 3 formats/item, yt-dlp OK, check.py passes |
| [x] | `pornmz` | page URL "Unsupported URL", no `formats` | site changed `itemprop="contentURL"` -> `contentUrl`; selector now case-insensitive. Enrichment concurrency 10 + 25s per-item timeout. Origin is slow (15s+/detail) so first feed 25-50s. check.py passes |
| [ ] | `coomer` | page URL "Unsupported URL"; media host CF-gated here | check whether `formats` are populated for videos |
| [x] | `coomer` | page URL "Unsupported URL"; media host CF-gated here | image-only posts were listed as videos with no playable URL: now dropped, and a video in `attachments` is picked up. Also fixed: `page=` is ignored by the API (every page 2+ was a copy of page 1; now `o=` offset), search results shared one cache key, and compressed API bodies were not decoded (enabled wreq gzip/br/deflate/zstd). n*.coomer.st CDN connect-times-out from this host, and the API 429s this IP after bursts, so playback/"girl" search could not be verified end to end |
| [ ] | `tube8` | page URL 404; `/proxy/tube8/{id}` 502 | |
| [ ] | `hypnotube` | page URLs 404 on all sampled items | URL construction |
| [ ] | `fullpornerorg` | `lulustream` proxy 403 / 502 | |

164
src/providers/coomer.rs
View File

@@ -37,6 +37,8 @@ const REFERER: &str = "https://coomer.st/";
const JSON_ACCEPT: &str = "text/css,*/*;q=0.1";
const CACHE_TTL_SECS: u64 = 60 * 15;
const MAX_PAGE: u16 = 50;
/// The API pages by offset (`o`), in blocks of this many posts. `page=` is ignored.
const API_PAGE_SIZE: u16 = 50;
const DEFAULT_PER_PAGE: usize = 30;
const USER_POSTS_FETCH_CONCURRENCY: usize = 4;
@@ -239,19 +241,18 @@ impl CoomerProvider {
}
}
fn offset_for(page: u16) -> u32 {
(page.max(1).min(MAX_PAGE) as u32 - 1) * API_PAGE_SIZE as u32
}
fn build_main_url(target: &Target, page: u16) -> String {
let page = page.max(1).min(MAX_PAGE);
let offset = Self::offset_for(page);
match target {
Target::Latest => format!("{}/api/v1/posts?limit=50&page={page}", BASE_URL),
Target::Popular => format!("{}/api/v1/posts/popular?limit=50&page={page}", BASE_URL),
// Search is non-paginated: ?page=N is ignored by the server for
// q=... requests, but we still emit a `page=1` for consistency.
Target::Latest => format!("{}/api/v1/posts?o={offset}", BASE_URL),
Target::Popular => format!("{}/api/v1/posts/popular?o={offset}", BASE_URL),
Target::Search { query } => {
let encoded = percent_encode(query);
format!(
"{}/api/v1/posts?q={encoded}&limit=50&page=1",
BASE_URL
)
format!("{}/api/v1/posts?q={encoded}&o={offset}", BASE_URL)
}
// Uploader URL is built separately.
Target::Uploader { .. } => String::new(),
@@ -259,14 +260,12 @@ impl CoomerProvider {
}
fn build_uploader_url(service: &Service, username: &str, page: u16) -> String {
let page = page.max(1).min(MAX_PAGE);
// The user-posts endpoint is also non-paginated, but we send page=1 for
// clarity. Service comes from the enum so the path is stable.
format!(
"{}/api/v1/{}/user/{}/posts?limit=50&page={page}",
"{}/api/v1/{}/user/{}/posts?o={}",
BASE_URL,
service.as_str(),
percent_encode(username),
Self::offset_for(page),
)
}
@@ -308,25 +307,21 @@ impl CoomerProvider {
per_page_limit: usize,
options: &ServerOptions,
) -> Result<Vec<VideoItem>> {
let cache_key = match &target {
Target::Uploader { service, username } => {
format!("uploader::{}::{}::{}", service.as_str(), username, page)
}
_ => format!("main::{}::{}", target_label(&target), page),
};
if let Some((time, items)) = cache.get(&cache_key) {
if time.elapsed().unwrap_or_default().as_secs() < CACHE_TTL_SECS {
return Ok(items.clone());
}
}
let url = match &target {
Target::Uploader { service, username } => {
Self::build_uploader_url(service, username, page)
}
_ => Self::build_main_url(&target, page),
};
// Key on the full request URL: the search query must be part of it,
// otherwise every search shares one cached page.
let cache_key = format!("coomer::{url}");
if let Some((time, items)) = cache.get(&cache_key) {
if time.elapsed().unwrap_or_default().as_secs() < CACHE_TTL_SECS {
return Ok(items.clone());
}
}
let posts = self.fetch_json(options, "coomer.fetch_items", &url).await?;
eprintln!("[coomer] fetched {} posts from {}", posts.len(), url);
@@ -389,47 +384,46 @@ impl CoomerProvider {
});
let page_url = format!("{}/{}/user/{}/post/{}", BASE_URL, service, user, id);
// Coomer returns the file list in `attachments`; the main image/video
// lives in `file` for the first attachment. The `file.path` is the
// data-path (without the /data/ prefix); we have to prepend it.
let preview_file = post.file.as_ref();
let preview_url: Option<String> = preview_file
// Coomer returns the main file in `file` and the rest in `attachments`.
// Pick the first *video* across both: a video can sit in an attachment
// while `file` is a cover image. Image-only posts have nothing
// playable (and yt-dlp can't resolve the post page), so they are
// dropped from this video channel.
let is_video_ext =
|ext: &str| matches!(ext.to_ascii_lowercase().as_str(), "mp4" | "webm" | "mov" | "mkv" | "m4v");
let mut candidates: Vec<&ApiFile> = Vec::new();
if let Some(file) = post.file.as_ref() {
candidates.push(file);
}
if let Some(attachments) = post.attachments.as_ref() {
candidates.extend(attachments.iter());
}
let video_file = candidates.into_iter().find(|file| {
file.path.as_deref().is_some_and(|path| !path.is_empty())
&& file
.name
.as_deref()
.and_then(extension_from_name)
.or_else(|| file.path.as_deref().and_then(extension_from_name))
.is_some_and(|ext| is_video_ext(ext))
})?;
let preview_url: Option<String> = post
.file
.as_ref()
.and_then(|file| file.path.as_deref())
.filter(|path| !path.is_empty())
.map(|path| format!("{}/data/{}", BASE_URL, path.trim_start_matches('/')));
let mut format_url: Option<String> = None;
let mut ext_hint: Option<String> = None;
if let Some(file) = preview_file {
if let Some(name) = file.name.as_deref() {
if let Some(ext) = extension_from_name(name) {
ext_hint = Some(ext.to_string());
}
}
// The main file is the playable media. For images it's still
// served as the media URL; for videos it's the actual .mp4.
if let Some(path) = file.path.as_deref() {
format_url = Some(format!("{}/data/{}", BASE_URL, path.trim_start_matches('/')));
}
}
if format_url.is_none() {
if let Some(attach) = post.attachments.as_ref().and_then(|a| a.first()) {
if let Some(path) = attach.path.as_deref() {
format_url = Some(format!(
"{}/data/{}",
BASE_URL,
path.trim_start_matches('/')
));
}
if ext_hint.is_none() {
if let Some(name) = attach.name.as_deref() {
if let Some(ext) = extension_from_name(name) {
ext_hint = Some(ext.to_string());
}
}
}
}
}
let format_url: Option<String> = video_file
.path
.as_deref()
.map(|path| format!("{}/data/{}", BASE_URL, path.trim_start_matches('/')));
let ext_hint: Option<String> = video_file
.name
.as_deref()
.and_then(extension_from_name)
.or_else(|| video_file.path.as_deref().and_then(extension_from_name))
.map(|ext| ext.to_ascii_lowercase());
// Final fallback for the thumbnail: the preview file's data URL, or
// the format URL if the file itself is the only image.
@@ -461,7 +455,7 @@ impl CoomerProvider {
if let Some(ref url) = format_url {
let ext = ext_hint.unwrap_or_else(|| "mp4".to_string());
let is_video = matches!(ext.as_str(), "mp4" | "webm" | "mov" | "mkv");
let is_video = is_video_ext(&ext);
// An image-only post (very common - many creators post photos,
// not just videos) has nothing playable; emitting a "format"
// pointing at a static jpg/png just gives players a broken
@@ -660,11 +654,11 @@ mod tests {
fn builds_latest_url() {
assert_eq!(
CoomerProvider::build_main_url(&Target::Latest, 1),
"https://coomer.st/api/v1/posts?limit=50&page=1"
"https://coomer.st/api/v1/posts?o=0"
);
assert_eq!(
CoomerProvider::build_main_url(&Target::Latest, 3),
"https://coomer.st/api/v1/posts?limit=50&page=3"
"https://coomer.st/api/v1/posts?o=100"
);
}
@@ -672,7 +666,7 @@ mod tests {
fn builds_popular_url() {
assert_eq!(
CoomerProvider::build_main_url(&Target::Popular, 1),
"https://coomer.st/api/v1/posts/popular?limit=50&page=1"
"https://coomer.st/api/v1/posts/popular?o=0"
);
}
@@ -683,22 +677,24 @@ mod tests {
};
let url = CoomerProvider::build_main_url(&target, 1);
assert!(url.contains("q=milf%20%2B%20solo"));
assert!(url.contains("limit=50"));
assert!(url.ends_with("&o=0"));
let page_three = CoomerProvider::build_main_url(&target, 3);
assert!(page_three.ends_with("&o=100"));
}
#[test]
fn builds_uploader_url_per_service() {
assert_eq!(
CoomerProvider::build_uploader_url(&Service::Onlyfans, "tabycatxoxo", 1),
"https://coomer.st/api/v1/onlyfans/user/tabycatxoxo/posts?limit=50&page=1"
"https://coomer.st/api/v1/onlyfans/user/tabycatxoxo/posts?o=0"
);
assert_eq!(
CoomerProvider::build_uploader_url(&Service::Fansly, "ruby", 2),
"https://coomer.st/api/v1/fansly/user/ruby/posts?limit=50&page=2"
"https://coomer.st/api/v1/fansly/user/ruby/posts?o=50"
);
assert_eq!(
CoomerProvider::build_uploader_url(&Service::Candfans, "model", 1),
"https://coomer.st/api/v1/candfans/user/model/posts?limit=50&page=1"
"https://coomer.st/api/v1/candfans/user/model/posts?o=0"
);
}
@@ -835,11 +831,25 @@ mod tests {
"file": {"name": "p.jpg", "path": "/x/p.jpg"}
}"#;
let post: ApiPost = serde_json::from_str(json).expect("parse");
// Image-only posts have nothing playable and are dropped.
assert!(CoomerProvider::build_video_item(post).is_none());
}
#[test]
fn build_video_item_uses_video_attachment_when_file_is_image() {
let json = r#"{
"id": "43",
"user": "ruby",
"service": "fansly",
"title": "clip",
"file": {"name": "cover.jpg", "path": "/c/cover.jpg"},
"attachments": [{"name": "a.jpg", "path": "/a/a.jpg"}, {"name": "v.MP4", "path": "/v/v.mp4"}]
}"#;
let post: ApiPost = serde_json::from_str(json).expect("parse");
let item = CoomerProvider::build_video_item(post).expect("item");
assert_eq!(item.title, "Hello there friends");
assert_eq!(item.thumb, "https://coomer.st/data/x/p.jpg");
// For an image attachment, no formats are emitted (preview is enough).
assert!(item.formats.is_none());
let formats = item.formats.expect("video attachment must yield a format");
assert_eq!(formats[0].url, "https://coomer.st/data/v/v.mp4");
assert_eq!(item.thumb, "https://coomer.st/data/c/cover.jpg");
}
#[test]