From 2d35f29f44489a52fd760d74d7461375e4eecdb7 Mon Sep 17 00:00:00 2001 From: Simon Date: Sat, 10 Oct 2026 09:33:36 +0000 Subject: [PATCH] Fix xxxtik empty results; replay solved identity on raw requests Raw requester calls did not adopt the remembered FlareSolverr user agent, so a RedGifs token minted under it was rejected on every gif lookup. Tag/creator pages that mix non-RedGifs sources are refilled from later cursor batches instead of coming back empty. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01MUYdCh2nCVsQYgJAT7gwSi --- docs/provider-fix-todo.md | 2 +- src/providers/xxxtik.rs | 53 ++++++++++++++++++++++++++++++++++++--- src/util/requester.rs | 4 +++ 3 files changed, 54 insertions(+), 5 deletions(-) diff --git a/docs/provider-fix-todo.md b/docs/provider-fix-todo.md index 8839c6b..ad0ffc1 100644 --- a/docs/provider-fix-todo.md +++ b/docs/provider-fix-todo.md @@ -47,7 +47,7 @@ Status: `[ ]` open, `[~]` in progress, `[x]` fixed and verified, `[-]` not repro | [x] | `chaturbate` | "girl" and "milf" both 0 | format arguments swapped in the search URL (`girl/api/...?keywords=https://chaturbate.com`, a relative-URL builder error); fixed + unit test. Also stale-cache branch never refreshed. 90 results per query, yt-dlp resolves the live stream | | [x] | `cumlouder` | unknown tag takes 146s before returning 0 | a real 404 went through the requester's Jina/FlareSolverr fallback chain; now the raw response is read, 404 returns empty immediately (0s), 5xx/429 still retried. check.py passes | | [x] | `vjav` | "girl" 0, "milf" 10 | not term-specific: "girl" exactly matches a member username with no uploads, so the implicit uploader match hijacked the keyword search. Empty uploader/tag matches now fall back to keyword search (site API returns 34k results for "girl"). check.py passes | -| [ ] | `xxxtik` | "girl" 0 (not re-run serially) | | +| [x] | `xxxtik` | "girl" 0 (not re-run serially) | three things. (1) shared requester: raw requests did not replay the remembered FlareSolverr identity, so a RedGifs token minted under the solved UA was rejected (401) on every gif lookup after one transient connect error (empty feed, 71s); `get_raw_with_headers_timeout` now adopts it. (2) tag/creator pages mixing non-RedGifs posts (erome/onlyfans links) emptied whole pages; now pulls up to 4 raw batches per page. (3) "girl" itself has no RedGifs posts in its first 600 entries, so empty is correct there; "milf" returns 10. Feed/p2 10 items, check.py passes. First request can still take ~50s when api.redgifs.com has a connect error (FlareSolverr detour) | | [x] | `hypnotube` | "milf" 0, "girl" 20 | same root cause as above (second search on a session returned nothing), not term-specific | ## 5. Slow channels diff --git a/src/providers/xxxtik.rs b/src/providers/xxxtik.rs index c85dc95..f97219c 100644 --- a/src/providers/xxxtik.rs +++ b/src/providers/xxxtik.rs @@ -30,6 +30,7 @@ const RESOLVE_CONCURRENCY: usize = 8; // xxxtik's listing endpoints are cursor-based (the numeric `id` of the last item // returned), not page-number based, so reaching page N requires walking N // sequential requests from the start of the feed. +const MAX_REFILL_FETCHES: usize = 4; const MAX_PAGE_WALK: u16 = 25; // xxxtik exposes ~60k tags through /tag/options, far too many to background-load @@ -423,18 +424,43 @@ impl XxxtikProvider { let mut items = Vec::new(); for _ in 0..page { - let batch = self.fetch_one_page(path, cursor, per_page, options).await?; - if batch.is_empty() { + // Tag and creator pages mix in posts whose source is not RedGifs + // (erome links, empty sources); those cannot be resolved here and + // used to leave whole pages empty ("girl" returned nothing). Keep + // pulling raw batches, bounded, until a page has enough + // resolvable posts. + let mut collected: Vec = Vec::new(); + let mut exhausted = false; + for _ in 0..MAX_REFILL_FETCHES { + let batch = self.fetch_one_page(path, cursor, per_page, options).await?; + if batch.is_empty() { + exhausted = true; + break; + } + cursor = batch.last().map(|post| post.id).unwrap_or(cursor); + let short_batch = batch.len() < per_page; + collected.extend(batch.into_iter().filter(Self::is_redgifs_post)); + if collected.len() >= per_page || short_batch { + break; + } + } + if exhausted && collected.is_empty() { items = Vec::new(); break; } - cursor = batch.last().map(|post| post.id).unwrap_or(cursor); - items = batch; + items = collected; } Ok(items) } + fn is_redgifs_post(post: &Post) -> bool { + post.source + .as_deref() + .map(|source| source.contains("redgifs.com/")) + .unwrap_or(false) + } + async fn fetch_target_posts( &self, target: &Target, @@ -791,6 +817,25 @@ impl Provider for XxxtikProvider { #[cfg(test)] mod tests { + #[test] + fn only_redgifs_sources_are_resolvable() { + let mk = |source: Option<&str>| super::Post { + id: 1, + uuid: "u".to_string(), + description: None, + source: source.map(str::to_string), + status: None, + created_at: String::new(), + author: Default::default(), + tags: vec![], + }; + assert!(XxxtikProvider::is_redgifs_post(&mk(Some("https://www.redgifs.com/watch/abc")))); + assert!(XxxtikProvider::is_redgifs_post(&mk(Some("https://redgifs.com/watch/abc")))); + assert!(!XxxtikProvider::is_redgifs_post(&mk(Some("https://www.erome.com/a/CE5z46ne#39")))); + assert!(!XxxtikProvider::is_redgifs_post(&mk(Some("")))); + assert!(!XxxtikProvider::is_redgifs_post(&mk(None))); + } + use super::*; #[test] diff --git a/src/util/requester.rs b/src/util/requester.rs index 500b660..256a0a4 100644 --- a/src/util/requester.rs +++ b/src/util/requester.rs @@ -496,6 +496,10 @@ impl Requester { timeout: Option, http_version: Option, ) -> Result { + // Replay the solved FlareSolverr identity here too: a token minted + // through `get_with_headers` under that user agent is rejected (401) + // when the follow-up raw request goes out with the default one. + self.adopt_remembered_identity(url); let cookie_preview = self.debug_cookie_preview_from_owned_headers(url, &headers); #[cfg(not(feature = "debug"))] let _ = &cookie_preview;