Fix xxxtik empty results; replay solved identity on raw requests
Raw requester calls did not adopt the remembered FlareSolverr user agent, so a RedGifs token minted under it was rejected on every gif lookup. Tag/creator pages that mix non-RedGifs sources are refilled from later cursor batches instead of coming back empty. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01MUYdCh2nCVsQYgJAT7gwSi
This commit is contained in:
@@ -30,6 +30,7 @@ const RESOLVE_CONCURRENCY: usize = 8;
|
||||
// xxxtik's listing endpoints are cursor-based (the numeric `id` of the last item
|
||||
// returned), not page-number based, so reaching page N requires walking N
|
||||
// sequential requests from the start of the feed.
|
||||
const MAX_REFILL_FETCHES: usize = 4;
|
||||
const MAX_PAGE_WALK: u16 = 25;
|
||||
|
||||
// xxxtik exposes ~60k tags through /tag/options, far too many to background-load
|
||||
@@ -423,18 +424,43 @@ impl XxxtikProvider {
|
||||
let mut items = Vec::new();
|
||||
|
||||
for _ in 0..page {
|
||||
let batch = self.fetch_one_page(path, cursor, per_page, options).await?;
|
||||
if batch.is_empty() {
|
||||
// Tag and creator pages mix in posts whose source is not RedGifs
|
||||
// (erome links, empty sources); those cannot be resolved here and
|
||||
// used to leave whole pages empty ("girl" returned nothing). Keep
|
||||
// pulling raw batches, bounded, until a page has enough
|
||||
// resolvable posts.
|
||||
let mut collected: Vec<Post> = Vec::new();
|
||||
let mut exhausted = false;
|
||||
for _ in 0..MAX_REFILL_FETCHES {
|
||||
let batch = self.fetch_one_page(path, cursor, per_page, options).await?;
|
||||
if batch.is_empty() {
|
||||
exhausted = true;
|
||||
break;
|
||||
}
|
||||
cursor = batch.last().map(|post| post.id).unwrap_or(cursor);
|
||||
let short_batch = batch.len() < per_page;
|
||||
collected.extend(batch.into_iter().filter(Self::is_redgifs_post));
|
||||
if collected.len() >= per_page || short_batch {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if exhausted && collected.is_empty() {
|
||||
items = Vec::new();
|
||||
break;
|
||||
}
|
||||
cursor = batch.last().map(|post| post.id).unwrap_or(cursor);
|
||||
items = batch;
|
||||
items = collected;
|
||||
}
|
||||
|
||||
Ok(items)
|
||||
}
|
||||
|
||||
fn is_redgifs_post(post: &Post) -> bool {
|
||||
post.source
|
||||
.as_deref()
|
||||
.map(|source| source.contains("redgifs.com/"))
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
async fn fetch_target_posts(
|
||||
&self,
|
||||
target: &Target,
|
||||
@@ -791,6 +817,25 @@ impl Provider for XxxtikProvider {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[test]
|
||||
fn only_redgifs_sources_are_resolvable() {
|
||||
let mk = |source: Option<&str>| super::Post {
|
||||
id: 1,
|
||||
uuid: "u".to_string(),
|
||||
description: None,
|
||||
source: source.map(str::to_string),
|
||||
status: None,
|
||||
created_at: String::new(),
|
||||
author: Default::default(),
|
||||
tags: vec![],
|
||||
};
|
||||
assert!(XxxtikProvider::is_redgifs_post(&mk(Some("https://www.redgifs.com/watch/abc"))));
|
||||
assert!(XxxtikProvider::is_redgifs_post(&mk(Some("https://redgifs.com/watch/abc"))));
|
||||
assert!(!XxxtikProvider::is_redgifs_post(&mk(Some("https://www.erome.com/a/CE5z46ne#39"))));
|
||||
assert!(!XxxtikProvider::is_redgifs_post(&mk(Some(""))));
|
||||
assert!(!XxxtikProvider::is_redgifs_post(&mk(None)));
|
||||
}
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
|
||||
Reference in New Issue
Block a user