thepornbunny: resolve pornstar/studio/tag names; get_probe treats redirect-away as a miss

This commit is contained in:
Simon
2026-10-17 05:53:43 +00:00
parent 2646ca5659
commit e34801f6f8
2 changed files with 43 additions and 2 deletions

View File

@@ -661,6 +661,10 @@ impl Requester {
Ok(response)
}
fn path_of(url: &str) -> Option<String> {
url::Url::parse(url).ok().map(|u| u.path().to_string())
}
/// Existence probe for archive pages (pornstar / studio / tag / category).
///
/// Reads the raw response so a real 404/410 answers immediately with
@@ -678,6 +682,14 @@ impl Requester {
match self.get_raw_with_headers(url, headers.clone()).await {
Ok(response) => {
let status = response.status().as_u16();
// An unknown archive often redirects to the home page or a
// search page instead of answering 404: a redirect that
// lands on a different path is a miss.
let redirected_away = Self::path_of(url)
.zip(Some(response.uri().path().to_string()))
.is_some_and(|(asked, landed)| {
!asked.trim_end_matches('/').eq_ignore_ascii_case(landed.trim_end_matches('/'))
});
let body = response.text().await.unwrap_or_default();
if status == 404 || status == 410 {
return Ok(None);
@@ -688,6 +700,9 @@ impl Requester {
{
return self.get_with_headers(url, headers, None).await.map(Some);
}
if redirected_away {
return Ok(None);
}
if status >= 500 || status == 429 {
last_err = format!("HTTP {status}");
continue;