Add get_probe/probe_first_archive; fullporner and fullpornerorg resolve pornstar names without prefix
This commit is contained in:
@@ -430,40 +430,13 @@ impl HoopornProvider {
|
||||
.await
|
||||
}
|
||||
|
||||
/// Fetches a listing page. A real 404 (unknown archive, page past the end)
|
||||
/// is `None` and answers immediately; a Cloudflare challenge is handed to
|
||||
/// the requester's full chain (FlareSolverr).
|
||||
/// Fetches a listing page; `None` for a real 404 (see `Requester::get_probe`).
|
||||
async fn fetch_html(requester: &mut Requester, url: &str) -> Result<Option<String>> {
|
||||
let headers = vec![("Referer".to_string(), format!("{BASE_URL}/"))];
|
||||
let mut last_err = String::new();
|
||||
for _ in 0..2 {
|
||||
match requester.get_raw_with_headers(url, headers.clone()).await {
|
||||
Ok(response) => {
|
||||
let status = response.status().as_u16();
|
||||
let body = response.text().await.unwrap_or_default();
|
||||
if status == 404 {
|
||||
return Ok(None);
|
||||
}
|
||||
if status == 403
|
||||
|| status == 503
|
||||
|| Requester::looks_like_cloudflare_challenge(&body)
|
||||
{
|
||||
return requester
|
||||
.get_with_headers(url, headers, None)
|
||||
.await
|
||||
.map(Some)
|
||||
.map_err(|e| Error::from(format!("fetch failed for {url}: {e}")));
|
||||
}
|
||||
if status >= 500 || status == 429 {
|
||||
last_err = format!("HTTP {status}");
|
||||
continue;
|
||||
}
|
||||
return Ok(Some(body));
|
||||
}
|
||||
Err(e) => last_err = e.to_string(),
|
||||
}
|
||||
}
|
||||
Err(Error::from(format!("fetch failed for {url}: {last_err}")))
|
||||
requester
|
||||
.get_probe(url, headers)
|
||||
.await
|
||||
.map_err(|e| Error::from(format!("fetch failed for {url}: {e}")))
|
||||
}
|
||||
|
||||
async fn fetch(target: &Target, page: u32, options: &ServerOptions) -> Result<Vec<VideoItem>> {
|
||||
|
||||
Reference in New Issue
Block a user