diff --git a/docs/provider-fix-todo.md b/docs/provider-fix-todo.md index 88b483b..1575613 100644 --- a/docs/provider-fix-todo.md +++ b/docs/provider-fix-todo.md @@ -24,7 +24,7 @@ Status: `[ ]` open, `[~]` in progress, `[x]` fixed and verified, `[-]` not repro | [x] | `thaiporntv` | page URL "Unsupported URL", no `formats` | player attribute renamed `data-enc` -> `data-video-sources` (reversed base64 JSON `{quality,url}`); decoder handles both. 3 formats/item, yt-dlp OK, check.py passes | | [x] | `pornmz` | page URL "Unsupported URL", no `formats` | site changed `itemprop="contentURL"` -> `contentUrl`; selector now case-insensitive. Enrichment concurrency 10 + 25s per-item timeout. Origin is slow (15s+/detail) so first feed 25-50s. check.py passes | | [x] | `coomer` | page URL "Unsupported URL"; media host CF-gated here | image-only posts were listed as videos with no playable URL: now dropped, and a video in `attachments` is picked up. Also fixed: `page=` is ignored by the API (every page 2+ was a copy of page 1; now `o=` offset), search results shared one cache key, and compressed API bodies were not decoded (enabled wreq gzip/br/deflate/zstd). n*.coomer.st CDN connect-times-out from this host, and the API 429s this IP after bursts, so playback/"girl" search could not be verified end to end | -| [ ] | `tube8` | page URL 404; `/proxy/tube8/{id}` 502 | | +| [x] | `tube8` | page URL 404; `/proxy/tube8/{id}` 502 | sponsored `channel` cards carry a stale `data-video-id` (404) while the card link is the live page; id/URL now come from the link. Signed `pix-cdn77` thumbs 403 without a Referer: new host-restricted `/proxy/aylo-thumb` (also fixes ph-shorties). check.py passes | | [ ] | `hypnotube` | page URLs 404 on all sampled items | URL construction | | [ ] | `fullpornerorg` | `lulustream` proxy 403 / 502 | | | [ ] | `ripnsfw` | `doodstream` format 502 | | @@ -36,7 +36,7 @@ Status: `[ ]` open, `[~]` in progress, `[x]` fixed and verified, `[-]` not repro | | Channel | Symptom | Notes | |---|---|---|---| | [ ] | `xxdbx` | malformed thumbs `https:/7004….jpg` | built as `"https:" + src` | -| [ ] | `ph-shorties` | thumbs return 403 | | +| [x] | `ph-shorties` | thumbs return 403 | same Referer-gated `pix-cdn77.phncdn.com` thumbs; routed through `aylo-thumb` proxy. check.py passes | | [ ] | `noodlemagazine` | thumbs unreachable (-1 / 400) in check.py | thumb proxy | ## 4. Search @@ -64,6 +64,11 @@ Status: `[ ]` open, `[~]` in progress, `[x]` fixed and verified, `[-]` not repro |---|---|---| | [ ] | `check.py` "url points to hottub server" | contradicts the hottub skill (proxy-needing videos use the proxy as `item.url`). Affects spankbang, hqporner, pornhd3x, pimpbunny, sxyprn, noodlemagazine. Decide: exempt proxy URLs that resolve, rather than hard-fail. | +## Found while fixing (pre-existing, unrelated) + +- `src/proxy.rs`: the `pornhub-thumb` resource was registered as `/proxy/pornhub-thumb` inside the `/proxy` scope (unreachable); fixed alongside `aylo-thumb`. No provider uses it today. +- Failing unit tests, untouched files: `proxies::hanimecdn::tests::prefixes_relative_paths_with_hanime_cdn_host`, `proxies::vidara::tests::normalizes_detail_request_with_path_only`. + ## Environmental / needs the production IP (not worked here) `porndish`, `supjav`, `jable`, `fapello`, `missav`, `wowxxx`, `hentaihaven` return Cloudflare 403 to diff --git a/src/providers/coomer.rs b/src/providers/coomer.rs index d84b39d..efa82b0 100644 --- a/src/providers/coomer.rs +++ b/src/providers/coomer.rs @@ -489,15 +489,6 @@ impl CoomerProvider { } } -fn target_label(target: &Target) -> &'static str { - match target { - Target::Latest => "latest", - Target::Popular => "popular", - Target::Search { .. } => "search", - Target::Uploader { .. } => "uploader", - } -} - fn percent_encode(value: &str) -> String { let mut out = String::with_capacity(value.len()); for byte in value.as_bytes() { diff --git a/src/providers/mod.rs b/src/providers/mod.rs index 0ae20ac..4e1926a 100644 --- a/src/providers/mod.rs +++ b/src/providers/mod.rs @@ -644,6 +644,25 @@ pub fn requester_or_default( } #[allow(dead_code)] +/// Routes signed Aylo CDN thumbnails (`pix-cdn77.t8cdn.com`, +/// `pix-cdn77.phncdn.com`) through the local `aylo-thumb` proxy, because the +/// CDN answers 403 unless a matching Referer is sent. Other URLs are returned +/// unchanged. +pub fn proxy_aylo_thumb(options: &ServerOptions, thumb: &str) -> String { + let needs_proxy = thumb + .strip_prefix("https://") + .or_else(|| thumb.strip_prefix("http://")) + .and_then(|rest| rest.split('/').next()) + .is_some_and(|host| { + host.starts_with("pix-cdn77.") + && (host.ends_with(".t8cdn.com") || host.ends_with(".phncdn.com")) + }); + if !needs_proxy { + return thumb.to_string(); + } + build_proxy_url(options, "aylo-thumb", &strip_url_scheme(thumb)) +} + pub fn strip_url_scheme(url: &str) -> String { url.strip_prefix("https://") .or_else(|| url.strip_prefix("http://")) diff --git a/src/providers/pornhub_shorties.rs b/src/providers/pornhub_shorties.rs index 9764119..5b2ec14 100644 --- a/src/providers/pornhub_shorties.rs +++ b/src/providers/pornhub_shorties.rs @@ -1,6 +1,6 @@ use crate::DbPool; use crate::api::ClientVersion; -use crate::providers::{Provider, report_provider_error, requester_or_default}; +use crate::providers::{Provider, proxy_aylo_thumb, report_provider_error, requester_or_default}; use crate::status::*; use crate::util::cache::VideoCache; use crate::videos::{ServerOptions, VideoItem}; @@ -408,6 +408,11 @@ impl PornhubShortiesProvider { let items: Vec = shorties .into_iter() .filter_map(Self::item_to_video) + .map(|mut video| { + // The signed pix-cdn77 thumbnails 403 without a Referer. + video.thumb = proxy_aylo_thumb(&options, &video.thumb); + video + }) .collect(); if items.is_empty() { diff --git a/src/providers/tube8.rs b/src/providers/tube8.rs index fb03acf..909ee89 100644 --- a/src/providers/tube8.rs +++ b/src/providers/tube8.rs @@ -1,6 +1,8 @@ use crate::DbPool; use crate::api::ClientVersion; -use crate::providers::{Provider, build_proxy_url, report_provider_error, requester_or_default}; +use crate::providers::{ + Provider, build_proxy_url, proxy_aylo_thumb, report_provider_error, requester_or_default, +}; use crate::status::*; use crate::util::cache::VideoCache; use crate::util::parse_abbreviated_number; @@ -257,11 +259,24 @@ impl Tube8Provider { let mut items = Vec::new(); for card in document.select(&card_sel) { - let id = card - .value() - .attr("data-video-id") - .map(|v| v.to_string()) - .filter(|v| !v.is_empty()) + // The card's own link is authoritative: sponsored `channel` cards + // carry a `data-video-id` that 404s while the link target + // (`/porn-video//`) is the live page. + let link_id = link_sel + .as_ref() + .and_then(|s| card.select(s).next()) + .and_then(|v| v.value().attr("href")) + .and_then(|href| href.split("/porn-video/").nth(1)) + .and_then(|rest| rest.split('/').next()) + .filter(|v| !v.is_empty() && v.chars().all(|c| c.is_ascii_digit())) + .map(|v| v.to_string()); + let id = link_id + .or_else(|| { + card.value() + .attr("data-video-id") + .map(|v| v.to_string()) + .filter(|v| !v.is_empty()) + }) .unwrap_or_default(); if id.is_empty() { continue; @@ -296,6 +311,7 @@ impl Tube8Provider { .or_else(|| v.value().attr("src")) }) .map(|v| self.normalize_url(v)) + .map(|v| proxy_aylo_thumb(options, &v)) .unwrap_or_default(); // Preview flipbook clip from the thumbnail link's data-mediabook @@ -536,6 +552,27 @@ mod tests { ); } + #[test] + fn uses_link_target_id_when_card_video_id_is_stale() { + let p = provider(); + let html = r#" + "#; + let opts = ServerOptions { + client_version: None, featured: None, category: None, sites: None, filter: None, + language: None, public_url_base: None, requester: None, network: None, stars: None, + categories: None, duration: None, sort: None, sexuality: None, + }; + let items = p.parse_items(html, &opts); + assert_eq!(items.len(), 1); + assert_eq!(items[0].id, "67452091"); + assert_eq!(items[0].url, "https://www.tube8.com/porn-video/67452091/"); + } + #[test] fn parses_video_cards() { let p = provider(); diff --git a/src/proxies/aylothumb.rs b/src/proxies/aylothumb.rs new file mode 100644 index 0000000..37eb0c5 --- /dev/null +++ b/src/proxies/aylothumb.rs @@ -0,0 +1,126 @@ +use ntex::http::header::{CONTENT_LENGTH, CONTENT_TYPE}; +use ntex::{ + http::Response, + web::{self, HttpRequest, error}, +}; +use url::Url; + +use crate::util::requester::Requester; + +/// Referer the image CDN expects, or `None` when the host is not allowed. +/// +/// The signed `pix-cdn77.*` thumbnail hosts used by tube8 and the pornhub +/// shorties feed answer 403 without a matching Referer, so apps that load +/// the URL directly get broken images. +fn referer_for_host(host: &str) -> Option<&'static str> { + let host = host.to_ascii_lowercase(); + if host == "t8cdn.com" || host.ends_with(".t8cdn.com") { + Some("https://www.tube8.com/") + } else if host == "phncdn.com" || host.ends_with(".phncdn.com") { + Some("https://www.pornhub.com/") + } else { + None + } +} + +fn is_image_path(path: &str) -> bool { + let path = path.to_ascii_lowercase(); + // The signed thumbnail paths end in processing options such as + // `/plain/rs:fit:320:180/vts:3`, not a file extension, so accept those too. + [".jpg", ".jpeg", ".png", ".webp", ".avif", ".gif"] + .iter() + .any(|ext| path.ends_with(ext)) + || path.contains("/plain/") +} + +fn upstream_url(req: &HttpRequest) -> Option<(String, &'static str)> { + let endpoint = req.match_info().query("endpoint").trim_start_matches('/'); + let mut image_url = if endpoint.starts_with("http://") || endpoint.starts_with("https://") { + endpoint.to_string() + } else { + format!("https://{endpoint}") + }; + + // The signed URL's own query (`hash`, `validto`) arrives as the request query. + let query = req.query_string(); + if !query.is_empty() && !image_url.contains('?') { + image_url.push('?'); + image_url.push_str(query); + } + + let parsed = Url::parse(&image_url).ok()?; + if parsed.scheme() != "https" { + return None; + } + let referer = referer_for_host(parsed.host_str()?)?; + if !is_image_path(parsed.path()) { + return None; + } + Some((image_url, referer)) +} + +pub async fn get_image( + req: HttpRequest, + requester: web::types::State, +) -> Result { + let Some((image_url, referer)) = upstream_url(&req) else { + return Ok(web::HttpResponse::BadRequest().finish()); + }; + + let upstream = match requester + .get_ref() + .clone() + .get_raw_with_headers( + image_url.as_str(), + vec![("Referer".to_string(), referer.to_string())], + ) + .await + { + Ok(response) => response, + Err(_) => return Ok(web::HttpResponse::NotFound().finish()), + }; + + let status = upstream.status(); + let headers = upstream.headers().clone(); + let bytes = upstream.bytes().await.map_err(error::ErrorBadGateway)?; + + let mut resp = Response::build(status); + if let Some(ct) = headers.get(CONTENT_TYPE) { + if let Ok(ct_str) = ct.to_str() { + resp.set_header(CONTENT_TYPE, ct_str); + } + } + if let Some(cl) = headers.get(CONTENT_LENGTH) { + if let Ok(cl_str) = cl.to_str() { + resp.set_header(CONTENT_LENGTH, cl_str); + } + } + + Ok(resp.body(bytes.to_vec())) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn maps_hosts_to_referers_and_rejects_others() { + assert_eq!( + referer_for_host("pix-cdn77.t8cdn.com"), + Some("https://www.tube8.com/") + ); + assert_eq!( + referer_for_host("pix-cdn77.phncdn.com"), + Some("https://www.pornhub.com/") + ); + assert_eq!(referer_for_host("evil.example.com"), None); + assert_eq!(referer_for_host("t8cdn.com.evil.example"), None); + } + + #[test] + fn accepts_signed_processing_paths_and_extensions() { + assert!(is_image_path("/c6531/videos/1/original.mov/plain/rs:fit:320:180/vts:3")); + assert!(is_image_path("/videos/thumb.JPG")); + assert!(!is_image_path("/videos/clip.mp4")); + } +} diff --git a/src/proxies/mod.rs b/src/proxies/mod.rs index ee6e487..8f819ac 100644 --- a/src/proxies/mod.rs +++ b/src/proxies/mod.rs @@ -18,6 +18,7 @@ use crate::proxies::lulustream::LulustreamProxy; use crate::proxies::thaiporntv::ThaipornTvProxy; pub mod allpornstream; +pub mod aylothumb; pub mod melonstube; pub mod animeidhentai; pub mod archivebate; diff --git a/src/proxy.rs b/src/proxy.rs index e46ddfc..7b3b11b 100644 --- a/src/proxy.rs +++ b/src/proxy.rs @@ -149,7 +149,12 @@ pub fn config(cfg: &mut web::ServiceConfig) { .route(web::get().to(crate::proxies::hotbunnythumb::get_image)), ); cfg.service( - web::resource("/proxy/pornhub-thumb/{endpoint}*") + web::resource("/aylo-thumb/{endpoint}*") + .route(web::post().to(crate::proxies::aylothumb::get_image)) + .route(web::get().to(crate::proxies::aylothumb::get_image)), + ); + cfg.service( + web::resource("/pornhub-thumb/{endpoint}*") .route(web::post().to(crate::proxies::pornhubthumb::get_image)) .route(web::get().to(crate::proxies::pornhubthumb::get_image)), );