diff --git a/src/providers/eporner.rs b/src/providers/eporner.rs index b7d4404..9bf2130 100644 --- a/src/providers/eporner.rs +++ b/src/providers/eporner.rs @@ -8,12 +8,14 @@ use crate::util::cache::VideoCache; use crate::util::parse_abbreviated_number; use crate::util::requester::Requester; use crate::util::time::parse_time_to_seconds; -use crate::videos::{ServerOptions, VideoItem}; +use crate::videos::{ServerOptions, VideoFormat, VideoItem}; use async_trait::async_trait; use error_chain::error_chain; +use futures::stream::{self, StreamExt}; use htmlentity::entity::{ICodedDataTrait, decode}; +use regex::Regex; use scraper::{ElementRef, Html, Selector}; -use std::sync::{Arc, RwLock}; +use std::sync::{Arc, OnceLock, RwLock}; use std::{collections::HashMap, thread, vec}; use wreq::Version; @@ -29,6 +31,8 @@ const FIREFOX_UA: &str = "Mozilla/5.0 (X11; Linux x86_64; rv:147.0) Gecko/20100101 Firefox/147.0"; const HTML_ACCEPT: &str = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8"; +// Bounded concurrency for the per-item detail-page -> xhr format enrichment. +const ENRICH_CONCURRENCY: usize = 6; error_chain! { foreign_links { @@ -126,6 +130,23 @@ pub struct EpornerProvider { pornstar_map: Arc>>, } +#[derive(Debug, Clone)] +struct EpornerCard { + id: String, + /// Alphanumeric id used in the page URL (`/video-{alnum_id}/...`) and by + /// the `/xhr/video/{alnum_id}` format API — distinct from `id`, which is + /// the numeric `data-id` used for the item's channel-qualified identity. + alnum_id: String, + title: String, + page_url: String, + thumb: String, + duration: u32, + rating: Option, + views: Option, + uploader: Option, + uploader_url: Option, +} + impl EpornerProvider { pub fn new() -> Self { let provider = Self { @@ -330,7 +351,19 @@ impl EpornerProvider { digits.parse::().ok().map(|v| v / 100.0) } - fn parse_list_page(html: &str) -> Result> { + fn alnum_id_regex() -> &'static Regex { + static RE: OnceLock = OnceLock::new(); + RE.get_or_init(|| Regex::new(r"/video-([A-Za-z0-9]+)").expect("valid regex")) + } + + fn alnum_id_from_href(href: &str) -> Option { + Self::alnum_id_regex() + .captures(href) + .and_then(|c| c.get(1)) + .map(|m| m.as_str().to_string()) + } + + fn parse_list_cards(html: &str) -> Result> { let document = Html::parse_document(html); let card_sel = Self::selector("div.mb[data-id]")?; let img_sel = Self::selector("div.mbimg a img[src]")?; @@ -340,7 +373,7 @@ impl EpornerProvider { let views_sel = Self::selector("span.mbvie")?; let uploader_sel = Self::selector("span.mb-uploader a[href]")?; - let mut items = Vec::new(); + let mut cards = Vec::new(); for card in document.select(&card_sel) { let id = match card.value().attr("data-id") { @@ -357,6 +390,9 @@ impl EpornerProvider { if page_url.is_empty() { continue; } + let Some(alnum_id) = Self::alnum_id_from_href(href) else { + continue; + }; let title = link .value() @@ -392,44 +428,165 @@ impl EpornerProvider { .and_then(|el| Self::parse_views(&Self::text_of(&el))); let uploader_el = card.select(&uploader_sel).next(); - let uploader_name = uploader_el.as_ref().map(|el| Self::text_of(el)); + let uploader_name = uploader_el + .as_ref() + .map(|el| Self::text_of(el)) + .filter(|n| !n.is_empty()); let uploader_url = uploader_el - .and_then(|el| el.value().attr("href").map(Self::normalize_url)); + .and_then(|el| el.value().attr("href").map(Self::normalize_url)) + .filter(|u| !u.is_empty()); - let mut item = VideoItem::new( + cards.push(EpornerCard { id, - title.trim().to_string(), + alnum_id, + title: title.trim().to_string(), page_url, - CHANNEL_ID.to_string(), thumb, duration, - ); - if let Some(r) = rating { - item.rating = Some(r); - } - if let Some(v) = views { - item.views = Some(v); - } - if let Some(name) = uploader_name.filter(|n| !n.is_empty()) { - item.uploader = Some(name); - } - if let Some(url) = uploader_url.filter(|u| !u.is_empty()) { - let uploader_id = url - .trim_end_matches('/') - .rsplit('/') - .next() - .unwrap_or_default() - .to_string(); - if !uploader_id.is_empty() { - item.uploaderId = Some(format!("{CHANNEL_ID}:{uploader_id}")); - } - item.uploaderUrl = Some(url); - } - - items.push(item); + rating, + views, + uploader: uploader_name, + uploader_url, + }); } - Ok(items) + Ok(cards) + } + + fn hash_regex() -> &'static Regex { + static RE: OnceLock = OnceLock::new(); + RE.get_or_init(|| Regex::new(r#"hash\s*[:=]\s*['"]([0-9a-f]{32})"#).expect("valid regex")) + } + + fn height_regex() -> &'static Regex { + static RE: OnceLock = OnceLock::new(); + RE.get_or_init(|| Regex::new(r"^(\d+)").expect("valid regex")) + } + + fn base36(mut n: u32) -> String { + const DIGITS: &[u8] = b"0123456789abcdefghijklmnopqrstuvwxyz"; + if n == 0 { + return "0".to_string(); + } + let mut buf = Vec::new(); + while n > 0 { + buf.push(DIGITS[(n % 36) as usize]); + n /= 36; + } + buf.reverse(); + String::from_utf8(buf).unwrap_or_default() + } + + /// Reverse-engineered from the site's player JS: the page's 32-hex-char + /// `hash` is split into four 8-hex-char chunks, each parsed as a u32 and + /// base36-encoded, then concatenated. The `/xhr/video/{id}` format API + /// requires this derived value or it serves a restricted/preview + /// response instead of the real sources. + fn calc_hash(hash: &str) -> Option { + if hash.len() != 32 || !hash.is_ascii() { + return None; + } + let mut out = String::new(); + for chunk_start in (0..32).step_by(8) { + let chunk = &hash[chunk_start..chunk_start + 8]; + let n = u32::from_str_radix(chunk, 16).ok()?; + out.push_str(&Self::base36(n)); + } + Some(out) + } + + /// Fetches the detail page for its `hash`, then calls the site's own + /// `/xhr/video/{alnum_id}` JSON API (the same one the site's own player + /// uses) to get real direct mp4 URLs across all available resolutions. + /// Without this, the client is left to open the bare page URL, which + /// eporner serves as a small restricted/preview clip rather than the + /// full video. + async fn resolve_formats( + requester: &mut Requester, + alnum_id: &str, + page_url: &str, + ) -> Option> { + let detail_html = Self::fetch_html(requester, page_url).await.ok()?; + let hash = &Self::hash_regex().captures(&detail_html)?[1]; + let calc = Self::calc_hash(hash)?; + + let api_url = format!( + "{BASE_URL}/xhr/video/{alnum_id}?hash={calc}&device=generic&domain=www.eporner.com&fallback=false" + ); + let json_text = requester + .get_with_headers(&api_url, Self::html_headers(page_url), Some(Version::HTTP_11)) + .await + .ok()?; + let value: serde_json::Value = serde_json::from_str(&json_text).ok()?; + + if value.get("available").and_then(|v| v.as_bool()) == Some(false) { + return None; + } + + let mp4_sources = value.get("sources")?.get("mp4")?.as_object()?; + let mut ranked: Vec<(u32, VideoFormat)> = Vec::new(); + for (key, entry) in mp4_sources { + let Some(src) = entry.get("src").and_then(|v| v.as_str()) else { + continue; + }; + let label = entry + .get("labelShort") + .and_then(|v| v.as_str()) + .unwrap_or(key.as_str()); + let height = Self::height_regex() + .captures(label) + .and_then(|c| c[1].parse::().ok()) + .unwrap_or(0); + let format = VideoFormat::new(src.to_string(), label.to_string(), "mp4".to_string()) + .format_id(key.clone()) + .format_note(key.clone()) + .height(height); + ranked.push((height, format)); + } + if ranked.is_empty() { + return None; + } + ranked.sort_by(|a, b| b.0.cmp(&a.0)); + Some(ranked.into_iter().map(|(_, f)| f).collect()) + } + + async fn build_item(requester: &mut Requester, card: EpornerCard) -> VideoItem { + let formats = Self::resolve_formats(requester, &card.alnum_id, &card.page_url).await; + + let mut item = VideoItem::new( + card.id, + card.title, + card.page_url, + CHANNEL_ID.to_string(), + card.thumb, + card.duration, + ); + if let Some(formats) = formats { + item.formats = Some(formats); + } + if let Some(r) = card.rating { + item.rating = Some(r); + } + if let Some(v) = card.views { + item.views = Some(v); + } + if let Some(name) = card.uploader { + item.uploader = Some(name); + } + if let Some(url) = card.uploader_url { + let uploader_id = url + .trim_end_matches('/') + .rsplit('/') + .next() + .unwrap_or_default() + .to_string(); + if !uploader_id.is_empty() { + item.uploaderId = Some(format!("{CHANNEL_ID}:{uploader_id}")); + } + item.uploaderUrl = Some(url); + } + + item } async fn load_pornstars(pornstar_map: Arc>>) -> Result<()> { @@ -591,17 +748,20 @@ impl EpornerProvider { return Ok(vec![]); } - let items = self.parse_list_page_limited(&html, per_page)?; + let cards = Self::parse_list_cards(&html)?; + let items = stream::iter(cards.into_iter().take(per_page.max(1)).map(|card| { + let mut req = requester.clone(); + async move { Self::build_item(&mut req, card).await } + })) + .buffer_unordered(ENRICH_CONCURRENCY) + .collect::>() + .await; + if !items.is_empty() { cache.insert(cache_key, items.clone()); } Ok(items) } - - fn parse_list_page_limited(&self, html: &str, limit: usize) -> Result> { - let all = Self::parse_list_page(html)?; - Ok(all.into_iter().take(limit.max(1)).collect()) - } } #[async_trait] @@ -642,3 +802,37 @@ impl Provider for EpornerProvider { Some(self.build_channel(cv)) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn calc_hash_matches_site_algorithm() { + // Reverse-engineered from vjs.js; verified against a live page/xhr pair. + let hash = "06aadb04e6f6e853e25f5a583f173617"; + assert_eq!(hash.len(), 32); + assert_eq!( + EpornerProvider::calc_hash(hash).unwrap(), + "1ulk3o1s31fmb1qt66aghi70uv" + ); + } + + #[test] + fn calc_hash_rejects_wrong_length() { + assert!(EpornerProvider::calc_hash("abc").is_none()); + } + + #[test] + fn alnum_id_from_href_extracts_path_segment() { + assert_eq!( + EpornerProvider::alnum_id_from_href("/video-jQDTzcHvtjv/vj-blacked/"), + Some("jQDTzcHvtjv".to_string()) + ); + assert_eq!( + EpornerProvider::alnum_id_from_href("https://www.eporner.com/video-5Ssyrtrs9FG/-/"), + Some("5Ssyrtrs9FG".to_string()) + ); + assert_eq!(EpornerProvider::alnum_id_from_href("/hd-porn/95008/foo/"), None); + } +}