diff --git a/docs/provider-catalog.md b/docs/provider-catalog.md index b3f45c4..8787081 100644 --- a/docs/provider-catalog.md +++ b/docs/provider-catalog.md @@ -50,7 +50,7 @@ This is the current implementation inventory as of this snapshot of the repo. Us | `shooshtime` | `onlyfans` | no | yes | Redirect proxy plus dedicated media route. | | `spankbang` | `mainstream-tube` | no | yes | Best template for redirect proxy plus anti-bot fetches. | | `thaiporntv` | `mainstream-tube` | no | yes | Decodes `data-enc` attribute for proxied HLS playback. | -| `supjav` | `jav` | no | no | JAV/HLS and uploader-id examples. | +| `supjav` | `jav` | no | yes | JAV/HLS provider; detail page URLs for `video.url`, proxied HLS format URLs via `/proxy/supjav/...`. | | `sxyprn` | `mainstream-tube` | no | yes | Redirect proxy helper usage. | | `tnaflix` | `mainstream-tube` | no | no | Mainstream tube provider. | | `tokyomotion` | `jav` | no | no | JAV/tube hybrid. | @@ -91,6 +91,7 @@ These resolve a provider-specific input into a `302 Location`. - `/proxy/pimpbunny/{endpoint}*` - `/proxy/allpornstream/{endpoint}*` - `/proxy/tube8/{endpoint}*` +- `/proxy/supjav/{endpoint}*` - `/proxy/jable/{slug}*` - `/proxy/thepornbunny/{slug}*` diff --git a/src/providers/shooshtime.rs b/src/providers/shooshtime.rs index eab4f36..39cb867 100644 --- a/src/providers/shooshtime.rs +++ b/src/providers/shooshtime.rs @@ -13,7 +13,6 @@ use crate::videos::{ServerOptions, VideoFormat, VideoItem}; use async_trait::async_trait; use chrono::NaiveDate; use error_chain::error_chain; -use futures::stream::{self, StreamExt}; use htmlentity::entity::{ICodedDataTrait, decode}; use regex::Regex; use scraper::{ElementRef, Html, Selector}; @@ -644,7 +643,7 @@ impl ShooshtimeProvider { target.push_str(&quality.replace(' ', "%20")); } - build_proxy_url(options, "shooshtime", &target) + build_proxy_url(options, "shooshtime-media", &target) } fn search_sort_param(sort: &str) -> Option<&'static str> { @@ -1157,7 +1156,11 @@ impl ShooshtimeProvider { } let proxied_url = self.proxied_video(options, page_url, None); if !proxied_url.is_empty() { - item.url = proxied_url; + item.url = page_url.to_string(); + formats.push( + VideoFormat::new(proxied_url, "Best".to_string(), "mp4".to_string()) + .format_id("best".to_string()), + ); } if !formats.is_empty() { item = item.formats(formats); @@ -1194,40 +1197,6 @@ impl ShooshtimeProvider { Ok(item) } - async fn enrich_video(&self, item: VideoItem, options: &ServerOptions) -> VideoItem { - let page_url = item.url.clone(); - let original_item = item.clone(); - - let mut requester = match options.requester.clone() { - Some(requester) => requester, - None => Requester::new(), - }; - - let html = match requester.get(&page_url, None).await { - Ok(html) => html, - Err(error) => { - report_provider_error_background( - "shooshtime", - "enrich_video.request", - &format!("url={}; error={error}", page_url), - ); - return item; - } - }; - - match self.apply_detail_video(item, &html, &page_url, options) { - Ok(item) => item, - Err(error) => { - report_provider_error_background( - "shooshtime", - "enrich_video.parse", - &format!("url={}; error={error}", page_url), - ); - original_item - } - } - } - async fn fetch_items_for_url( &self, cache: VideoCache, @@ -1256,23 +1225,12 @@ impl ShooshtimeProvider { } }; - let list_videos = self.parse_list_videos(&html)?; - if list_videos.is_empty() { + let items = self.parse_list_videos(&html)?; + if items.is_empty() { return Ok(vec![]); } - let items = stream::iter(list_videos.into_iter().map(|video| { - let provider = self.clone(); - let options = options.clone(); - async move { provider.enrich_video(video, &options).await } - })) - .buffer_unordered(6) - .collect::>() - .await; - - if !items.is_empty() { - cache.insert(url, items.clone()); - } + cache.insert(url, items.clone()); Ok(items) } @@ -1422,7 +1380,7 @@ mod tests { assert_eq!( provider.proxied_video(&options, "https://shooshtime.com/videos/example/123/", None,), - "https://example.com/proxy/shooshtime/shooshtime.com/videos/example/123/" + "https://example.com/proxy/shooshtime-media/shooshtime.com/videos/example/123/" ); assert_eq!( provider.proxied_video( @@ -1430,7 +1388,7 @@ mod tests { "https://shooshtime.com/videos/example/123/", Some("720p"), ), - "https://example.com/proxy/shooshtime/shooshtime.com/videos/example/123/__quality__/720p" + "https://example.com/proxy/shooshtime-media/shooshtime.com/videos/example/123/__quality__/720p" ); } } diff --git a/src/providers/supjav.rs b/src/providers/supjav.rs index 8729641..07a2825 100644 --- a/src/providers/supjav.rs +++ b/src/providers/supjav.rs @@ -1,6 +1,9 @@ use crate::DbPool; use crate::api::ClientVersion; -use crate::providers::{Provider, report_provider_error, report_provider_error_background}; +use crate::providers::{ + Provider, build_proxy_url, report_provider_error, report_provider_error_background, + strip_url_scheme, +}; use crate::status::*; use crate::util::cache::VideoCache; use crate::util::parse_abbreviated_number; @@ -1603,6 +1606,7 @@ impl Provider for SupjavProvider { let _ = pool; let page = page.parse::().unwrap_or(1); let per_page_limit = per_page.parse::().unwrap_or(24); + let rewrite_options = options.clone(); let result = match query { Some(query) if !query.trim().is_empty() => { @@ -1613,7 +1617,25 @@ impl Provider for SupjavProvider { }; match result { - Ok(videos) => videos, + Ok(mut videos) => { + for video in &mut videos { + if let Some(formats) = video.formats.as_mut() { + for format in formats { + if format.url.starts_with("/proxy/supjav/") + || format.url.contains("/proxy/supjav/") + { + continue; + } + format.url = build_proxy_url( + &rewrite_options, + CHANNEL_ID, + &strip_url_scheme(&format.url), + ); + } + } + } + videos + } Err(error) => { report_provider_error(CHANNEL_ID, "get_videos", &error.to_string()).await; vec![] @@ -1875,10 +1897,15 @@ mod tests { "supjav items must not serialize embed" ); assert!( - first.url.contains(".m3u8"), - "expected direct m3u8 url, got {}", + first.url.starts_with(BASE_URL), + "expected supjav page url, got {}", first.url ); + let formats = first.formats.as_ref().expect("supjav item should have formats"); + let master_format = formats + .iter() + .find(|f| f.url.contains(".m3u8") || f.url.contains("/proxy/supjav/")) + .expect("formats should contain a proxied m3u8 url"); let mut requester = Requester::new(); let thumb_response = requester @@ -1896,7 +1923,7 @@ mod tests { ); let ytdlp = Command::new("yt-dlp") - .args(["--no-warnings", "--simulate", "--skip-download", &first.url]) + .args(["--no-warnings", "--simulate", "--skip-download", &master_format.url]) .output() .expect("yt-dlp should run"); assert!( diff --git a/src/proxies/mod.rs b/src/proxies/mod.rs index c114c9d..394f177 100644 --- a/src/proxies/mod.rs +++ b/src/proxies/mod.rs @@ -5,12 +5,12 @@ use crate::proxies::doodstream::DoodstreamProxy; use crate::proxies::heavyfetish::HeavyfetishProxy; use crate::proxies::hqporner::HqpornerProxy; use crate::proxies::pornhd3x::Pornhd3xProxy; +use crate::proxies::supjav::SupjavProxy; use crate::proxies::tube8::Tube8Proxy; use ntex::web; use crate::proxies::pimpbunny::PimpbunnyProxy; use crate::proxies::porndish::PorndishProxy; -use crate::proxies::shooshtime::ShooshtimeProxy; use crate::proxies::spankbang::SpankbangProxy; use crate::proxies::vjav::VjavProxy; use crate::{proxies::sxyprn::SxyprnProxy, util::requester::Requester}; @@ -36,6 +36,7 @@ pub mod pornhd3x; pub mod pornhubthumb; pub mod shooshtime; pub mod spankbang; +pub mod supjav; pub mod sxyprn; pub mod thaiporntv; pub mod jable; @@ -56,7 +57,6 @@ pub enum AnyProxy { Pimpbunny(PimpbunnyProxy), Porndish(PorndishProxy), Spankbang(SpankbangProxy), - Shooshtime(ShooshtimeProxy), Hqporner(HqpornerProxy), Heavyfetish(HeavyfetishProxy), Vjav(VjavProxy), @@ -64,6 +64,7 @@ pub enum AnyProxy { Clapdat(ClapdatProxy), ThaipornTv(ThaipornTvProxy), Tube8(Tube8Proxy), + Supjav(SupjavProxy), } pub trait Proxy { @@ -83,7 +84,6 @@ impl Proxy for AnyProxy { AnyProxy::Pimpbunny(p) => p.get_video_url(url, requester).await, AnyProxy::Porndish(p) => p.get_video_url(url, requester).await, AnyProxy::Spankbang(p) => p.get_video_url(url, requester).await, - AnyProxy::Shooshtime(p) => p.get_video_url(url, requester).await, AnyProxy::Hqporner(p) => p.get_video_url(url, requester).await, AnyProxy::Heavyfetish(p) => p.get_video_url(url, requester).await, AnyProxy::Vjav(p) => p.get_video_url(url, requester).await, @@ -91,6 +91,7 @@ impl Proxy for AnyProxy { AnyProxy::Clapdat(p) => p.get_video_url(url, requester).await, AnyProxy::ThaipornTv(p) => p.get_video_url(url, requester).await, AnyProxy::Tube8(p) => p.get_video_url(url, requester).await, + AnyProxy::Supjav(p) => p.get_video_url(url, requester).await, } } } diff --git a/src/proxies/supjav.rs b/src/proxies/supjav.rs new file mode 100644 index 0000000..31aeaaf --- /dev/null +++ b/src/proxies/supjav.rs @@ -0,0 +1,62 @@ +use ntex::web; +use url::Url; + +use crate::util::requester::Requester; + +#[derive(Debug, Clone)] +pub struct SupjavProxy {} + +impl SupjavProxy { + pub fn new() -> Self { + Self {} + } + + fn normalize_target(endpoint: &str) -> Option { + let endpoint = endpoint.trim().trim_start_matches('/'); + if endpoint.is_empty() { + return None; + } + + let target = if endpoint.starts_with("http://") || endpoint.starts_with("https://") { + endpoint.to_string() + } else { + format!("https://{endpoint}") + }; + + Self::is_allowed_media_url(&target).then_some(target) + } + + fn is_allowed_media_url(url: &str) -> bool { + let Some(parsed) = Url::parse(url).ok() else { + return false; + }; + if parsed.scheme() != "https" { + return false; + } + + let Some(host) = parsed.host_str() else { + return false; + }; + let host = host.to_ascii_lowercase(); + if !(host == "turbovidhls.com" + || host == "turboviplay.com" + || host.ends_with(".turboviplay.com") + || host.ends_with(".turbovidhls.com")) + { + return false; + } + + parsed.path().to_ascii_lowercase().contains(".m3u8") + } +} + +impl crate::proxies::Proxy for SupjavProxy { + async fn get_video_url( + &self, + url: String, + _requester: web::types::State, + ) -> String { + Self::normalize_target(&url).unwrap_or_default() + } +} + diff --git a/src/proxy.rs b/src/proxy.rs index bf8e53b..92c0885 100644 --- a/src/proxy.rs +++ b/src/proxy.rs @@ -10,8 +10,8 @@ use crate::proxies::javtiful::JavtifulProxy; use crate::proxies::pimpbunny::PimpbunnyProxy; use crate::proxies::porndish::PorndishProxy; use crate::proxies::pornhd3x::Pornhd3xProxy; -use crate::proxies::shooshtime::ShooshtimeProxy; use crate::proxies::spankbang::SpankbangProxy; +use crate::proxies::supjav::SupjavProxy; use crate::proxies::sxyprn::SxyprnProxy; use crate::proxies::tube8::Tube8Proxy; use crate::proxies::vjav::VjavProxy; @@ -82,11 +82,6 @@ pub fn config(cfg: &mut web::ServiceConfig) { .route(web::post().to(proxy2redirect)) .route(web::get().to(proxy2redirect)), ) - .service( - web::resource("/shooshtime/{endpoint}*") - .route(web::post().to(proxy2redirect)) - .route(web::get().to(proxy2redirect)), - ) .service( web::resource("/vidara/{endpoint}*") .route(web::post().to(proxy2redirect)) @@ -142,6 +137,12 @@ pub fn config(cfg: &mut web::ServiceConfig) { .route(web::post().to(proxy2redirect)) .route(web::get().to(proxy2redirect)), ); + cfg.service( + web::resource("/supjav/{endpoint}*") + .route(web::post().to(proxy2redirect)) + .route(web::get().to(proxy2redirect)) + .route(web::head().to(proxy2redirect)), + ); cfg.service( web::resource("/jable/{slug}*") .route(web::get().to(crate::proxies::jable::redirect_to_page)) @@ -185,8 +186,7 @@ fn get_proxy(proxy: &str) -> Option { "heavyfetish" => Some(AnyProxy::Heavyfetish(HeavyfetishProxy::new())), "vjav" => Some(AnyProxy::Vjav(VjavProxy::new())), "pornhd3x" => Some(AnyProxy::Pornhd3x(Pornhd3xProxy::new())), - "shooshtime" => Some(AnyProxy::Shooshtime(ShooshtimeProxy::new())), - "vidara" => Some(AnyProxy::Vidara(VidaraProxy::new())), +"vidara" => Some(AnyProxy::Vidara(VidaraProxy::new())), "pimpbunny" => Some(AnyProxy::Pimpbunny(PimpbunnyProxy::new())), "porndish" => Some(AnyProxy::Porndish(PorndishProxy::new())), "spankbang" => Some(AnyProxy::Spankbang(SpankbangProxy::new())), @@ -194,6 +194,7 @@ fn get_proxy(proxy: &str) -> Option { "thaiporntv" => Some(AnyProxy::ThaipornTv(ThaipornTvProxy::new())), "allpornstream" => Some(AnyProxy::AllPornStream(AllPornStreamProxy::new())), "tube8" => Some(AnyProxy::Tube8(Tube8Proxy::new())), + "supjav" => Some(AnyProxy::Supjav(SupjavProxy::new())), _ => None, } }