supjav and shooshtime fixes

docs/provider-catalog.md

@@ -50,7 +50,7 @@ This is the current implementation inventory as of this snapshot of the repo. Us
 | `shooshtime` | `onlyfans` | no | yes | Redirect proxy plus dedicated media route. |
 | `spankbang` | `mainstream-tube` | no | yes | Best template for redirect proxy plus anti-bot fetches. |
 | `thaiporntv` | `mainstream-tube` | no | yes | Decodes `data-enc` attribute for proxied HLS playback. |
-| `supjav` | `jav` | no | no | JAV/HLS and uploader-id examples. |
+| `supjav` | `jav` | no | yes | JAV/HLS provider; detail page URLs for `video.url`, proxied HLS format URLs via `/proxy/supjav/...`. |
 | `sxyprn` | `mainstream-tube` | no | yes | Redirect proxy helper usage. |
 | `tnaflix` | `mainstream-tube` | no | no | Mainstream tube provider. |
 | `tokyomotion` | `jav` | no | no | JAV/tube hybrid. |
@@ -91,6 +91,7 @@ These resolve a provider-specific input into a `302 Location`.
 - `/proxy/pimpbunny/{endpoint}*`
 - `/proxy/allpornstream/{endpoint}*`
 - `/proxy/tube8/{endpoint}*`
+- `/proxy/supjav/{endpoint}*`
 - `/proxy/jable/{slug}*`
 - `/proxy/thepornbunny/{slug}*`
 

src/providers/shooshtime.rs

@@ -13,7 +13,6 @@ use crate::videos::{ServerOptions, VideoFormat, VideoItem};
 use async_trait::async_trait;
 use chrono::NaiveDate;
 use error_chain::error_chain;
-use futures::stream::{self, StreamExt};
 use htmlentity::entity::{ICodedDataTrait, decode};
 use regex::Regex;
 use scraper::{ElementRef, Html, Selector};
@@ -644,7 +643,7 @@ impl ShooshtimeProvider {
             target.push_str(&quality.replace(' ', "%20"));
         }
 
-        build_proxy_url(options, "shooshtime", &target)
+        build_proxy_url(options, "shooshtime-media", &target)
     }
 
     fn search_sort_param(sort: &str) -> Option<&'static str> {
@@ -1157,7 +1156,11 @@ impl ShooshtimeProvider {
         }
         let proxied_url = self.proxied_video(options, page_url, None);
         if !proxied_url.is_empty() {
-            item.url = proxied_url;
+            item.url = page_url.to_string();
+            formats.push(
+                VideoFormat::new(proxied_url, "Best".to_string(), "mp4".to_string())
+                    .format_id("best".to_string()),
+            );
         }
         if !formats.is_empty() {
             item = item.formats(formats);
@@ -1194,40 +1197,6 @@ impl ShooshtimeProvider {
         Ok(item)
     }
 
-    async fn enrich_video(&self, item: VideoItem, options: &ServerOptions) -> VideoItem {
-        let page_url = item.url.clone();
-        let original_item = item.clone();
-
-        let mut requester = match options.requester.clone() {
-            Some(requester) => requester,
-            None => Requester::new(),
-        };
-
-        let html = match requester.get(&page_url, None).await {
-            Ok(html) => html,
-            Err(error) => {
-                report_provider_error_background(
-                    "shooshtime",
-                    "enrich_video.request",
-                    &format!("url={}; error={error}", page_url),
-                );
-                return item;
-            }
-        };
-
-        match self.apply_detail_video(item, &html, &page_url, options) {
-            Ok(item) => item,
-            Err(error) => {
-                report_provider_error_background(
-                    "shooshtime",
-                    "enrich_video.parse",
-                    &format!("url={}; error={error}", page_url),
-                );
-                original_item
-            }
-        }
-    }
-
     async fn fetch_items_for_url(
         &self,
         cache: VideoCache,
@@ -1256,23 +1225,12 @@ impl ShooshtimeProvider {
             }
         };
 
-        let list_videos = self.parse_list_videos(&html)?;
+        let items = self.parse_list_videos(&html)?;
-        if list_videos.is_empty() {
+        if items.is_empty() {
             return Ok(vec![]);
         }
 
-        let items = stream::iter(list_videos.into_iter().map(|video| {
-            let provider = self.clone();
-            let options = options.clone();
-            async move { provider.enrich_video(video, &options).await }
-        }))
-        .buffer_unordered(6)
-        .collect::<Vec<_>>()
-        .await;
-
-        if !items.is_empty() {
         cache.insert(url, items.clone());
-        }
 
         Ok(items)
     }
@@ -1422,7 +1380,7 @@ mod tests {
 
         assert_eq!(
             provider.proxied_video(&options, "https://shooshtime.com/videos/example/123/", None,),
-            "https://example.com/proxy/shooshtime/shooshtime.com/videos/example/123/"
+            "https://example.com/proxy/shooshtime-media/shooshtime.com/videos/example/123/"
         );
         assert_eq!(
             provider.proxied_video(
@@ -1430,7 +1388,7 @@ mod tests {
                 "https://shooshtime.com/videos/example/123/",
                 Some("720p"),
             ),
-            "https://example.com/proxy/shooshtime/shooshtime.com/videos/example/123/__quality__/720p"
+            "https://example.com/proxy/shooshtime-media/shooshtime.com/videos/example/123/__quality__/720p"
         );
     }
 }

src/providers/supjav.rs

@@ -1,6 +1,9 @@
 use crate::DbPool;
 use crate::api::ClientVersion;
-use crate::providers::{Provider, report_provider_error, report_provider_error_background};
+use crate::providers::{
+    Provider, build_proxy_url, report_provider_error, report_provider_error_background,
+    strip_url_scheme,
+};
 use crate::status::*;
 use crate::util::cache::VideoCache;
 use crate::util::parse_abbreviated_number;
@@ -1603,6 +1606,7 @@ impl Provider for SupjavProvider {
         let _ = pool;
         let page = page.parse::<u16>().unwrap_or(1);
         let per_page_limit = per_page.parse::<usize>().unwrap_or(24);
+        let rewrite_options = options.clone();
 
         let result = match query {
             Some(query) if !query.trim().is_empty() => {
@@ -1613,7 +1617,25 @@ impl Provider for SupjavProvider {
         };
 
         match result {
-            Ok(videos) => videos,
+            Ok(mut videos) => {
+                for video in &mut videos {
+                    if let Some(formats) = video.formats.as_mut() {
+                        for format in formats {
+                            if format.url.starts_with("/proxy/supjav/")
+                                || format.url.contains("/proxy/supjav/")
+                            {
+                                continue;
+                            }
+                            format.url = build_proxy_url(
+                                &rewrite_options,
+                                CHANNEL_ID,
+                                &strip_url_scheme(&format.url),
+                            );
+                        }
+                    }
+                }
+                videos
+            }
             Err(error) => {
                 report_provider_error(CHANNEL_ID, "get_videos", &error.to_string()).await;
                 vec![]
@@ -1875,10 +1897,15 @@ mod tests {
             "supjav items must not serialize embed"
         );
         assert!(
-            first.url.contains(".m3u8"),
+            first.url.starts_with(BASE_URL),
-            "expected direct m3u8 url, got {}",
+            "expected supjav page url, got {}",
             first.url
         );
+        let formats = first.formats.as_ref().expect("supjav item should have formats");
+        let master_format = formats
+            .iter()
+            .find(|f| f.url.contains(".m3u8") || f.url.contains("/proxy/supjav/"))
+            .expect("formats should contain a proxied m3u8 url");
 
         let mut requester = Requester::new();
         let thumb_response = requester
@@ -1896,7 +1923,7 @@ mod tests {
         );
 
         let ytdlp = Command::new("yt-dlp")
-            .args(["--no-warnings", "--simulate", "--skip-download", &first.url])
+            .args(["--no-warnings", "--simulate", "--skip-download", &master_format.url])
             .output()
             .expect("yt-dlp should run");
         assert!(

src/proxies/mod.rs

@@ -5,12 +5,12 @@ use crate::proxies::doodstream::DoodstreamProxy;
 use crate::proxies::heavyfetish::HeavyfetishProxy;
 use crate::proxies::hqporner::HqpornerProxy;
 use crate::proxies::pornhd3x::Pornhd3xProxy;
+use crate::proxies::supjav::SupjavProxy;
 use crate::proxies::tube8::Tube8Proxy;
 use ntex::web;
 
 use crate::proxies::pimpbunny::PimpbunnyProxy;
 use crate::proxies::porndish::PorndishProxy;
-use crate::proxies::shooshtime::ShooshtimeProxy;
 use crate::proxies::spankbang::SpankbangProxy;
 use crate::proxies::vjav::VjavProxy;
 use crate::{proxies::sxyprn::SxyprnProxy, util::requester::Requester};
@@ -36,6 +36,7 @@ pub mod pornhd3x;
 pub mod pornhubthumb;
 pub mod shooshtime;
 pub mod spankbang;
+pub mod supjav;
 pub mod sxyprn;
 pub mod thaiporntv;
 pub mod jable;
@@ -56,7 +57,6 @@ pub enum AnyProxy {
     Pimpbunny(PimpbunnyProxy),
     Porndish(PorndishProxy),
     Spankbang(SpankbangProxy),
-    Shooshtime(ShooshtimeProxy),
     Hqporner(HqpornerProxy),
     Heavyfetish(HeavyfetishProxy),
     Vjav(VjavProxy),
@@ -64,6 +64,7 @@ pub enum AnyProxy {
     Clapdat(ClapdatProxy),
     ThaipornTv(ThaipornTvProxy),
     Tube8(Tube8Proxy),
+    Supjav(SupjavProxy),
 }
 
 pub trait Proxy {
@@ -83,7 +84,6 @@ impl Proxy for AnyProxy {
             AnyProxy::Pimpbunny(p) => p.get_video_url(url, requester).await,
             AnyProxy::Porndish(p) => p.get_video_url(url, requester).await,
             AnyProxy::Spankbang(p) => p.get_video_url(url, requester).await,
-            AnyProxy::Shooshtime(p) => p.get_video_url(url, requester).await,
             AnyProxy::Hqporner(p) => p.get_video_url(url, requester).await,
             AnyProxy::Heavyfetish(p) => p.get_video_url(url, requester).await,
             AnyProxy::Vjav(p) => p.get_video_url(url, requester).await,
@@ -91,6 +91,7 @@ impl Proxy for AnyProxy {
             AnyProxy::Clapdat(p) => p.get_video_url(url, requester).await,
             AnyProxy::ThaipornTv(p) => p.get_video_url(url, requester).await,
             AnyProxy::Tube8(p) => p.get_video_url(url, requester).await,
+            AnyProxy::Supjav(p) => p.get_video_url(url, requester).await,
         }
     }
 }

src/proxies/supjav.rs

@@ -0,0 +1,62 @@
+use ntex::web;
+use url::Url;
+
+use crate::util::requester::Requester;
+
+#[derive(Debug, Clone)]
+pub struct SupjavProxy {}
+
+impl SupjavProxy {
+    pub fn new() -> Self {
+        Self {}
+    }
+
+    fn normalize_target(endpoint: &str) -> Option<String> {
+        let endpoint = endpoint.trim().trim_start_matches('/');
+        if endpoint.is_empty() {
+            return None;
+        }
+
+        let target = if endpoint.starts_with("http://") || endpoint.starts_with("https://") {
+            endpoint.to_string()
+        } else {
+            format!("https://{endpoint}")
+        };
+
+        Self::is_allowed_media_url(&target).then_some(target)
+    }
+
+    fn is_allowed_media_url(url: &str) -> bool {
+        let Some(parsed) = Url::parse(url).ok() else {
+            return false;
+        };
+        if parsed.scheme() != "https" {
+            return false;
+        }
+
+        let Some(host) = parsed.host_str() else {
+            return false;
+        };
+        let host = host.to_ascii_lowercase();
+        if !(host == "turbovidhls.com"
+            || host == "turboviplay.com"
+            || host.ends_with(".turboviplay.com")
+            || host.ends_with(".turbovidhls.com"))
+        {
+            return false;
+        }
+
+        parsed.path().to_ascii_lowercase().contains(".m3u8")
+    }
+}
+
+impl crate::proxies::Proxy for SupjavProxy {
+    async fn get_video_url(
+        &self,
+        url: String,
+        _requester: web::types::State<Requester>,
+    ) -> String {
+        Self::normalize_target(&url).unwrap_or_default()
+    }
+}
+

src/proxy.rs

@@ -10,8 +10,8 @@ use crate::proxies::javtiful::JavtifulProxy;
 use crate::proxies::pimpbunny::PimpbunnyProxy;
 use crate::proxies::porndish::PorndishProxy;
 use crate::proxies::pornhd3x::Pornhd3xProxy;
-use crate::proxies::shooshtime::ShooshtimeProxy;
 use crate::proxies::spankbang::SpankbangProxy;
+use crate::proxies::supjav::SupjavProxy;
 use crate::proxies::sxyprn::SxyprnProxy;
 use crate::proxies::tube8::Tube8Proxy;
 use crate::proxies::vjav::VjavProxy;
@@ -82,11 +82,6 @@ pub fn config(cfg: &mut web::ServiceConfig) {
             .route(web::post().to(proxy2redirect))
             .route(web::get().to(proxy2redirect)),
     )
-    .service(
-        web::resource("/shooshtime/{endpoint}*")
-            .route(web::post().to(proxy2redirect))
-            .route(web::get().to(proxy2redirect)),
-    )
     .service(
         web::resource("/vidara/{endpoint}*")
             .route(web::post().to(proxy2redirect))
@@ -142,6 +137,12 @@ pub fn config(cfg: &mut web::ServiceConfig) {
             .route(web::post().to(proxy2redirect))
             .route(web::get().to(proxy2redirect)),
     );
+    cfg.service(
+        web::resource("/supjav/{endpoint}*")
+            .route(web::post().to(proxy2redirect))
+            .route(web::get().to(proxy2redirect))
+            .route(web::head().to(proxy2redirect)),
+    );
     cfg.service(
         web::resource("/jable/{slug}*")
             .route(web::get().to(crate::proxies::jable::redirect_to_page))
@@ -185,8 +186,7 @@ fn get_proxy(proxy: &str) -> Option<AnyProxy> {
         "heavyfetish" => Some(AnyProxy::Heavyfetish(HeavyfetishProxy::new())),
         "vjav" => Some(AnyProxy::Vjav(VjavProxy::new())),
         "pornhd3x" => Some(AnyProxy::Pornhd3x(Pornhd3xProxy::new())),
-        "shooshtime" => Some(AnyProxy::Shooshtime(ShooshtimeProxy::new())),
+"vidara" => Some(AnyProxy::Vidara(VidaraProxy::new())),
-        "vidara" => Some(AnyProxy::Vidara(VidaraProxy::new())),
         "pimpbunny" => Some(AnyProxy::Pimpbunny(PimpbunnyProxy::new())),
         "porndish" => Some(AnyProxy::Porndish(PorndishProxy::new())),
         "spankbang" => Some(AnyProxy::Spankbang(SpankbangProxy::new())),
@@ -194,6 +194,7 @@ fn get_proxy(proxy: &str) -> Option<AnyProxy> {
         "thaiporntv" => Some(AnyProxy::ThaipornTv(ThaipornTvProxy::new())),
         "allpornstream" => Some(AnyProxy::AllPornStream(AllPornStreamProxy::new())),
         "tube8" => Some(AnyProxy::Tube8(Tube8Proxy::new())),
+        "supjav" => Some(AnyProxy::Supjav(SupjavProxy::new())),
         _ => None,
     }
 }