noodlemagazine proxy implementation

src/providers/noodlemagazine.rs

@@ -4,13 +4,11 @@ use crate::providers::Provider;
 use crate::status::*;
 use crate::util::cache::VideoCache;
 use crate::util::parse_abbreviated_number;
-use crate::util::requester::Requester;
 use crate::util::time::parse_time_to_seconds;
 use crate::videos::{ServerOptions, VideoFormat, VideoItem};
 
 use async_trait::async_trait;
 use error_chain::error_chain;
-use futures::future::join_all;
 use htmlentity::entity::{ICodedDataTrait, decode};
 use std::vec;
 use titlecase::Titlecase;
@@ -82,7 +80,7 @@ impl NoodlemagazineProvider {
             .await
             .unwrap_or_default();
 
-        let items = self.get_video_items_from_html(text, requester).await;
+        let items = self.get_video_items_from_html(text);
 
         if items.is_empty() {
             Ok(old_items)
@@ -119,7 +117,7 @@ impl NoodlemagazineProvider {
             .await
             .unwrap_or_default();
 
-        let items = self.get_video_items_from_html(text, requester).await;
+        let items = self.get_video_items_from_html(text);
 
         if items.is_empty() {
             Ok(old_items)
@@ -130,11 +128,7 @@ impl NoodlemagazineProvider {
         }
     }
 
-    async fn get_video_items_from_html(
+    fn get_video_items_from_html(&self, html: String) -> Vec<VideoItem> {
-        &self,
-        html: String,
-        requester: Requester,
-    ) -> Vec<VideoItem> {
         if html.is_empty() || html.contains("404 Not Found") {
             return vec![];
         }
@@ -152,22 +146,23 @@ impl NoodlemagazineProvider {
             None => return vec![],
         };
 
-        let raw_videos = list
+        list.split("<div class=\"item\">")
-            .split("<div class=\"item\">")
             .skip(1)
-            .map(|s| s.to_string());
+            .filter_map(|segment| self.get_video_item(segment.to_string()).ok())
-
+            .collect()
-        let futures = raw_videos.map(|v| self.get_video_item(v, requester.clone()));
-        let results = join_all(futures).await;
-
-        results.into_iter().filter_map(Result::ok).collect()
     }
 
-    async fn get_video_item(
+    fn proxy_url(&self, video_url: &str) -> String {
-        &self,
+        let target = video_url
-        video_segment: String,
+            .strip_prefix("https://")
-        requester: Requester,
+            .or_else(|| video_url.strip_prefix("http://"))
-    ) -> Result<VideoItem> {
+            .unwrap_or(video_url)
+            .trim_start_matches('/');
+
+        format!("https://hottub.spacemoehre.de/proxy/noodlemagazine/{target}")
+    }
+
+    fn get_video_item(&self, video_segment: String) -> Result<VideoItem> {
         let href = video_segment
             .split("<a href=\"")
             .nth(1)
@@ -217,54 +212,22 @@ impl NoodlemagazineProvider {
             .and_then(|s| s.split('<').next())
             .and_then(|v| parse_abbreviated_number(v.trim()))
             .unwrap_or(0);
-
+        let proxy_url = self.proxy_url(&video_url);
-        let formats = self
-            .extract_media(&video_url, requester)
-            .await
-            .ok_or_else(|| Error::from("media extraction failed"))?;
 
         Ok(VideoItem::new(
             id,
             title,
-            video_url,
+            proxy_url.clone(),
             "noodlemagazine".into(),
             thumb,
             duration,
         )
         .views(views)
-        .formats(formats))
+        .formats(vec![
-    }
+            VideoFormat::new(proxy_url, "auto".into(), "video/mp4".into())
-
+                .format_id("auto".into())
-    async fn extract_media(
+                .format_note("proxied".into()),
-        &self,
+        ]))
-        video_url: &String,
-        mut requester: Requester,
-    ) -> Option<Vec<VideoFormat>> {
-        let text = requester
-            .get(video_url, Some(Version::HTTP_2))
-            .await
-            .unwrap_or_default();
-
-        let json_str = text.split("window.playlist = ").nth(1)?.split(';').next()?;
-
-        let json: serde_json::Value = serde_json::from_str(json_str).ok()?;
-        let sources = json["sources"].as_array()?;
-
-        let mut formats = vec![];
-
-        for s in sources {
-            let file = s["file"].as_str()?.to_string();
-            let label = s["label"].as_str().unwrap_or("unknown").to_string();
-
-            formats.push(
-                VideoFormat::new(file, label.clone(), "video/mp4".into())
-                    .format_id(label.clone())
-                    .format_note(label.clone())
-                    .http_header("Referer".into(), video_url.clone()),
-            );
-        }
-
-        Some(formats.into_iter().rev().collect())
     }
 }
 
@@ -300,3 +263,44 @@ impl Provider for NoodlemagazineProvider {
         Some(self.build_channel(clientversion))
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::NoodlemagazineProvider;
+
+    #[test]
+    fn rewrites_video_pages_to_hottub_proxy() {
+        let provider = NoodlemagazineProvider::new();
+
+        assert_eq!(
+            provider.proxy_url("https://noodlemagazine.com/watch/-123_456"),
+            "https://hottub.spacemoehre.de/proxy/noodlemagazine/noodlemagazine.com/watch/-123_456"
+        );
+    }
+
+    #[test]
+    fn parses_listing_without_detail_page_requests() {
+        let provider = NoodlemagazineProvider::new();
+        let html = r#"
+            <div class="list_videos" id="list_videos">
+                <div class="item">
+                    <a href="/watch/-123_456">
+                        <img data-src="https://thumb.example/test.jpg" />
+                    </a>
+                    <div class="title">sample &amp; title</div>
+                    <svg><use></use></svg>#clock-o"></use></svg>12:34<
+                    <svg><use></use></svg>#eye"></use></svg>1.2K<
+                </div>
+            >Show more</div>
+        "#;
+
+        let items = provider.get_video_items_from_html(html.to_string());
+
+        assert_eq!(items.len(), 1);
+        assert_eq!(
+            items[0].url,
+            "https://hottub.spacemoehre.de/proxy/noodlemagazine/noodlemagazine.com/watch/-123_456"
+        );
+        assert_eq!(items[0].formats.as_ref().map(|f| f.len()), Some(1));
+    }
+}

src/proxies/mod.rs

@@ -1,11 +1,13 @@
 use ntex::web;
 
+use crate::proxies::noodlemagazine::NoodlemagazineProxy;
 use crate::proxies::spankbang::SpankbangProxy;
 use crate::{proxies::sxyprn::SxyprnProxy, util::requester::Requester};
 
 pub mod hanimecdn;
 pub mod hqpornerthumb;
 pub mod javtiful;
+pub mod noodlemagazine;
 pub mod spankbang;
 pub mod sxyprn;
 
@@ -14,6 +16,7 @@ pub enum AnyProxy {
     Sxyprn(SxyprnProxy),
     Javtiful(javtiful::JavtifulProxy),
     Spankbang(SpankbangProxy),
+    Noodlemagazine(NoodlemagazineProxy),
 }
 
 pub trait Proxy {
@@ -26,6 +29,7 @@ impl Proxy for AnyProxy {
             AnyProxy::Sxyprn(p) => p.get_video_url(url, requester).await,
             AnyProxy::Javtiful(p) => p.get_video_url(url, requester).await,
             AnyProxy::Spankbang(p) => p.get_video_url(url, requester).await,
+            AnyProxy::Noodlemagazine(p) => p.get_video_url(url, requester).await,
         }
     }
 }

src/proxies/noodlemagazine.rs

@@ -0,0 +1,110 @@
+use ntex::web;
+use serde_json::Value;
+use wreq::Version;
+
+use crate::util::requester::Requester;
+
+#[derive(Debug, Clone)]
+pub struct NoodlemagazineProxy {}
+
+impl NoodlemagazineProxy {
+    pub fn new() -> Self {
+        NoodlemagazineProxy {}
+    }
+
+    fn extract_playlist(text: &str) -> Option<&str> {
+        text.split("window.playlist = ").nth(1)?.split(';').next()
+    }
+
+    fn source_score(source: &Value) -> (u8, u32) {
+        let file = source["file"].as_str().unwrap_or_default();
+        let label = source["label"].as_str().unwrap_or_default();
+        let is_hls = u8::from(file.contains(".m3u8"));
+        let quality = label
+            .chars()
+            .filter(|c| c.is_ascii_digit())
+            .collect::<String>()
+            .parse::<u32>()
+            .unwrap_or(0);
+
+        (is_hls, quality)
+    }
+
+    fn select_best_source(playlist: &str) -> Option<String> {
+        let json: Value = serde_json::from_str(playlist).ok()?;
+        let sources = json["sources"].as_array()?;
+
+        sources
+            .iter()
+            .filter(|source| {
+                source["file"]
+                    .as_str()
+                    .map(|file| !file.is_empty())
+                    .unwrap_or(false)
+            })
+            .max_by_key(|source| Self::source_score(source))
+            .and_then(|source| source["file"].as_str())
+            .map(str::to_string)
+    }
+
+    pub async fn get_video_url(
+        &self,
+        url: String,
+        requester: web::types::State<Requester>,
+    ) -> String {
+        let mut requester = requester.get_ref().clone();
+        let url = if url.starts_with("http://") || url.starts_with("https://") {
+            url
+        } else {
+            format!("https://{}", url.trim_start_matches('/'))
+        };
+        let text = requester
+            .get(&url, Some(Version::HTTP_2))
+            .await
+            .unwrap_or_default();
+        if text.is_empty() {
+            return String::new();
+        }
+
+        let Some(playlist) = Self::extract_playlist(&text) else {
+            return String::new();
+        };
+
+        Self::select_best_source(playlist).unwrap_or_default()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::NoodlemagazineProxy;
+
+    #[test]
+    fn extracts_playlist_from_page() {
+        let html = r#"
+            <script>
+                window.playlist = {"sources":[{"file":"https://cdn.example/360.mp4","label":"360p"}]};
+            </script>
+        "#;
+
+        assert_eq!(
+            NoodlemagazineProxy::extract_playlist(html),
+            Some(r#"{"sources":[{"file":"https://cdn.example/360.mp4","label":"360p"}]}"#)
+        );
+    }
+
+    #[test]
+    fn prefers_hls_then_highest_quality() {
+        let playlist = r#"{
+            "sources": [
+                {"file":"https://cdn.example/360.mp4","label":"360p"},
+                {"file":"https://cdn.example/720.mp4","label":"720p"},
+                {"file":"https://cdn.example/master.m3u8","label":"1080p"}
+            ]
+        }"#;
+
+        assert_eq!(
+            NoodlemagazineProxy::select_best_source(playlist).as_deref(),
+            Some("https://cdn.example/master.m3u8")
+        );
+    }
+}

src/proxy.rs

@@ -1,6 +1,7 @@
 use ntex::web::{self, HttpRequest};
 
 use crate::proxies::javtiful::JavtifulProxy;
+use crate::proxies::noodlemagazine::NoodlemagazineProxy;
 use crate::proxies::spankbang::SpankbangProxy;
 use crate::proxies::sxyprn::SxyprnProxy;
 use crate::proxies::*;
@@ -22,6 +23,11 @@ pub fn config(cfg: &mut web::ServiceConfig) {
             .route(web::post().to(proxy2redirect))
             .route(web::get().to(proxy2redirect)),
     )
+    .service(
+        web::resource("/noodlemagazine/{endpoint}*")
+            .route(web::post().to(proxy2redirect))
+            .route(web::get().to(proxy2redirect)),
+    )
     .service(
         web::resource("/hanime-cdn/{endpoint}*")
             .route(web::post().to(crate::proxies::hanimecdn::get_image))
@@ -54,6 +60,7 @@ fn get_proxy(proxy: &str) -> Option<AnyProxy> {
         "sxyprn" => Some(AnyProxy::Sxyprn(SxyprnProxy::new())),
         "javtiful" => Some(AnyProxy::Javtiful(JavtifulProxy::new())),
         "spankbang" => Some(AnyProxy::Spankbang(SpankbangProxy::new())),
+        "noodlemagazine" => Some(AnyProxy::Noodlemagazine(NoodlemagazineProxy::new())),
         _ => None,
     }
 }