noodlemagazine proxy implementation
This commit is contained in:
Simon
@@ -4,13 +4,11 @@ use crate::providers::Provider;
|
||||
use crate::status::*;
|
||||
use crate::util::cache::VideoCache;
|
||||
use crate::util::parse_abbreviated_number;
|
||||
use crate::util::requester::Requester;
|
||||
use crate::util::time::parse_time_to_seconds;
|
||||
use crate::videos::{ServerOptions, VideoFormat, VideoItem};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use error_chain::error_chain;
|
||||
use futures::future::join_all;
|
||||
use htmlentity::entity::{ICodedDataTrait, decode};
|
||||
use std::vec;
|
||||
use titlecase::Titlecase;
|
||||
@@ -82,7 +80,7 @@ impl NoodlemagazineProvider {
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
|
||||
let items = self.get_video_items_from_html(text, requester).await;
|
||||
let items = self.get_video_items_from_html(text);
|
||||
|
||||
if items.is_empty() {
|
||||
Ok(old_items)
|
||||
@@ -119,7 +117,7 @@ impl NoodlemagazineProvider {
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
|
||||
let items = self.get_video_items_from_html(text, requester).await;
|
||||
let items = self.get_video_items_from_html(text);
|
||||
|
||||
if items.is_empty() {
|
||||
Ok(old_items)
|
||||
@@ -130,11 +128,7 @@ impl NoodlemagazineProvider {
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_video_items_from_html(
|
||||
&self,
|
||||
html: String,
|
||||
requester: Requester,
|
||||
) -> Vec<VideoItem> {
|
||||
fn get_video_items_from_html(&self, html: String) -> Vec<VideoItem> {
|
||||
if html.is_empty() || html.contains("404 Not Found") {
|
||||
return vec![];
|
||||
}
|
||||
@@ -152,22 +146,23 @@ impl NoodlemagazineProvider {
|
||||
None => return vec![],
|
||||
};
|
||||
|
||||
let raw_videos = list
|
||||
.split("<div class=\"item\">")
|
||||
list.split("<div class=\"item\">")
|
||||
.skip(1)
|
||||
.map(|s| s.to_string());
|
||||
|
||||
let futures = raw_videos.map(|v| self.get_video_item(v, requester.clone()));
|
||||
let results = join_all(futures).await;
|
||||
|
||||
results.into_iter().filter_map(Result::ok).collect()
|
||||
.filter_map(|segment| self.get_video_item(segment.to_string()).ok())
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn get_video_item(
|
||||
&self,
|
||||
video_segment: String,
|
||||
requester: Requester,
|
||||
) -> Result<VideoItem> {
|
||||
fn proxy_url(&self, video_url: &str) -> String {
|
||||
let target = video_url
|
||||
.strip_prefix("https://")
|
||||
.or_else(|| video_url.strip_prefix("http://"))
|
||||
.unwrap_or(video_url)
|
||||
.trim_start_matches('/');
|
||||
|
||||
format!("https://hottub.spacemoehre.de/proxy/noodlemagazine/{target}")
|
||||
}
|
||||
|
||||
fn get_video_item(&self, video_segment: String) -> Result<VideoItem> {
|
||||
let href = video_segment
|
||||
.split("<a href=\"")
|
||||
.nth(1)
|
||||
@@ -217,54 +212,22 @@ impl NoodlemagazineProvider {
|
||||
.and_then(|s| s.split('<').next())
|
||||
.and_then(|v| parse_abbreviated_number(v.trim()))
|
||||
.unwrap_or(0);
|
||||
|
||||
let formats = self
|
||||
.extract_media(&video_url, requester)
|
||||
.await
|
||||
.ok_or_else(|| Error::from("media extraction failed"))?;
|
||||
let proxy_url = self.proxy_url(&video_url);
|
||||
|
||||
Ok(VideoItem::new(
|
||||
id,
|
||||
title,
|
||||
video_url,
|
||||
proxy_url.clone(),
|
||||
"noodlemagazine".into(),
|
||||
thumb,
|
||||
duration,
|
||||
)
|
||||
.views(views)
|
||||
.formats(formats))
|
||||
}
|
||||
|
||||
async fn extract_media(
|
||||
&self,
|
||||
video_url: &String,
|
||||
mut requester: Requester,
|
||||
) -> Option<Vec<VideoFormat>> {
|
||||
let text = requester
|
||||
.get(video_url, Some(Version::HTTP_2))
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
|
||||
let json_str = text.split("window.playlist = ").nth(1)?.split(';').next()?;
|
||||
|
||||
let json: serde_json::Value = serde_json::from_str(json_str).ok()?;
|
||||
let sources = json["sources"].as_array()?;
|
||||
|
||||
let mut formats = vec![];
|
||||
|
||||
for s in sources {
|
||||
let file = s["file"].as_str()?.to_string();
|
||||
let label = s["label"].as_str().unwrap_or("unknown").to_string();
|
||||
|
||||
formats.push(
|
||||
VideoFormat::new(file, label.clone(), "video/mp4".into())
|
||||
.format_id(label.clone())
|
||||
.format_note(label.clone())
|
||||
.http_header("Referer".into(), video_url.clone()),
|
||||
);
|
||||
}
|
||||
|
||||
Some(formats.into_iter().rev().collect())
|
||||
.formats(vec![
|
||||
VideoFormat::new(proxy_url, "auto".into(), "video/mp4".into())
|
||||
.format_id("auto".into())
|
||||
.format_note("proxied".into()),
|
||||
]))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -300,3 +263,44 @@ impl Provider for NoodlemagazineProvider {
|
||||
Some(self.build_channel(clientversion))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::NoodlemagazineProvider;
|
||||
|
||||
#[test]
|
||||
fn rewrites_video_pages_to_hottub_proxy() {
|
||||
let provider = NoodlemagazineProvider::new();
|
||||
|
||||
assert_eq!(
|
||||
provider.proxy_url("https://noodlemagazine.com/watch/-123_456"),
|
||||
"https://hottub.spacemoehre.de/proxy/noodlemagazine/noodlemagazine.com/watch/-123_456"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_listing_without_detail_page_requests() {
|
||||
let provider = NoodlemagazineProvider::new();
|
||||
let html = r#"
|
||||
<div class="list_videos" id="list_videos">
|
||||
<div class="item">
|
||||
<a href="/watch/-123_456">
|
||||
<img data-src="https://thumb.example/test.jpg" />
|
||||
</a>
|
||||
<div class="title">sample & title</div>
|
||||
<svg><use></use></svg>#clock-o"></use></svg>12:34<
|
||||
<svg><use></use></svg>#eye"></use></svg>1.2K<
|
||||
</div>
|
||||
>Show more</div>
|
||||
"#;
|
||||
|
||||
let items = provider.get_video_items_from_html(html.to_string());
|
||||
|
||||
assert_eq!(items.len(), 1);
|
||||
assert_eq!(
|
||||
items[0].url,
|
||||
"https://hottub.spacemoehre.de/proxy/noodlemagazine/noodlemagazine.com/watch/-123_456"
|
||||
);
|
||||
assert_eq!(items[0].formats.as_ref().map(|f| f.len()), Some(1));
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user