adjusted requester to supply http::version itself

This commit is contained in:
Simon
2025-11-30 06:53:21 +00:00
parent 2de6a7d42b
commit d856ade32b
30 changed files with 154 additions and 162 deletions

View File

@@ -81,7 +81,7 @@ impl BeegProvider {
async fn load_stars(stars: Arc<RwLock<Vec<FilterOption>>>) -> Result<()> {
let mut requester = util::requester::Requester::new();
let text = requester
.get("https://store.externulls.com/tag/facts/tags?get_original=true&slug=index")
.get("https://store.externulls.com/tag/facts/tags?get_original=true&slug=index", None)
.await
.unwrap();
let json: serde_json::Value = serde_json::from_str::<serde_json::Value>(&text).unwrap();
@@ -103,7 +103,7 @@ impl BeegProvider {
async fn load_categories(categories: Arc<RwLock<Vec<FilterOption>>>) -> Result<()> {
let mut requester = util::requester::Requester::new();
let text = requester
.get("https://store.externulls.com/tag/facts/tags?get_original=true&slug=index")
.get("https://store.externulls.com/tag/facts/tags?get_original=true&slug=index", None)
.await
.unwrap();
let json: serde_json::Value = serde_json::from_str::<serde_json::Value>(&text).unwrap();
@@ -125,7 +125,7 @@ impl BeegProvider {
async fn load_sites(sites: Arc<RwLock<Vec<FilterOption>>>) -> Result<()> {
let mut requester = util::requester::Requester::new();
let text = requester
.get("https://store.externulls.com/tag/facts/tags?get_original=true&slug=index")
.get("https://store.externulls.com/tag/facts/tags?get_original=true&slug=index", None)
.await
.unwrap();
let json: serde_json::Value = serde_json::from_str::<serde_json::Value>(&text).unwrap();
@@ -263,7 +263,7 @@ impl BeegProvider {
}
};
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap();
let text = requester.get(&video_url, None).await.unwrap();
let json: serde_json::Value = serde_json::from_str::<serde_json::Value>(&text).unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(json.clone());
if !video_items.is_empty() {
@@ -304,7 +304,7 @@ impl BeegProvider {
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap();
let text = requester.get(&video_url, None).await.unwrap();
let json: serde_json::Value = serde_json::from_str::<serde_json::Value>(&text).unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(json.clone());
if !video_items.is_empty() {

View File

@@ -56,7 +56,7 @@ impl FreshpornoProvider {
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap();
let text = requester.get(&video_url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);
@@ -93,7 +93,7 @@ impl FreshpornoProvider {
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap();
let text = requester.get(&video_url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);

View File

@@ -153,7 +153,7 @@ impl HanimeProvider {
let url = format!("https://h.freeanimehentai.net/api/v8/video?id={}&", hit.slug);
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&url).await.unwrap();
let text = requester.get(&url, None).await.unwrap();
let urls = text.split("\"servers\"").collect::<Vec<&str>>()[1];
let mut url_vec = vec![];

View File

@@ -64,7 +64,7 @@ impl HqpornerProvider {
async fn load_stars(base_url: &str, stars: Arc<RwLock<Vec<FilterOption>>>) -> Result<()> {
let mut requester = Requester::new();
let text = requester
.get(format!("{}/girls", &base_url).as_str())
.get(format!("{}/girls", &base_url).as_str(), None)
.await
.unwrap();
let stars_div = text
@@ -96,7 +96,7 @@ impl HqpornerProvider {
async fn load_categories(base_url: &str, categories: Arc<RwLock<Vec<FilterOption>>>) -> Result<()> {
let mut requester = Requester::new();
let text = requester
.get(format!("{}/categories", &base_url).as_str())
.get(format!("{}/categories", &base_url).as_str(), None)
.await
.unwrap();
let categories_div = text
@@ -176,7 +176,7 @@ impl HqpornerProvider {
};
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap();
let text = requester.get(&video_url, None).await.unwrap();
let video_items: Vec<VideoItem> = self
.get_video_items_from_html(text.clone(), &mut requester)
.await;
@@ -223,7 +223,7 @@ impl HqpornerProvider {
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap();
let text = requester.get(&video_url, None).await.unwrap();
let video_items: Vec<VideoItem> = self
.get_video_items_from_html(text.clone(), &mut requester)
.await;

View File

@@ -53,7 +53,7 @@ impl MissavProvider {
}
};
let text = requester.get(&url_str).await.unwrap();
let text = requester.get(&url_str, None).await.unwrap();
// Pass a reference to options if needed, or reconstruct as needed
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone(), pool, requester).await;
if !video_items.is_empty() {
@@ -89,7 +89,7 @@ impl MissavProvider {
vec![]
}
};
let text = requester.get(&url_str).await.unwrap();
let text = requester.get(&url_str, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone(), pool, requester).await;
if !video_items.is_empty() {
cache.remove(&url_str);
@@ -148,7 +148,7 @@ impl MissavProvider {
}
}
drop(conn);
let vid = requester.get(&url_str).await.unwrap();
let vid = requester.get(&url_str, None).await.unwrap();
let mut title = vid.split("<meta property=\"og:title\" content=\"").collect::<Vec<&str>>()[1]
.split("\"")
.collect::<Vec<&str>>()[0].trim()

View File

@@ -3,12 +3,15 @@ use crate::api::ClientVersion;
use crate::providers::Provider;
use crate::status::*;
use crate::util::cache::VideoCache;
use crate::util::requester::Requester;
use crate::util::parse_abbreviated_number;
use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoItem};
use crate::videos::{ServerOptions, VideoFormat, VideoItem};
use async_trait::async_trait;
use error_chain::error_chain;
use futures::future::join_all;
use htmlentity::entity::{ICodedDataTrait, decode};
use wreq::Version;
use std::{vec};
use titlecase::Titlecase;
@@ -76,9 +79,9 @@ impl NoodlemagazineProvider {
};
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap();
let text = requester.get(&video_url, Some(Version::HTTP_2)).await.unwrap();
let video_items: Vec<VideoItem> = self
.get_video_items_from_html(text.clone())
.get_video_items_from_html(text.clone(), requester.clone())
.await;
if !video_items.is_empty() {
cache.remove(&video_url);
@@ -116,9 +119,9 @@ impl NoodlemagazineProvider {
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap();
let text = requester.get(&video_url, Some(Version::HTTP_2)).await.unwrap();
let video_items: Vec<VideoItem> = self
.get_video_items_from_html(text.clone())
.get_video_items_from_html(text.clone(), requester.clone())
.await;
if !video_items.is_empty() {
cache.remove(&video_url);
@@ -132,6 +135,7 @@ impl NoodlemagazineProvider {
async fn get_video_items_from_html(
&self,
html: String,
requester: Requester,
) -> Vec<VideoItem> {
if html.is_empty() || html.contains("404 Not Found") {
return vec![];
@@ -143,24 +147,18 @@ impl NoodlemagazineProvider {
.collect::<Vec<&str>>()[1..]
.to_vec();
let mut video_items: Vec<VideoItem> = vec![];
for raw_video in raw_videos {
if let Ok(video_item) = self.get_video_item(raw_video.to_string()).await {
video_items.push(video_item);
}
}
// let futures = raw_videos
// .into_iter()
// .map(|el| self.get_video_item(el.to_string()));
// let results: Vec<Result<VideoItem>> = join_all(futures).await;
// let video_items: Vec<VideoItem> = results.into_iter().filter_map(Result::ok).collect();
let futures = raw_videos
.into_iter()
.map(|el| self.get_video_item(el.to_string(), requester.clone()));
let results: Vec<Result<VideoItem>> = join_all(futures).await;
let video_items: Vec<VideoItem> = results.into_iter().filter_map(Result::ok).collect();
return video_items;
}
async fn get_video_item(
&self,
video_segment: String,
requester: Requester,
) -> Result<VideoItem> {
let video_url: String = format!(
"{}{}",
@@ -208,22 +206,52 @@ impl NoodlemagazineProvider {
.collect::<Vec<&str>>()[0]
.trim()).unwrap_or(0);
let proxy_url = format!(
"https://hottub.spacemoehre.de/proxy/noodlemagazine/watch/{}",
video_url.split("/").collect::<Vec<&str>>()[4]
);
let formats = match self.extract_media(&video_url, requester.clone()).await{
Some(f) => f,
None => return Err(Error::from("Failed to extract media formats")),
};
let video_item = VideoItem::new(
id,
title,
proxy_url,
video_url,
"noodlemagazine".to_string(),
thumb,
duration,
)
.views(views)
.formats(formats)
;
return Ok(video_item);
}
async fn extract_media(&self, video_url: &String, mut requester: Requester) -> Option<Vec<VideoFormat>> {
let text = requester.get(video_url, Some(Version::HTTP_2)).await.unwrap_or("".to_string());
if text.is_empty() {
return None;
}
let mut formats = vec![];
let json_str = text.split("window.playlist = ")
.collect::<Vec<&str>>()[1]
.split(";")
.collect::<Vec<&str>>()[0];
let json: serde_json::Value = serde_json::from_str(json_str).unwrap();
let sources = json["sources"].as_array().unwrap();
for source in sources {
let file = source["file"].as_str().unwrap().to_string();
let label = source["label"].as_str().unwrap_or("unknown").to_string();
let format = VideoFormat::new(
file,
"video/mp4".to_string(),
label.clone(),
)
.format_id(label.clone())
.format_note(label.clone())
.http_header("Referer".to_string(), video_url.clone())
;
formats.push(format);
}
return Some(formats);
}
}
#[async_trait]

View File

@@ -84,7 +84,7 @@ impl OmgxxxProvider {
let mut requester = util::requester::Requester::new();
for page in [1..10].into_iter().flatten() {
let text = requester
.get(format!("{}/models/total-videos/{}/?gender_id=0", &base_url, page).as_str())
.get(format!("{}/models/total-videos/{}/?gender_id=0", &base_url, page).as_str(), None)
.await
.unwrap();
if text.contains("404 Not Found") || text.is_empty() {
@@ -122,7 +122,7 @@ impl OmgxxxProvider {
loop {
page += 1;
let text = requester
.get(format!("{}/sites/{}/", &base_url, page).as_str())
.get(format!("{}/sites/{}/", &base_url, page).as_str(), None)
.await
.unwrap();
if text.contains("404 Not Found") || text.is_empty() {
@@ -158,7 +158,7 @@ impl OmgxxxProvider {
async fn load_networks(base_url: &str, networks: Arc<RwLock<Vec<FilterOption>>>) -> Result<()> {
let mut requester = util::requester::Requester::new();
let text = requester.get(&base_url).await.unwrap();
let text = requester.get(&base_url, None).await.unwrap();
let networks_div = text.split("class=\"sites__list\"").collect::<Vec<&str>>()[1]
.split("</div>")
.collect::<Vec<&str>>()[0];
@@ -345,7 +345,7 @@ impl OmgxxxProvider {
};
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap();
let text = requester.get(&video_url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);
@@ -393,7 +393,7 @@ impl OmgxxxProvider {
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap();
let text = requester.get(&video_url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);

View File

@@ -51,7 +51,7 @@ impl ParadisehillProvider {
}
};
let text = requester.get(&url_str).await.unwrap();
let text = requester.get(&url_str, None).await.unwrap();
// Pass a reference to options if needed, or reconstruct as needed
let video_items: Vec<VideoItem> = self
.get_video_items_from_html(text.clone(), requester)
@@ -93,7 +93,7 @@ impl ParadisehillProvider {
vec![]
}
};
let text = requester.get(&url_str).await.unwrap();
let text = requester.get(&url_str, None).await.unwrap();
let video_items: Vec<VideoItem> = self
.get_video_items_from_html(text.clone(), requester)
.await;
@@ -143,7 +143,7 @@ impl ParadisehillProvider {
}
async fn get_video_item(&self, url_str: String, mut requester: Requester) -> Result<VideoItem> {
let vid = requester.get(&url_str).await.unwrap();
let vid = requester.get(&url_str, None).await.unwrap();
let mut title = vid
.split("<meta property=\"og:title\" content=\"")
.collect::<Vec<&str>>()[1]

View File

@@ -71,7 +71,7 @@ impl PerverzijaProvider {
};
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&url_str).await.unwrap();
let text = requester.get(&url_str, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone(), pool);
if !video_items.is_empty() {
cache.remove(&url_str);
@@ -122,7 +122,7 @@ impl PerverzijaProvider {
};
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&url_str).await.unwrap();
let text = requester.get(&url_str, None).await.unwrap();
let video_items: Vec<VideoItem> = match query_parse {
true => {
self.get_video_items_from_html_query(text.clone(), pool)

View File

@@ -175,7 +175,7 @@ impl PmvhavenProvider {
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap();
let text = requester.get(&video_url, None).await.unwrap();
let json = serde_json::from_str::<serde_json::Value>(&text).unwrap_or(serde_json::Value::Null);
let video_items: Vec<VideoItem> = self
.get_video_items_from_json(json)

View File

@@ -62,7 +62,7 @@ impl Porn00Provider {
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap();
let text = requester.get(&video_url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);
@@ -99,7 +99,7 @@ impl Porn00Provider {
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap();
let text = requester.get(&video_url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);

View File

@@ -53,7 +53,7 @@ impl PornhatProvider {
}
};
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap();
let text = requester.get(&video_url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);
@@ -92,7 +92,7 @@ impl PornhatProvider {
}
};
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap();
let text = requester.get(&video_url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);

View File

@@ -49,7 +49,7 @@ impl PornhubProvider {
};
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap();
let text = requester.get(&video_url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone(),"<ul id=\"video");
if !video_items.is_empty() {
cache.remove(&video_url);
@@ -116,7 +116,7 @@ impl PornhubProvider {
};
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap();
let text = requester.get(&video_url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone(),split_string);
if !video_items.is_empty() {
cache.remove(&video_url);

View File

@@ -96,7 +96,7 @@ impl PornxpProvider {
};
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap();
let text = requester.get(&video_url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);
@@ -142,7 +142,7 @@ impl PornxpProvider {
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap();
let text = requester.get(&video_url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);

View File

@@ -64,7 +64,7 @@ impl PornzogProvider {
let mut requester = options.requester.clone().unwrap();
println!("Fetching URL: {}", video_url);
let text = requester.get(&video_url).await.unwrap();
let text = requester.get(&video_url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);

View File

@@ -50,7 +50,7 @@ impl RedtubeProvider {
}
};
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap();
let text = requester.get(&video_url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);
@@ -87,7 +87,7 @@ impl RedtubeProvider {
}
};
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap();
let text = requester.get(&video_url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html_query(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);

View File

@@ -115,7 +115,7 @@ fn build_channel(&self, clientversion: ClientVersion) -> Channel {
};
}
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&url).await.unwrap();
let text = requester.get(&url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&url);
@@ -163,7 +163,7 @@ fn build_channel(&self, clientversion: ClientVersion) -> Channel {
}
};
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&url).await.unwrap();
let text = requester.get(&url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&url);

View File

@@ -67,7 +67,7 @@ impl Rule34videoProvider {
};
}
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&url).await.unwrap();
let text = requester.get(&url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&url);
@@ -115,7 +115,7 @@ impl Rule34videoProvider {
}
};
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&url).await.unwrap();
let text = requester.get(&url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&url);

View File

@@ -81,7 +81,7 @@ impl SxyprnProvider {
}
};
let text = requester.get(&url_str).await.unwrap();
let text = requester.get(&url_str, None).await.unwrap();
// Pass a reference to options if needed, or reconstruct as needed
let video_items: Vec<VideoItem> = self
.get_video_items_from_html(text.clone(), pool, requester)
@@ -131,7 +131,7 @@ impl SxyprnProvider {
vec![]
}
};
let text = requester.get(&url_str).await.unwrap();
let text = requester.get(&url_str, None).await.unwrap();
let video_items: Vec<VideoItem> = self
.get_video_items_from_html(text.clone(), pool, requester)
.await;

View File

@@ -380,7 +380,7 @@ impl TnaflixProvider {
};
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap();
let text = requester.get(&video_url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);
@@ -427,7 +427,7 @@ impl TnaflixProvider {
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap();
let text = requester.get(&video_url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);

View File

@@ -106,7 +106,7 @@ impl XxdbxProvider {
};
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap();
let text = requester.get(&video_url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);
@@ -156,7 +156,7 @@ impl XxdbxProvider {
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap();
let text = requester.get(&video_url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);

View File

@@ -62,7 +62,7 @@ impl XxthotsProvider {
}
};
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap();
let text = requester.get(&video_url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);
@@ -99,7 +99,7 @@ impl XxthotsProvider {
}
};
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap();
let text = requester.get(&video_url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);

View File

@@ -59,7 +59,7 @@ impl YoujizzProvider {
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap();
let text = requester.get(&video_url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);
@@ -95,7 +95,7 @@ impl YoujizzProvider {
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url).await.unwrap();
let text = requester.get(&video_url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);