simon/hottub
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

provider refactors and fixes

Browse Source
This commit is contained in:
Simon
2026-03-05 13:28:38 +00:00
parent 060d8e7937
commit 8157e223fe
33 changed files with 3051 additions and 1694 deletions
Download Patch File Download Diff File Expand all files Collapse all files

939
src/api.rs
View File

File diff suppressed because it is too large Load Diff

98
src/providers/all.rs
View File

@@ -1,12 +1,14 @@
use std::fs;
use std::time::Duration;
use async_trait::async_trait;
use capitalize::Capitalize;
use cute::c;
use error_chain::error_chain;
use futures::StreamExt;
use futures::stream::FuturesUnordered;
use crate::api::{get_provider, ClientVersion};
use crate::providers::{DynProvider, Provider};
use crate::status::Channel;
use crate::providers::{DynProvider, Provider, report_provider_error, run_provider_guarded};
use crate::status::{Channel, ChannelOption, FilterOption};
use crate::util::cache::VideoCache;
use crate::util::interleave;
use crate::videos::{ServerOptions, VideoItem};
@@ -45,24 +47,50 @@ impl Provider for AllProvider {
) -> Vec<VideoItem> {
let mut sites_str = options.clone().sites.unwrap_or_default();
if sites_str.is_empty() {
let files = fs::read_dir("./src/providers").unwrap();
let providers = files.map(|entry| entry.unwrap().file_name())
.filter(|name| name.to_str().unwrap().ends_with(".rs"))
.filter(|name| !name.to_str().unwrap().contains("mod.rs") && !name.to_str().unwrap().contains("all.rs"))
.map(|name| name.to_str().unwrap().replace(".rs", ""))
let files = match fs::read_dir("./src/providers") {
Ok(files) => files,
Err(e) => {
report_provider_error("all", "all.get_videos.read_dir", &e.to_string()).await;
return vec![];
}
};
let providers = files
.filter_map(|entry| entry.ok())
.filter_map(|entry| entry.file_name().into_string().ok())
.filter(|name| name.ends_with(".rs"))
.filter(|name| !name.contains("mod.rs") && !name.contains("all.rs"))
.map(|name| name.replace(".rs", ""))
.collect::<Vec<String>>();
sites_str = providers.join(",");
}
let providers: Vec<DynProvider> = sites_str
let providers: Vec<(String, DynProvider)> = sites_str
.split(',')
.map(str::trim)
.filter(|s| !s.is_empty())
.filter_map(|s| get_provider(s))
.filter_map(|s| {
let provider = get_provider(s);
if provider.is_none() {
Some((s.to_string(), None))
} else {
provider.map(|p| (s.to_string(), Some(p)))
}
})
.filter_map(|(name, provider)| match provider {
Some(provider) => Some((name, provider)),
None => {
// fire-and-forget reporting of missing provider keys
tokio::spawn(async move {
report_provider_error("all", "all.get_videos.unknown_provider", &name).await;
});
None
}
})
.collect();
let mut futures = FuturesUnordered::new();
for provider in providers {
for (provider_name, provider) in providers {
let cache = cache.clone();
let pool = pool.clone();
let sort = sort.clone();
@@ -70,10 +98,16 @@ impl Provider for AllProvider {
let page = page.clone();
let per_page = per_page.clone();
let options = options.clone();
let provider_name_cloned = provider_name.clone();
// Spawn the task so it lives independently of this function
futures.push(tokio::spawn(async move {
provider.get_videos(cache, pool, sort, query, page, per_page, options).await
run_provider_guarded(
&provider_name_cloned,
"all.get_videos.provider_task",
provider.get_videos(cache, pool, sort, query, page, per_page, options),
)
.await
}));
}
@@ -85,9 +119,11 @@ impl Provider for AllProvider {
loop {
tokio::select! {
Some(result) = futures.next() => {
// Ignore errors (panics or task cancellations)
if let Ok(videos) = result {
all_results.push(videos);
match result {
Ok(videos) => all_results.push(videos),
Err(e) => {
report_provider_error("all", "all.get_videos.join_error", &e.to_string()).await;
}
}
},
_ = &mut timeout_timer => {
@@ -105,17 +141,41 @@ impl Provider for AllProvider {
fn get_channel(&self, clientversion: ClientVersion) -> Option<Channel> {
let _ = clientversion;
let files = fs::read_dir("./src/providers").ok()?;
let providers = files
.filter_map(|entry| entry.ok())
.filter_map(|entry| entry.file_name().into_string().ok())
.filter(|name| name.ends_with(".rs"))
.filter(|name| {
!name.contains("mod.rs")
&& !name.contains("all.rs")
})
.map(|name| name.replace(".rs", ""))
.collect::<Vec<String>>();
let sites = c![FilterOption {
id: x.to_string(),
title: x.capitalize().to_string(),
}, for x in providers.iter()];
Some(Channel {
id: "placeholder".to_string(),
name: "PLACEHOLDER".to_string(),
description: "PLACEHOLDER FOR PARENT CLASS".to_string(),
id: "all".to_string(),
name: "All".to_string(),
description: "Query from all sites of this Server".to_string(),
premium: false,
favicon: "https://hottub.spacemoehre.de/favicon.ico".to_string(),
status: "active".to_string(),
categories: vec![],
options: vec![],
options: vec![ChannelOption {
id: "sites".to_string(),
title: "Sites".to_string(),
description: "What Sites to use".to_string(),
systemImage: "list.number".to_string(),
colorName: "green".to_string(),
options: sites,
multiSelect: true,
}],
nsfw: true,
cacheDuration: None,
cacheDuration: Some(1800),
})
}
}

65
src/providers/beeg.rs
View File

@@ -1,6 +1,6 @@
use crate::DbPool;
use crate::api::ClientVersion;
use crate::providers::Provider;
use crate::providers::{Provider, report_provider_error_background};
use crate::util::cache::VideoCache;
use crate::util::parse_abbreviated_number;
use crate::videos::{ServerOptions, VideoItem};
@@ -203,23 +203,20 @@ impl BeegProvider {
options: ServerOptions,
) -> Result<Vec<VideoItem>> {
let mut slug = "";
if options.categories.is_some()
&& !options.categories.as_ref().unwrap().is_empty()
&& options.categories.as_ref().unwrap() != "all"
{
slug = options.categories.as_ref().unwrap();
if let Some(categories) = options.categories.as_ref() {
if !categories.is_empty() && categories != "all" {
slug = categories;
}
}
if options.sites.is_some()
&& !options.sites.as_ref().unwrap().is_empty()
&& options.sites.as_ref().unwrap() != "all"
{
slug = options.sites.as_ref().unwrap();
if let Some(sites) = options.sites.as_ref() {
if !sites.is_empty() && sites != "all" {
slug = sites;
}
}
if options.stars.is_some()
&& !options.stars.as_ref().unwrap().is_empty()
&& options.stars.as_ref().unwrap() != "all"
{
slug = options.stars.as_ref().unwrap();
if let Some(stars) = options.stars.as_ref() {
if !stars.is_empty() && stars != "all" {
slug = stars;
}
}
let video_url = format!(
"https://store.externulls.com/facts/tag?limit=100&offset={}{}",
@@ -240,9 +237,21 @@ impl BeegProvider {
vec![]
}
};
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url, None).await.unwrap();
let json: serde_json::Value = serde_json::from_str::<serde_json::Value>(&text).unwrap();
let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let text = match requester.get(&video_url, None).await {
Ok(text) => text,
Err(e) => {
report_provider_error_background("beeg", "get.request", &e.to_string());
return Ok(old_items);
}
};
let json: serde_json::Value = match serde_json::from_str::<serde_json::Value>(&text) {
Ok(json) => json,
Err(e) => {
report_provider_error_background("beeg", "get.parse_json", &e.to_string());
return Ok(old_items);
}
};
let video_items: Vec<VideoItem> = self.get_video_items_from_html(json.clone());
if !video_items.is_empty() {
cache.remove(&video_url);
@@ -280,10 +289,22 @@ impl BeegProvider {
}
};
let mut requester = options.requester.clone().unwrap();
let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let text = requester.get(&video_url, None).await.unwrap();
let json: serde_json::Value = serde_json::from_str::<serde_json::Value>(&text).unwrap();
let text = match requester.get(&video_url, None).await {
Ok(text) => text,
Err(e) => {
report_provider_error_background("beeg", "query.request", &e.to_string());
return Ok(old_items);
}
};
let json: serde_json::Value = match serde_json::from_str::<serde_json::Value>(&text) {
Ok(json) => json,
Err(e) => {
report_provider_error_background("beeg", "query.parse_json", &e.to_string());
return Ok(old_items);
}
};
let video_items: Vec<VideoItem> = self.get_video_items_from_html(json.clone());
if !video_items.is_empty() {
cache.remove(&video_url);

96
src/providers/chaturbate.rs
View File

@@ -1,6 +1,6 @@
use crate::DbPool;
use crate::api::ClientVersion;
use crate::providers::Provider;
use crate::providers::{Provider, report_provider_error};
use crate::status::*;
use crate::util::cache::VideoCache;
use crate::videos::{ServerOptions, VideoItem};
@@ -89,17 +89,38 @@ impl ChaturbateProvider {
}
};
let mut requester = options.requester.clone().unwrap();
let text = requester
let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let response = match requester
.get_raw_with_headers(
&video_url,
vec![("X-Requested-With".to_string(), "XMLHttpRequest".to_string())],
)
.await
.unwrap()
.text()
.await
.unwrap();
{
Ok(response) => response,
Err(e) => {
report_provider_error(
"chaturbate",
"get.request",
&format!("url={video_url}; error={e}"),
)
.await;
return Ok(old_items);
}
};
let text = match response.text().await {
Ok(text) => text,
Err(e) => {
report_provider_error(
"chaturbate",
"get.response_text",
&format!("url={video_url}; error={e}"),
)
.await;
return Ok(old_items);
}
};
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);
@@ -139,18 +160,38 @@ impl ChaturbateProvider {
}
};
let mut requester = options.requester.clone().unwrap();
let text = requester
let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let response = match requester
.get_raw_with_headers(
&video_url,
vec![("X-Requested-With".to_string(), "XMLHttpRequest".to_string())],
)
.await
.unwrap()
.text()
.await
.unwrap();
{
Ok(response) => response,
Err(e) => {
report_provider_error(
"chaturbate",
"query.request",
&format!("url={video_url}; error={e}"),
)
.await;
return Ok(old_items);
}
};
let text = match response.text().await {
Ok(text) => text,
Err(e) => {
report_provider_error(
"chaturbate",
"query.response_text",
&format!("url={video_url}; error={e}"),
)
.await;
return Ok(old_items);
}
};
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);
@@ -171,7 +212,18 @@ impl ChaturbateProvider {
println!("Failed to parse JSON: {}", e);
serde_json::Value::Null
});
for video_segment in json.get("rooms").unwrap().as_array().unwrap_or(&vec![]) {
let rooms = match json.get("rooms").and_then(|v| v.as_array()) {
Some(rooms) => rooms,
None => {
crate::providers::report_provider_error_background(
"chaturbate",
"get_video_items_from_html.rooms_missing",
"missing rooms array",
);
return items;
}
};
for video_segment in rooms {
if video_segment
.get("has_password")
.unwrap_or(&serde_json::Value::Bool(false))
@@ -184,10 +236,18 @@ impl ChaturbateProvider {
// for (index, line) in vid.iter().enumerate() {
// println!("Line {}: {}", index, line);
// }
let username = video_segment
let Some(username) = video_segment
.get("username")
.and_then(|v| v.as_str())
.map(String::from).unwrap();
.map(String::from)
else {
crate::providers::report_provider_error_background(
"chaturbate",
"get_video_items_from_html.username_missing",
"missing username field",
);
continue;
};
let video_url: String = format!("{}/{}/", self.url, username);
let mut title = video_segment
.get("room_subject")
@@ -202,7 +262,7 @@ impl ChaturbateProvider {
.unwrap_or(&serde_json::Value::String("".to_string()))
.as_str()
.unwrap_or("")
.split("?").collect::<Vec<&str>>()[0]
.split("?").collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
let views = video_segment
.get("viewers")

294
src/providers/freepornvideosxxx.rs
View File

@@ -1,6 +1,6 @@
use crate::DbPool;
use crate::api::ClientVersion;
use crate::providers::Provider;
use crate::providers::{Provider, report_provider_error, report_provider_error_background};
use crate::util::cache::VideoCache;
use crate::util::parse_abbreviated_number;
use crate::util::time::parse_time_to_seconds;
@@ -58,10 +58,20 @@ impl FreepornvideosxxxProvider {
thread::spawn(move || {
// Create a tiny runtime just for these async tasks
let rt = tokio::runtime::Builder::new_current_thread()
let rt = match tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("build tokio runtime");
{
Ok(rt) => rt,
Err(e) => {
report_provider_error_background(
"freepornvideosxxx",
"spawn_initial_load.runtime_build",
&e.to_string(),
);
return;
}
};
rt.block_on(async move {
// If you have a streaming sites loader, call it here too
@@ -83,13 +93,23 @@ impl FreepornvideosxxxProvider {
async fn load_stars(base_url: &str, stars: Arc<RwLock<Vec<FilterOption>>>) -> Result<()> {
let mut requester = util::requester::Requester::new();
for page in [1..10].into_iter().flatten() {
let text = requester
let text = match requester
.get(
format!("{}/models/total-videos/{}/?gender_id=0", &base_url, page).as_str(),
None,
)
.await
.unwrap();
{
Ok(text) => text,
Err(e) => {
report_provider_error_background(
"freepornvideosxxx",
"load_stars.request",
&format!("url={base_url}; page={page}; error={e}"),
);
break;
}
};
if text.contains("404 Not Found") || text.is_empty() {
break;
}
@@ -97,19 +117,20 @@ impl FreepornvideosxxxProvider {
.split("<div class=\"list-models\">")
.collect::<Vec<&str>>()
.last()
.unwrap()
.copied()
.unwrap_or_default()
.split("custom_list_models_models_list_pagination")
.collect::<Vec<&str>>()[0];
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default();
for stars_element in stars_div.split("<a ").collect::<Vec<&str>>()[1..].to_vec() {
let star_url = stars_element.split("href=\"").collect::<Vec<&str>>()[1]
let star_url = stars_element.split("href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0];
let star_id = star_url.split("/").collect::<Vec<&str>>()[4].to_string();
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default();
let star_id = star_url.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string();
let star_name = stars_element
.split("<strong class=\"title\">")
.collect::<Vec<&str>>()[1]
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
Self::push_unique(
&stars,
@@ -130,26 +151,36 @@ impl FreepornvideosxxxProvider {
page += 1;
let text = requester
.get(format!("{}/sites/{}/", &base_url, page).as_str(), None)
.await
.unwrap();
.await;
let text = match text {
Ok(text) => text,
Err(e) => {
report_provider_error_background(
"freepornvideosxxx",
"load_sites.request",
&format!("url={base_url}; page={page}; error={e}"),
);
break;
}
};
if text.contains("404 Not Found") || text.is_empty() {
break;
}
let sites_div = text
.split("id=\"list_content_sources_sponsors_list_items\"")
.collect::<Vec<&str>>()[1]
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("class=\"pagination\"")
.collect::<Vec<&str>>()[0];
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default();
for sites_element in
sites_div.split("class=\"headline\"").collect::<Vec<&str>>()[1..].to_vec()
{
let site_url = sites_element.split("href=\"").collect::<Vec<&str>>()[1]
let site_url = sites_element.split("href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0];
let site_id = site_url.split("/").collect::<Vec<&str>>()[4].to_string();
let site_name = sites_element.split("<h2>").collect::<Vec<&str>>()[1]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default();
let site_id = site_url.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string();
let site_name = sites_element.split("<h2>").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
Self::push_unique(
&sites,
@@ -165,23 +196,33 @@ impl FreepornvideosxxxProvider {
async fn load_networks(base_url: &str, networks: Arc<RwLock<Vec<FilterOption>>>) -> Result<()> {
let mut requester = util::requester::Requester::new();
let text = requester.get(&base_url, None).await.unwrap();
let networks_div = text.split("class=\"sites__list\"").collect::<Vec<&str>>()[1]
let text = match requester.get(&base_url, None).await {
Ok(text) => text,
Err(e) => {
report_provider_error_background(
"freepornvideosxxx",
"load_networks.request",
&format!("url={base_url}; error={e}"),
);
return Ok(());
}
};
let networks_div = text.split("class=\"sites__list\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("</div>")
.collect::<Vec<&str>>()[0];
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default();
for network_element in
networks_div.split("sites__item").collect::<Vec<&str>>()[1..].to_vec()
{
if network_element.contains("sites__all") {
continue;
}
let network_url = network_element.split("href=\"").collect::<Vec<&str>>()[1]
let network_url = network_element.split("href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0];
let network_id = network_url.split("/").collect::<Vec<&str>>()[4].to_string();
let network_name = network_element.split(">").collect::<Vec<&str>>()[1]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default();
let network_id = network_url.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string();
let network_name = network_element.split(">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
Self::push_unique(
&networks,
@@ -306,35 +347,20 @@ impl FreepornvideosxxxProvider {
"most-popular" => "/most-popular".to_string(),
_ => "".to_string(),
};
if options.network.is_some()
&& !options.network.as_ref().unwrap().is_empty()
&& options.network.as_ref().unwrap() != "all"
{
sort_string = format!(
"networks/{}{}",
options.network.as_ref().unwrap(),
alt_sort_string
);
if let Some(network) = options.network.as_deref() {
if !network.is_empty() && network != "all" {
sort_string = format!("networks/{}{}", network, alt_sort_string);
}
}
if options.sites.is_some()
&& !options.sites.as_ref().unwrap().is_empty()
&& options.sites.as_ref().unwrap() != "all"
{
sort_string = format!(
"sites/{}{}",
options.sites.as_ref().unwrap(),
alt_sort_string
);
if let Some(site) = options.sites.as_deref() {
if !site.is_empty() && site != "all" {
sort_string = format!("sites/{}{}", site, alt_sort_string);
}
}
if options.stars.is_some()
&& !options.stars.as_ref().unwrap().is_empty()
&& options.stars.as_ref().unwrap() != "all"
{
sort_string = format!(
"models/{}{}",
options.stars.as_ref().unwrap(),
alt_sort_string
);
if let Some(star) = options.stars.as_deref() {
if !star.is_empty() && star != "all" {
sort_string = format!("models/{}{}", star, alt_sort_string);
}
}
let video_url = format!("{}/{}/{}/", self.url, sort_string, page);
let old_items = match cache.get(&video_url) {
@@ -350,8 +376,20 @@ impl FreepornvideosxxxProvider {
}
};
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url, None).await.unwrap();
let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let text = match requester.get(&video_url, None).await {
Ok(text) => text,
Err(e) => {
report_provider_error(
"freepornvideosxxx",
"get.request",
&format!("url={video_url}; error={e}"),
)
.await;
return Ok(old_items);
}
};
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);
@@ -371,31 +409,41 @@ impl FreepornvideosxxxProvider {
) -> Result<Vec<VideoItem>> {
let mut search_type = "search";
let mut search_string = query.to_string().to_ascii_lowercase().trim().to_string();
match self
.stars
.read()
.unwrap()
.iter()
.find(|s| s.title.to_ascii_lowercase() == search_string)
{
Some(star) => {
search_type = "models";
search_string = star.id.clone();
match self.stars.read() {
Ok(stars) => {
if let Some(star) = stars
.iter()
.find(|s| s.title.to_ascii_lowercase() == search_string)
{
search_type = "models";
search_string = star.id.clone();
}
}
Err(e) => {
report_provider_error_background(
"freepornvideosxxx",
"query.stars_read",
&e.to_string(),
);
}
_ => {}
}
match self
.sites
.read()
.unwrap()
.iter()
.find(|s| s.title.to_ascii_lowercase() == search_string)
{
Some(site) => {
search_type = "sites";
search_string = site.id.clone();
match self.sites.read() {
Ok(sites) => {
if let Some(site) = sites
.iter()
.find(|s| s.title.to_ascii_lowercase() == search_string)
{
search_type = "sites";
search_string = site.id.clone();
}
}
Err(e) => {
report_provider_error_background(
"freepornvideosxxx",
"query.sites_read",
&e.to_string(),
);
}
_ => {}
}
let mut video_url = format!("{}/{}/{}/{}/", self.url, search_type, search_string, page);
video_url = video_url.replace(" ", "+");
@@ -414,9 +462,20 @@ impl FreepornvideosxxxProvider {
}
};
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url, None).await.unwrap();
let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let text = match requester.get(&video_url, None).await {
Ok(text) => text,
Err(e) => {
report_provider_error(
"freepornvideosxxx",
"query.request",
&format!("url={video_url}; error={e}"),
)
.await;
return Ok(old_items);
}
};
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);
@@ -429,7 +488,18 @@ impl FreepornvideosxxxProvider {
fn get_site_id_from_name(&self, site_name: &str) -> Option<String> {
// site_name.to_lowercase().replace(" ", "")
for site in self.sites.read().unwrap().iter() {
let sites_guard = match self.sites.read() {
Ok(guard) => guard,
Err(e) => {
report_provider_error_background(
"freepornvideosxxx",
"get_site_id_from_name.sites_read",
&e.to_string(),
);
return None;
}
};
for site in sites_guard.iter() {
if site
.title
.to_lowercase()
@@ -452,11 +522,11 @@ impl FreepornvideosxxxProvider {
if !html.contains("class=\"item\"") {
return items;
}
let raw_videos = html.split("videos_list_pagination").collect::<Vec<&str>>()[0]
let raw_videos = html.split("videos_list_pagination").collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.split(" class=\"pagination\" ")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.split("class=\"list-videos\"")
.collect::<Vec<&str>>()[1]
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("class=\"item\"")
.collect::<Vec<&str>>()[1..]
.to_vec();
@@ -465,39 +535,39 @@ impl FreepornvideosxxxProvider {
// for (index, line) in vid.iter().enumerate() {
// println!("Line {}: {}", index, line);
// }
let video_url: String = video_segment.split("<a href=\"").collect::<Vec<&str>>()[1]
let video_url: String = video_segment.split("<a href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
let mut title = video_segment.split(" title=\"").collect::<Vec<&str>>()[1]
let mut title = video_segment.split(" title=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
// html decode
title = decode(title.as_bytes()).to_string().unwrap_or(title);
let id = video_url.split("/").collect::<Vec<&str>>()[4].to_string();
let id = video_url.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string();
let thumb = match video_segment.split("<img ").collect::<Vec<&str>>()[1]
let thumb = match video_segment.split("<img ").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.contains("data-src=\"")
{
true => video_segment.split("<img ").collect::<Vec<&str>>()[1]
true => video_segment.split("<img ").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("data-src=\"")
.collect::<Vec<&str>>()[1]
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string(),
false => video_segment.split("<img ").collect::<Vec<&str>>()[1]
false => video_segment.split("<img ").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("src=\"")
.collect::<Vec<&str>>()[1]
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string(),
};
let raw_duration = video_segment
.split("<span class=\"duration\">")
.collect::<Vec<&str>>()[1]
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.split(" ")
.collect::<Vec<&str>>()
.last()
@@ -507,9 +577,9 @@ impl FreepornvideosxxxProvider {
let views = parse_abbreviated_number(
video_segment
.split("<div class=\"views\">")
.collect::<Vec<&str>>()[1]
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string()
.as_str(),
)
@@ -517,9 +587,9 @@ impl FreepornvideosxxxProvider {
let preview = video_segment
.split("data-preview=\"")
.collect::<Vec<&str>>()[1]
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
let site_name = title
.split("]")
@@ -533,9 +603,9 @@ impl FreepornvideosxxxProvider {
let mut tags = match video_segment.contains("class=\"models\">") {
true => video_segment
.split("class=\"models\">")
.collect::<Vec<&str>>()[1]
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("</div>")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.split("href=\"")
.collect::<Vec<&str>>()[1..]
.into_iter()
@@ -543,17 +613,17 @@ impl FreepornvideosxxxProvider {
Self::push_unique(
&self.stars,
FilterOption {
id: s.split("/").collect::<Vec<&str>>()[4].to_string(),
title: s.split(">").collect::<Vec<&str>>()[1]
id: s.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string(),
title: s.split(">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.trim()
.to_string(),
},
);
s.split(">").collect::<Vec<&str>>()[1]
s.split(">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.trim()
.to_string()
})

190
src/providers/hanime.rs
View File

@@ -4,9 +4,11 @@ use futures::future::join_all;
use serde_json::json;
use std::vec;
use crate::api::ClientVersion;
use crate::DbPool;
use crate::db;
use crate::providers::Provider;
use crate::providers::{Provider, report_provider_error, report_provider_error_background};
use crate::status::*;
use crate::util::cache::VideoCache;
use crate::videos::{self, ServerOptions, VideoItem};
@@ -124,13 +126,83 @@ impl HanimeProvider {
}
}
fn build_channel(&self, _clientversion: ClientVersion) -> Channel {
Channel {
id: "hanime".to_string(),
name: "Hanime".to_string(),
description: "Free Hentai from Hanime".to_string(),
premium: false,
favicon: "https://www.google.com/s2/favicons?sz=64&domain=hanime.tv".to_string(),
status: "active".to_string(),
categories: vec![],
options: vec![ChannelOption {
id: "sort".to_string(),
title: "Sort".to_string(),
description: "Sort the Videos".to_string(),
systemImage: "list.number".to_string(),
colorName: "blue".to_string(),
options: vec![
FilterOption {
id: "created_at_unix.desc".to_string(),
title: "Recent Upload".to_string(),
},
FilterOption {
id: "created_at_unix.asc".to_string(),
title: "Old Upload".to_string(),
},
FilterOption {
id: "views.desc".to_string(),
title: "Most Views".to_string(),
},
FilterOption {
id: "views.asc".to_string(),
title: "Least Views".to_string(),
},
FilterOption {
id: "likes.desc".to_string(),
title: "Most Likes".to_string(),
},
FilterOption {
id: "likes.asc".to_string(),
title: "Least Likes".to_string(),
},
FilterOption {
id: "released_at_unix.desc".to_string(),
title: "New".to_string(),
},
FilterOption {
id: "released_at_unix.asc".to_string(),
title: "Old".to_string(),
},
FilterOption {
id: "title_sortable.asc".to_string(),
title: "A - Z".to_string(),
},
FilterOption {
id: "title_sortable.desc".to_string(),
title: "Z - A".to_string(),
},
],
multiSelect: false,
}],
nsfw: true,
cacheDuration: None,
}
}
async fn get_video_item(
&self,
hit: HanimeSearchResult,
pool: DbPool,
options: ServerOptions,
) -> Result<VideoItem> {
let mut conn = pool.get().expect("couldn't get db connection from pool");
let mut conn = match pool.get() {
Ok(conn) => conn,
Err(e) => {
report_provider_error("hanime", "get_video_item.pool_get", &e.to_string()).await;
return Err(Error::from("Failed to get DB connection"));
}
};
let db_result = db::get_video(
&mut conn,
format!(
@@ -169,13 +241,24 @@ impl HanimeProvider {
"m3u8".to_string(),
)]));
} else {
let _ = db::delete_video(
&mut pool.get().expect("couldn't get db connection from pool"),
format!(
"https://h.freeanimehentai.net/api/v8/video?id={}&",
hit.slug.clone()
),
);
match pool.get() {
Ok(mut conn) => {
let _ = db::delete_video(
&mut conn,
format!(
"https://h.freeanimehentai.net/api/v8/video?id={}&",
hit.slug.clone()
),
);
}
Err(e) => {
report_provider_error_background(
"hanime",
"get_video_item.delete_video.pool_get",
&e.to_string(),
);
}
}
}
}
Ok(None) => (),
@@ -189,7 +272,7 @@ impl HanimeProvider {
id
);
let mut requester = options.requester.clone().unwrap();
let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let payload = json!({
"width": 571, "height": 703, "ab": "kh" }
);
@@ -216,41 +299,71 @@ impl HanimeProvider {
],
)
.await
.unwrap()
.map_err(|e| {
report_provider_error_background(
"hanime",
"get_video_item.get_raw_with_headers",
&e.to_string(),
);
Error::from(format!("Failed to fetch manifest response: {e}"))
})?
.text()
.await
.unwrap();
.map_err(|e| {
report_provider_error_background(
"hanime",
"get_video_item.response_text",
&e.to_string(),
);
Error::from(format!("Failed to decode manifest response body: {e}"))
})?;
if text.contains("Unautho") {
println!("Fetched video details for {}: {}", title, text);
return Err(Error::from("Unauthorized"));
}
let urls = text.split("streams").collect::<Vec<&str>>()[1];
let urls = text
.split("streams")
.nth(1)
.ok_or_else(|| Error::from("Missing streams section in manifest"))?;
let mut url_vec = vec![];
for el in urls.split("\"url\":\"").collect::<Vec<&str>>() {
let url = el.split("\"").collect::<Vec<&str>>()[0];
let url = el.split("\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default();
if !url.is_empty() && url.contains("m3u8") {
url_vec.push(url.to_string());
}
}
let mut conn = pool.get().expect("couldn't get db connection from pool");
let _ = db::insert_video(
&mut conn,
&format!(
"https://h.freeanimehentai.net/api/v8/video?id={}&",
hit.slug.clone()
),
&url_vec[0].clone(),
);
drop(conn);
let first_url = url_vec
.first()
.cloned()
.ok_or_else(|| Error::from("No stream URL found in manifest"))?;
match pool.get() {
Ok(mut conn) => {
let _ = db::insert_video(
&mut conn,
&format!(
"https://h.freeanimehentai.net/api/v8/video?id={}&",
hit.slug.clone()
),
&first_url,
);
}
Err(e) => {
report_provider_error_background(
"hanime",
"get_video_item.insert_video.pool_get",
&e.to_string(),
);
}
}
Ok(
VideoItem::new(id, title, url_vec[0].clone(), channel, thumb, duration)
VideoItem::new(id, title, first_url.clone(), channel, thumb, duration)
.tags(hit.tags)
.uploader(hit.brand)
.views(hit.views as u32)
.rating((hit.likes as f32 / (hit.likes + hit.dislikes) as f32) * 100 as f32)
.formats(vec![videos::VideoFormat::new(
url_vec[0].clone(),
first_url,
"1080".to_string(),
"m3u8".to_string(),
)]),
@@ -268,11 +381,11 @@ impl HanimeProvider {
) -> Result<Vec<VideoItem>> {
let index = format!("hanime:{}:{}:{}", query, page, sort);
let order_by = match sort.contains(".") {
true => sort.split(".").collect::<Vec<&str>>()[0].to_string(),
true => sort.split(".").collect::<Vec<&str>>().get(0).copied().unwrap_or_default().to_string(),
false => "created_at_unix".to_string(),
};
let ordering = match sort.contains(".") {
true => sort.split(".").collect::<Vec<&str>>()[1].to_string(),
true => sort.split(".").collect::<Vec<&str>>().get(1).copied().unwrap_or_default().to_string(),
false => "desc".to_string(),
};
let old_items = match cache.get(&index) {
@@ -295,11 +408,22 @@ impl HanimeProvider {
.order_by(order_by)
.ordering(ordering);
let mut requester = options.requester.clone().unwrap();
let response = requester
let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let response = match requester
.post_json("https://search.htv-services.com/search", &search, vec![])
.await
.unwrap();
{
Ok(response) => response,
Err(e) => {
report_provider_error(
"hanime",
"get.search_request",
&format!("query={query}; page={page}; error={e}"),
)
.await;
return Ok(old_items);
}
};
let hits = match response.json::<HanimeSearchResponse>().await {
Ok(resp) => resp.hits,
@@ -374,4 +498,8 @@ impl Provider for HanimeProvider {
}
}
}
fn get_channel(&self, clientversion: ClientVersion) -> Option<Channel> {
Some(self.build_channel(clientversion))
}
}

97
src/providers/hentaihaven.rs
View File

@@ -57,10 +57,15 @@ impl HentaihavenProvider {
categories: self
.categories
.read()
.unwrap()
.iter()
.map(|c| c.title.clone())
.collect(),
.map(|categories| categories.iter().map(|c| c.title.clone()).collect())
.unwrap_or_else(|e| {
crate::providers::report_provider_error_background(
"hentaihaven",
"build_channel.categories_read",
&e.to_string(),
);
vec![]
}),
options: vec![],
nsfw: true,
cacheDuration: None,
@@ -98,11 +103,20 @@ impl HentaihavenProvider {
}
};
let mut requester = options.requester.clone().unwrap();
let text = requester
.get(&video_url, Some(Version::HTTP_2))
.await
.unwrap();
let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let text = match requester.get(&video_url, Some(Version::HTTP_2)).await {
Ok(text) => text,
Err(e) => {
crate::providers::report_provider_error(
"hentaihaven",
"get.request",
&format!("url={video_url}; error={e}"),
)
.await;
return Ok(old_items);
}
};
let video_items: Vec<VideoItem> = self
.get_video_items_from_html(text.clone(), &mut requester, pool.clone())
.await;
@@ -143,11 +157,20 @@ impl HentaihavenProvider {
}
};
let mut requester = options.requester.clone().unwrap();
let text = requester
.get(&video_url, Some(Version::HTTP_2))
.await
.unwrap();
let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let text = match requester.get(&video_url, Some(Version::HTTP_2)).await {
Ok(text) => text,
Err(e) => {
crate::providers::report_provider_error(
"hentaihaven",
"query.request",
&format!("url={video_url}; error={e}"),
)
.await;
return Ok(old_items);
}
};
if page > 1
{
return Ok(vec![]);
@@ -308,7 +331,23 @@ impl HentaihavenProvider {
.and_then(|s| s.split('"').next())
.ok_or_else(|| ErrorKind::Parse("video url\n\n{seg}".into()))?
.to_string();
let mut conn = pool.get().expect("couldn't get db connection from pool");
let mut conn = match pool.get() {
Ok(conn) => conn,
Err(e) => {
let msg = format!("DB pool error: {}", e);
send_discord_error_report(
msg.clone(),
None,
Some("Hentai Haven Provider"),
Some("get_video_item.pool_get"),
file!(),
line!(),
module_path!(),
)
.await;
return Err(msg.into());
}
};
let db_result = db::get_video(&mut conn, video_url.clone());
drop(conn);
match db_result {
@@ -456,13 +495,27 @@ impl HentaihavenProvider {
.views(views)
.aspect_ratio(0.715);
let mut conn = pool.get().expect("couldn't get db connection from pool");
let _ = db::insert_video(
&mut conn,
&video_url,
&serde_json::to_string(&video_item).unwrap_or_default(),
);
drop(conn);
match pool.get() {
Ok(mut conn) => {
let _ = db::insert_video(
&mut conn,
&video_url,
&serde_json::to_string(&video_item).unwrap_or_default(),
);
}
Err(e) => {
send_discord_error_report(
format!("DB pool error: {}", e),
None,
Some("Hentai Haven Provider"),
Some("get_video_item.insert_video.pool_get"),
file!(),
line!(),
module_path!(),
)
.await;
}
}
Ok(video_item)
}

145
src/providers/homoxxx.rs
View File

@@ -1,5 +1,7 @@
use crate::api::ClientVersion;
use crate::DbPool;
use crate::providers::Provider;
use crate::providers::{Provider, report_provider_error};
use crate::status::*;
use crate::util::cache::VideoCache;
use crate::util::flaresolverr::{FlareSolverrRequest, Flaresolverr};
use crate::util::time::parse_time_to_seconds;
@@ -29,6 +31,42 @@ impl HomoxxxProvider {
url: "https://homo.xxx".to_string(),
}
}
fn build_channel(&self, _clientversion: ClientVersion) -> Channel {
Channel {
id: "homoxxx".to_string(),
name: "Homo.xxx".to_string(),
description: "Best Gay Porn".to_string(),
premium: false,
favicon: "https://www.google.com/s2/favicons?sz=64&domain=homo.xxx".to_string(),
status: "active".to_string(),
categories: vec![],
options: vec![ChannelOption {
id: "sort".to_string(),
title: "Sort".to_string(),
description: "Sort the Videos".to_string(),
systemImage: "list.number".to_string(),
colorName: "blue".to_string(),
options: vec![
FilterOption {
id: "new".to_string(),
title: "New".to_string(),
},
FilterOption {
id: "popular".to_string(),
title: "Popular".to_string(),
},
FilterOption {
id: "trending".to_string(),
title: "Trending".to_string(),
},
],
multiSelect: false,
}],
nsfw: true,
cacheDuration: Some(1800),
}
}
async fn get(
&self,
cache: VideoCache,
@@ -60,11 +98,25 @@ impl HomoxxxProvider {
let mut response = client.get(video_url.clone())
// .proxy(proxy.clone())
.send().await?;
if response.status().is_redirection(){
println!("Redirection detected, following to: {}", response.headers()["Location"].to_str().unwrap());
response = client.get(response.headers()["Location"].to_str().unwrap())
// .proxy(proxy.clone())
.send().await?;
if response.status().is_redirection() {
let location = match response.headers().get("Location").and_then(|h| h.to_str().ok()) {
Some(location) => location,
None => {
report_provider_error(
"homoxxx",
"get.redirect_location",
&format!("url={video_url}; missing/invalid Location header"),
)
.await;
return Ok(old_items);
}
};
println!("Redirection detected, following to: {}", location);
response = client
.get(location)
// .proxy(proxy.clone())
.send()
.await?;
}
if response.status().is_success() {
let text = response.text().await?;
@@ -77,7 +129,18 @@ impl HomoxxxProvider {
}
Ok(video_items)
} else {
let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set");
let flare_url = match env::var("FLARE_URL") {
Ok(url) => url,
Err(e) => {
report_provider_error(
"homoxxx",
"get.flare_url",
&e.to_string(),
)
.await;
return Ok(old_items);
}
};
let flare = Flaresolverr::new(flare_url);
let result = flare
.solve(FlareSolverrRequest {
@@ -115,7 +178,7 @@ impl HomoxxxProvider {
let mut video_url = format!("{}/search/{}/{}/", self.url, search_string, page);
if search_string.starts_with("@"){
let url_part = search_string.split("@").collect::<Vec<&str>>()[1].replace(":", "/");
let url_part = search_string.split("@").collect::<Vec<&str>>().get(1).copied().unwrap_or_default().replace(":", "/");
video_url = format!("{}/{}/", self.url, url_part);
}
// Check our Video Cache. If the result is younger than 1 hour, we return it.
@@ -140,11 +203,24 @@ impl HomoxxxProvider {
// .proxy(proxy.clone())
.send().await?;
if response.status().is_redirection(){
response = client.get(self.url.clone() + response.headers()["Location"].to_str().unwrap())
// .proxy(proxy.clone())
.send().await?;
if response.status().is_redirection() {
let location = match response.headers().get("Location").and_then(|h| h.to_str().ok()) {
Some(location) => location,
None => {
report_provider_error(
"homoxxx",
"query.redirect_location",
&format!("url={video_url}; missing/invalid Location header"),
)
.await;
return Ok(old_items);
}
};
response = client
.get(self.url.clone() + location)
// .proxy(proxy.clone())
.send()
.await?;
}
if response.status().is_success() {
@@ -158,7 +234,18 @@ impl HomoxxxProvider {
}
Ok(video_items)
} else {
let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set");
let flare_url = match env::var("FLARE_URL") {
Ok(url) => url,
Err(e) => {
report_provider_error(
"homoxxx",
"query.flare_url",
&e.to_string(),
)
.await;
return Ok(old_items);
}
};
let flare = Flaresolverr::new(flare_url);
let result = flare
.solve(FlareSolverrRequest {
@@ -190,7 +277,7 @@ impl HomoxxxProvider {
return vec![];
}
let mut items: Vec<VideoItem> = Vec::new();
let raw_videos = html.split("pagination").collect::<Vec<&str>>()[0]
let raw_videos = html.split("pagination").collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.split("<div class=\"item \">")
.collect::<Vec<&str>>()[1..]
.to_vec();
@@ -199,31 +286,31 @@ impl HomoxxxProvider {
// for (index, line) in vid.iter().enumerate() {
// println!("Line {}: {}", index, line);
// }
let video_url: String = video_segment.split("<a href=\"").collect::<Vec<&str>>()[1]
let video_url: String = video_segment.split("<a href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0].to_string();
let preview_url = video_segment.split("data-preview-custom=\"").collect::<Vec<&str>>()[1]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default().to_string();
let preview_url = video_segment.split("data-preview-custom=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
let mut title = video_segment.split("\" title=\"").collect::<Vec<&str>>()[1]
let mut title = video_segment.split("\" title=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
// html decode
title = decode(title.as_bytes()).to_string().unwrap_or(title);
let id = video_url.split("/").collect::<Vec<&str>>()[4].to_string();
let id = video_url.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string();
let raw_duration = video_segment
.split("<p class=\"duration_item\">").collect::<Vec<&str>>()[1]
.split("<p class=\"duration_item\">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
let thumb = video_segment.split("thumb lazyload").collect::<Vec<&str>>()[1]
.split("data-src=\"").collect::<Vec<&str>>()[1]
let thumb = video_segment.split("thumb lazyload").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("data-src=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
let video_item = VideoItem::new(
id,
@@ -276,4 +363,8 @@ impl Provider for HomoxxxProvider {
}
}
}
fn get_channel(&self, clientversion: ClientVersion) -> Option<Channel> {
Some(self.build_channel(clientversion))
}
}

22
src/providers/hqporner.rs
View File

@@ -289,7 +289,7 @@ impl HqpornerProvider {
}
})
.filter_map(Result::ok)
.filter(|item| !item.formats.clone().unwrap().is_empty())
.filter(|item| item.formats.as_ref().map(|formats| !formats.is_empty()).unwrap_or(false))
.collect()
}
@@ -409,8 +409,24 @@ impl HqpornerProvider {
vec![("Referer".to_string(), "https://hqporner.com/".into())],
).await;
}
let text2 = r
.unwrap()
let response = match r {
Ok(response) => response,
Err(e) => {
let err = format!("altplayer request failed: {e}");
send_discord_error_report(
err.clone(),
None,
Some("Hqporner Provider"),
Some(&player_url),
file!(),
line!(),
module_path!(),
)
.await;
return Ok((tags, formats));
}
};
let text2 = response
.text()
.await
.map_err(|e| Error::from(format!("Text conversion failed: {}", e)))?;

56
src/providers/hypnotube.rs
View File

@@ -137,10 +137,16 @@ impl HypnotubeProvider {
categories: self
.categories
.read()
.unwrap()
.iter()
.map(|c| c.title.clone())
.collect(),
.map(|categories| categories.iter().map(|c| c.title.clone()).collect())
.unwrap_or_else(|e| {
eprint!("Hypnotube categories lock error: {e}");
crate::providers::report_provider_error_background(
"hypnotube",
"build_channel.categories_read",
&e.to_string(),
);
vec![]
}),
options: vec![ChannelOption {
id: "sort".to_string(),
title: "Sort".to_string(),
@@ -206,11 +212,19 @@ impl HypnotubeProvider {
vec![]
}
};
let mut requester = options.requester.clone().unwrap();
let text = requester
.get(&video_url, Some(Version::HTTP_11))
.await
.unwrap();
let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let text = match requester.get(&video_url, Some(Version::HTTP_11)).await {
Ok(text) => text,
Err(e) => {
crate::providers::report_provider_error(
"hypnotube",
"get.request",
&format!("url={video_url}; error={e}"),
)
.await;
return old_items;
}
};
if text.contains("Sorry, no results were found.") {
return vec![];
}
@@ -259,21 +273,35 @@ impl HypnotubeProvider {
}
};
let mut requester = options.requester.clone().unwrap();
let text = match requester
let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let post_response = match requester
.post(
format!("{}/searchgate.php", self.url).as_str(),
format!("q={}&type=videos", query.replace(" ", "+")).as_str(),
vec![("Content-Type", "application/x-www-form-urlencoded")],
)
.await
.unwrap()
.text()
.await
{
Ok(response) => response,
Err(e) => {
crate::providers::report_provider_error(
"hypnotube",
"query.search_post",
&format!("url={video_url}; error={e}"),
)
.await;
return old_items;
}
};
let text = match post_response.text().await {
Ok(t) => t,
Err(e) => {
eprint!("Hypnotube search POST request failed: {}", e);
crate::providers::report_provider_error_background(
"hypnotube",
"query.search_post.text",
&e.to_string(),
);
return vec![];
}
};

45
src/providers/javtiful.rs
View File

@@ -58,10 +58,15 @@ impl JavtifulProvider {
categories: self
.categories
.read()
.unwrap()
.iter()
.map(|c| c.title.clone())
.collect(),
.map(|categories| categories.iter().map(|c| c.title.clone()).collect())
.unwrap_or_else(|e| {
crate::providers::report_provider_error_background(
"javtiful",
"build_channel.categories_read",
&e.to_string(),
);
vec![]
}),
options: vec![ChannelOption {
id: "sort".to_string(),
title: "Sort".to_string(),
@@ -130,8 +135,20 @@ impl JavtifulProvider {
}
};
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url, Some(Version::HTTP_2)).await.unwrap();
let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let text = match requester.get(&video_url, Some(Version::HTTP_2)).await {
Ok(text) => text,
Err(e) => {
crate::providers::report_provider_error(
"javtiful",
"get.request",
&format!("url={video_url}; error={e}"),
)
.await;
return Ok(old_items);
}
};
if page > 1 && !text.contains(&format!("<li class=\"page-item active\"><span class=\"page-link\">{}</span>", page)) {
return Ok(vec![]);
}
@@ -178,8 +195,20 @@ impl JavtifulProvider {
}
};
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url, Some(Version::HTTP_2)).await.unwrap();
let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let text = match requester.get(&video_url, Some(Version::HTTP_2)).await {
Ok(text) => text,
Err(e) => {
crate::providers::report_provider_error(
"javtiful",
"query.request",
&format!("url={video_url}; error={e}"),
)
.await;
return Ok(old_items);
}
};
if page > 1 && !text.contains(&format!("<li class=\"page-item active\"><span class=\"page-link\">{}</span>", page)) {
return Ok(vec![]);
}

155
src/providers/missav.rs
View File

@@ -5,8 +5,10 @@ use error_chain::error_chain;
use htmlentity::entity::{decode, ICodedDataTrait};
use futures::future::join_all;
use wreq::Version;
use crate::api::ClientVersion;
use crate::db;
use crate::providers::Provider;
use crate::status::*;
use crate::util::cache::VideoCache;
use crate::util::discord::{format_error_chain, send_discord_error_report};
use crate::videos::ServerOptions;
@@ -41,6 +43,140 @@ impl MissavProvider {
}
}
fn build_channel(&self, _clientversion: ClientVersion) -> Channel {
Channel {
id: "missav".to_string(),
name: "MissAV".to_string(),
description: "Watch HD JAV Online".to_string(),
premium: false,
favicon: "https://www.google.com/s2/favicons?sz=64&domain=missav.ws".to_string(),
status: "active".to_string(),
categories: vec![],
options: vec![
ChannelOption {
id: "sort".to_string(),
title: "Sort".to_string(),
description: "Sort the Videos".to_string(),
systemImage: "list.number".to_string(),
colorName: "blue".to_string(),
options: vec![
FilterOption {
id: "released_at".to_string(),
title: "Release Date".to_string(),
},
FilterOption {
id: "published_at".to_string(),
title: "Recent Update".to_string(),
},
FilterOption {
id: "today_views".to_string(),
title: "Today Views".to_string(),
},
FilterOption {
id: "weekly_views".to_string(),
title: "Weekly Views".to_string(),
},
FilterOption {
id: "monthly_views".to_string(),
title: "Monthly Views".to_string(),
},
FilterOption {
id: "views".to_string(),
title: "Total Views".to_string(),
},
],
multiSelect: false,
},
ChannelOption {
id: "filter".to_string(),
title: "Filter".to_string(),
description: "Filter the Videos".to_string(),
systemImage: "line.horizontal.3.decrease.circle".to_string(),
colorName: "green".to_string(),
options: vec![
FilterOption {
id: "new".to_string(),
title: "Recent update".to_string(),
},
FilterOption {
id: "release".to_string(),
title: "New Releases".to_string(),
},
FilterOption {
id: "uncensored-leak".to_string(),
title: "Uncensored".to_string(),
},
FilterOption {
id: "english-subtitle".to_string(),
title: "English subtitle".to_string(),
},
],
multiSelect: false,
},
ChannelOption {
id: "language".to_string(),
title: "Language".to_string(),
description: "What Language to fetch".to_string(),
systemImage: "flag.fill".to_string(),
colorName: "gray".to_string(),
options: vec![
FilterOption {
id: "en".to_string(),
title: "English".to_string(),
},
FilterOption {
id: "cn".to_string(),
title: "简体中文".to_string(),
},
FilterOption {
id: "ja".to_string(),
title: "日本語".to_string(),
},
FilterOption {
id: "ko".to_string(),
title: "한국의".to_string(),
},
FilterOption {
id: "ms".to_string(),
title: "Melayu".to_string(),
},
FilterOption {
id: "th".to_string(),
title: "ไทย".to_string(),
},
FilterOption {
id: "de".to_string(),
title: "Deutsch".to_string(),
},
FilterOption {
id: "fr".to_string(),
title: "Français".to_string(),
},
FilterOption {
id: "vi".to_string(),
title: "Tiếng Việt".to_string(),
},
FilterOption {
id: "id".to_string(),
title: "Bahasa Indonesia".to_string(),
},
FilterOption {
id: "fil".to_string(),
title: "Filipino".to_string(),
},
FilterOption {
id: "pt".to_string(),
title: "Português".to_string(),
},
],
multiSelect: false,
},
],
nsfw: true,
cacheDuration: None,
}
}
async fn get(&self, cache: VideoCache, pool: DbPool, page: u8, mut sort: String, options: ServerOptions) -> Result<Vec<VideoItem>> {
// Use ok_or to avoid unwrapping options
let language = options.language.as_ref().ok_or("Missing language")?;
@@ -185,9 +321,16 @@ impl MissavProvider {
let parts_str = vid.split("m3u8").nth(1)?.split("https").next()?;
let mut parts: Vec<&str> = parts_str.split('|').collect();
parts.reverse();
if parts.len() < 8 { return None; }
Some(format!("https://{}.{}/{}-{}-{}-{}-{}/playlist.m3u8",
parts[1], parts[2], parts[3], parts[4], parts[5], parts[6], parts[7]))
Some(format!(
"https://{}.{}/{}-{}-{}-{}-{}/playlist.m3u8",
parts.get(1)?,
parts.get(2)?,
parts.get(3)?,
parts.get(4)?,
parts.get(5)?,
parts.get(6)?,
parts.get(7)?
))
})().ok_or_else(|| ErrorKind::ParsingError(format!("video_url\n{:?}", vid).to_string()))?;
let video_item = VideoItem::new(id, title, video_url, "missav".to_string(), thumb, duration)
@@ -218,4 +361,8 @@ impl Provider for MissavProvider {
vec![]
})
}
}
fn get_channel(&self, clientversion: ClientVersion) -> Option<Channel> {
Some(self.build_channel(clientversion))
}
}

101
src/providers/mod.rs
View File

@@ -1,10 +1,13 @@
use async_trait::async_trait;
use futures::FutureExt;
use once_cell::sync::Lazy;
use rustc_hash::FxHashMap as HashMap;
use std::future::Future;
use std::panic::AssertUnwindSafe;
use std::sync::Arc;
use crate::{
DbPool, api::ClientVersion, status::Channel, util::cache::VideoCache, videos::{ServerOptions, VideoItem}
DbPool, api::ClientVersion, status::Channel, util::{cache::VideoCache, discord::send_discord_error_report, requester::Requester}, videos::{ServerOptions, VideoItem}
};
pub mod all;
@@ -49,6 +52,33 @@ pub type DynProvider = Arc<dyn Provider>;
pub static ALL_PROVIDERS: Lazy<HashMap<&'static str, DynProvider>> = Lazy::new(|| {
let mut m = HashMap::default();
m.insert("all", Arc::new(all::AllProvider::new()) as DynProvider);
m.insert("perverzija", Arc::new(perverzija::PerverzijaProvider::new()) as DynProvider);
m.insert("hanime", Arc::new(hanime::HanimeProvider::new()) as DynProvider);
m.insert("pornhub", Arc::new(pornhub::PornhubProvider::new()) as DynProvider);
m.insert(
"rule34video",
Arc::new(rule34video::Rule34videoProvider::new()) as DynProvider,
);
m.insert("redtube", Arc::new(redtube::RedtubeProvider::new()) as DynProvider);
m.insert("okporn", Arc::new(okporn::OkpornProvider::new()) as DynProvider);
m.insert("pornhat", Arc::new(pornhat::PornhatProvider::new()) as DynProvider);
m.insert(
"perfectgirls",
Arc::new(perfectgirls::PerfectgirlsProvider::new()) as DynProvider,
);
m.insert("okxxx", Arc::new(okxxx::OkxxxProvider::new()) as DynProvider);
m.insert("homoxxx", Arc::new(homoxxx::HomoxxxProvider::new()) as DynProvider);
m.insert("missav", Arc::new(missav::MissavProvider::new()) as DynProvider);
m.insert("xxthots", Arc::new(xxthots::XxthotsProvider::new()) as DynProvider);
m.insert("sxyprn", Arc::new(sxyprn::SxyprnProvider::new()) as DynProvider);
m.insert("porn00", Arc::new(porn00::Porn00Provider::new()) as DynProvider);
m.insert("youjizz", Arc::new(youjizz::YoujizzProvider::new()) as DynProvider);
m.insert(
"paradisehill",
Arc::new(paradisehill::ParadisehillProvider::new()) as DynProvider,
);
m.insert("pornzog", Arc::new(pornzog::PornzogProvider::new()) as DynProvider);
m.insert("omgxxx", Arc::new(omgxxx::OmgxxxProvider::new()) as DynProvider);
m.insert("beeg", Arc::new(beeg::BeegProvider::new()) as DynProvider);
m.insert("tnaflix", Arc::new(tnaflix::TnaflixProvider::new()) as DynProvider);
@@ -74,6 +104,75 @@ pub fn init_providers_now() {
Lazy::force(&ALL_PROVIDERS);
}
pub fn panic_payload_to_string(payload: Box<dyn std::any::Any + Send>) -> String {
if let Some(s) = payload.downcast_ref::<&str>() {
return (*s).to_string();
}
if let Some(s) = payload.downcast_ref::<String>() {
return s.clone();
}
"unknown panic payload".to_string()
}
pub async fn run_provider_guarded<F>(provider_name: &str, context: &str, fut: F) -> Vec<VideoItem>
where
F: Future<Output = Vec<VideoItem>>,
{
match AssertUnwindSafe(fut).catch_unwind().await {
Ok(videos) => videos,
Err(payload) => {
let panic_msg = panic_payload_to_string(payload);
let _ = send_discord_error_report(
format!("Provider panic: {}", provider_name),
None,
Some("Provider Guard"),
Some(&format!("context={}; panic={}", context, panic_msg)),
file!(),
line!(),
module_path!(),
)
.await;
vec![]
}
}
}
pub async fn report_provider_error(provider_name: &str, context: &str, msg: &str) {
let _ = send_discord_error_report(
format!("Provider error: {}", provider_name),
None,
Some("Provider Guard"),
Some(&format!("context={}; error={}", context, msg)),
file!(),
line!(),
module_path!(),
)
.await;
}
pub fn report_provider_error_background(provider_name: &str, context: &str, msg: &str) {
let provider_name = provider_name.to_string();
let context = context.to_string();
let msg = msg.to_string();
tokio::spawn(async move {
report_provider_error(&provider_name, &context, &msg).await;
});
}
pub fn requester_or_default(options: &ServerOptions, provider_name: &str, context: &str) -> Requester {
match options.requester.clone() {
Some(requester) => requester,
None => {
report_provider_error_background(
provider_name,
context,
"ServerOptions.requester missing; using default Requester",
);
Requester::new()
}
}
}
#[async_trait]
pub trait Provider: Send + Sync {

134
src/providers/okporn.rs
View File

@@ -1,5 +1,7 @@
use crate::api::ClientVersion;
use crate::DbPool;
use crate::providers::Provider;
use crate::providers::{Provider, report_provider_error};
use crate::status::*;
use crate::util::cache::VideoCache;
use crate::util::flaresolverr::{FlareSolverrRequest, Flaresolverr};
use crate::util::time::parse_time_to_seconds;
@@ -29,6 +31,42 @@ impl OkpornProvider {
url: "https://ok.porn".to_string(),
}
}
fn build_channel(&self, _clientversion: ClientVersion) -> Channel {
Channel {
id: "okporn".to_string(),
name: "Ok.porn".to_string(),
description: "Tons of HD porno movies".to_string(),
premium: false,
favicon: "https://www.google.com/s2/favicons?sz=64&domain=ok.porn".to_string(),
status: "active".to_string(),
categories: vec![],
options: vec![ChannelOption {
id: "sort".to_string(),
title: "Sort".to_string(),
description: "Sort the Videos".to_string(),
systemImage: "list.number".to_string(),
colorName: "blue".to_string(),
options: vec![
FilterOption {
id: "new".to_string(),
title: "New".to_string(),
},
FilterOption {
id: "popular".to_string(),
title: "Popular".to_string(),
},
FilterOption {
id: "trending".to_string(),
title: "Trending".to_string(),
},
],
multiSelect: false,
}],
nsfw: true,
cacheDuration: Some(1800),
}
}
async fn get(
&self,
cache: VideoCache,
@@ -60,11 +98,24 @@ impl OkpornProvider {
let mut response = client.get(video_url.clone())
// .proxy(proxy.clone())
.send().await?;
if response.status().is_redirection(){
response = client.get(self.url.clone() + response.headers()["Location"].to_str().unwrap())
// .proxy(proxy.clone())
.send().await?;
if response.status().is_redirection() {
let location = match response.headers().get("Location").and_then(|h| h.to_str().ok()) {
Some(location) => location,
None => {
report_provider_error(
"okporn",
"get.redirect_location",
&format!("url={video_url}; missing/invalid Location header"),
)
.await;
return Ok(old_items);
}
};
response = client
.get(self.url.clone() + location)
// .proxy(proxy.clone())
.send()
.await?;
}
if response.status().is_success() {
let text = response.text().await?;
@@ -77,7 +128,18 @@ impl OkpornProvider {
}
Ok(video_items)
} else {
let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set");
let flare_url = match env::var("FLARE_URL") {
Ok(url) => url,
Err(e) => {
report_provider_error(
"okporn",
"get.flare_url",
&e.to_string(),
)
.await;
return Ok(old_items);
}
};
let flare = Flaresolverr::new(flare_url);
let result = flare
.solve(FlareSolverrRequest {
@@ -136,11 +198,24 @@ impl OkpornProvider {
// .proxy(proxy.clone())
.send().await?;
if response.status().is_redirection(){
response = client.get(self.url.clone() + response.headers()["Location"].to_str().unwrap())
// .proxy(proxy.clone())
.send().await?;
if response.status().is_redirection() {
let location = match response.headers().get("Location").and_then(|h| h.to_str().ok()) {
Some(location) => location,
None => {
report_provider_error(
"okporn",
"query.redirect_location",
&format!("url={video_url}; missing/invalid Location header"),
)
.await;
return Ok(old_items);
}
};
response = client
.get(self.url.clone() + location)
// .proxy(proxy.clone())
.send()
.await?;
}
if response.status().is_success() {
@@ -154,7 +229,18 @@ impl OkpornProvider {
}
Ok(video_items)
} else {
let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set");
let flare_url = match env::var("FLARE_URL") {
Ok(url) => url,
Err(e) => {
report_provider_error(
"okporn",
"query.flare_url",
&e.to_string(),
)
.await;
return Ok(old_items);
}
};
let flare = Flaresolverr::new(flare_url);
let result = flare
.solve(FlareSolverrRequest {
@@ -195,25 +281,25 @@ impl OkpornProvider {
// for (index, line) in vid.iter().enumerate() {
// println!("Line {}: {}", index, line);
// }
let video_url: String = format!("{}{}", self.url, video_segment.split("<a href=\"").collect::<Vec<&str>>()[1]
let video_url: String = format!("{}{}", self.url, video_segment.split("<a href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0]);
let mut title = video_segment.split("\" title=\"").collect::<Vec<&str>>()[1]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default());
let mut title = video_segment.split("\" title=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
// html decode
title = decode(title.as_bytes()).to_string().unwrap_or(title);
let id = video_url.split("/").collect::<Vec<&str>>()[4].to_string();
let raw_duration = video_segment.split("<span class=\"duration_item\">").collect::<Vec<&str>>()[1]
let id = video_url.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string();
let raw_duration = video_segment.split("<span class=\"duration_item\">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
let thumb = video_segment.split("<img class=\"thumb lazy-load\" src=\"").collect::<Vec<&str>>()[1].split("data-original=\"").collect::<Vec<&str>>()[1]
let thumb = video_segment.split("<img class=\"thumb lazy-load\" src=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default().split("data-original=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
let video_item = VideoItem::new(
@@ -266,4 +352,8 @@ impl Provider for OkpornProvider {
}
}
}
fn get_channel(&self, clientversion: ClientVersion) -> Option<Channel> {
Some(self.build_channel(clientversion))
}
}

157
src/providers/okxxx.rs
View File

@@ -1,6 +1,8 @@
use crate::api::ClientVersion;
use crate::util::parse_abbreviated_number;
use crate::DbPool;
use crate::providers::Provider;
use crate::providers::{Provider, report_provider_error};
use crate::status::*;
use crate::util::cache::VideoCache;
use crate::util::flaresolverr::{FlareSolverrRequest, Flaresolverr};
use crate::util::time::parse_time_to_seconds;
@@ -30,6 +32,42 @@ impl OkxxxProvider {
url: "https://ok.xxx".to_string(),
}
}
fn build_channel(&self, _clientversion: ClientVersion) -> Channel {
Channel {
id: "okxxx".to_string(),
name: "Ok.xxx".to_string(),
description: "free porn tube!".to_string(),
premium: false,
favicon: "https://www.google.com/s2/favicons?sz=64&domain=ok.xxx".to_string(),
status: "active".to_string(),
categories: vec![],
options: vec![ChannelOption {
id: "sort".to_string(),
title: "Sort".to_string(),
description: "Sort the Videos".to_string(),
systemImage: "list.number".to_string(),
colorName: "blue".to_string(),
options: vec![
FilterOption {
id: "new".to_string(),
title: "New".to_string(),
},
FilterOption {
id: "popular".to_string(),
title: "Popular".to_string(),
},
FilterOption {
id: "trending".to_string(),
title: "Trending".to_string(),
},
],
multiSelect: false,
}],
nsfw: true,
cacheDuration: Some(1800),
}
}
async fn get(
&self,
cache: VideoCache,
@@ -61,11 +99,25 @@ impl OkxxxProvider {
let mut response = client.get(video_url.clone())
// .proxy(proxy.clone())
.send().await?;
if response.status().is_redirection(){
println!("Redirection detected, following to: {}", response.headers()["Location"].to_str().unwrap());
response = client.get(response.headers()["Location"].to_str().unwrap())
// .proxy(proxy.clone())
.send().await?;
if response.status().is_redirection() {
let location = match response.headers().get("Location").and_then(|h| h.to_str().ok()) {
Some(location) => location,
None => {
report_provider_error(
"okxxx",
"get.redirect_location",
&format!("url={video_url}; missing/invalid Location header"),
)
.await;
return Ok(old_items);
}
};
println!("Redirection detected, following to: {}", location);
response = client
.get(location)
// .proxy(proxy.clone())
.send()
.await?;
}
if response.status().is_success() {
let text = response.text().await?;
@@ -78,7 +130,18 @@ impl OkxxxProvider {
}
Ok(video_items)
} else {
let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set");
let flare_url = match env::var("FLARE_URL") {
Ok(url) => url,
Err(e) => {
report_provider_error(
"okxxx",
"get.flare_url",
&e.to_string(),
)
.await;
return Ok(old_items);
}
};
let flare = Flaresolverr::new(flare_url);
let result = flare
.solve(FlareSolverrRequest {
@@ -116,7 +179,7 @@ impl OkxxxProvider {
let mut video_url = format!("{}/search/{}/{}/", self.url, search_string, page);
if search_string.starts_with("@"){
let url_part = search_string.split("@").collect::<Vec<&str>>()[1].replace(":", "/");
let url_part = search_string.split("@").collect::<Vec<&str>>().get(1).copied().unwrap_or_default().replace(":", "/");
video_url = format!("{}/{}/", self.url, url_part);
}
// Check our Video Cache. If the result is younger than 1 hour, we return it.
@@ -141,11 +204,24 @@ impl OkxxxProvider {
// .proxy(proxy.clone())
.send().await?;
if response.status().is_redirection(){
response = client.get(self.url.clone() + response.headers()["Location"].to_str().unwrap())
// .proxy(proxy.clone())
.send().await?;
if response.status().is_redirection() {
let location = match response.headers().get("Location").and_then(|h| h.to_str().ok()) {
Some(location) => location,
None => {
report_provider_error(
"okxxx",
"query.redirect_location",
&format!("url={video_url}; missing/invalid Location header"),
)
.await;
return Ok(old_items);
}
};
response = client
.get(self.url.clone() + location)
// .proxy(proxy.clone())
.send()
.await?;
}
if response.status().is_success() {
@@ -159,7 +235,18 @@ impl OkxxxProvider {
}
Ok(video_items)
} else {
let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set");
let flare_url = match env::var("FLARE_URL") {
Ok(url) => url,
Err(e) => {
report_provider_error(
"okxxx",
"query.flare_url",
&e.to_string(),
)
.await;
return Ok(old_items);
}
};
let flare = Flaresolverr::new(flare_url);
let result = flare
.solve(FlareSolverrRequest {
@@ -191,7 +278,7 @@ impl OkxxxProvider {
return vec![];
}
let mut items: Vec<VideoItem> = Vec::new();
let raw_videos = html.split("<div class=\"pagination\"").collect::<Vec<&str>>()[0]
let raw_videos = html.split("<div class=\"pagination\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.split("item thumb-bl thumb-bl-video video_")
.collect::<Vec<&str>>()[1..]
.to_vec();
@@ -200,38 +287,38 @@ impl OkxxxProvider {
// for (index, line) in vid.iter().enumerate() {
// println!("Line {}: {}", index, line);
// }
let video_url: String = format!("{}{}", self.url, video_segment.split("<a href=\"").collect::<Vec<&str>>()[1]
let video_url: String = format!("{}{}", self.url, video_segment.split("<a href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0]);
let preview_url = video_segment.split("data-preview-custom=\"").collect::<Vec<&str>>()[1]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default());
let preview_url = video_segment.split("data-preview-custom=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
let mut title = video_segment.split("\" title=\"").collect::<Vec<&str>>()[1]
let mut title = video_segment.split("\" title=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
// html decode
title = decode(title.as_bytes()).to_string().unwrap_or(title);
let id = video_url.split("/").collect::<Vec<&str>>()[4].to_string();
let raw_duration = video_segment.split("fa fa-clock-o").collect::<Vec<&str>>()[1]
.split("<span>").collect::<Vec<&str>>()[1]
let id = video_url.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string();
let raw_duration = video_segment.split("fa fa-clock-o").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<span>").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
let thumb = format!("https:{}", video_segment.split(" class=\"thumb lazy-load\"").collect::<Vec<&str>>()[1]
.split("data-original=\"").collect::<Vec<&str>>()[1]
let thumb = format!("https:{}", video_segment.split(" class=\"thumb lazy-load\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("data-original=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string());
let mut tags = vec![];
if video_segment.contains("href=\"/sites/"){
let raw_tags = video_segment.split("href=\"/sites/").collect::<Vec<&str>>()[1..]
.iter()
.map(|s| s.split("/\"").collect::<Vec<&str>>()[0].to_string())
.map(|s| s.split("/\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default().to_string())
.collect::<Vec<String>>();
for tag in raw_tags {
if !tag.is_empty() {
@@ -242,7 +329,7 @@ impl OkxxxProvider {
if video_segment.contains("href=\"/models/"){
let raw_tags = video_segment.split("href=\"/models/").collect::<Vec<&str>>()[1..]
.iter()
.map(|s| s.split("/\"").collect::<Vec<&str>>()[0].to_string())
.map(|s| s.split("/\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default().to_string())
.collect::<Vec<String>>();
for tag in raw_tags {
if !tag.is_empty() {
@@ -251,10 +338,10 @@ impl OkxxxProvider {
}
}
let views_part = video_segment.split("fa fa-eye").collect::<Vec<&str>>()[1]
.split("<span>").collect::<Vec<&str>>()[1]
let views_part = video_segment.split("fa fa-eye").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<span>").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
let views = parse_abbreviated_number(&views_part).unwrap_or(0) as u32;
@@ -311,4 +398,8 @@ impl Provider for OkxxxProvider {
}
}
}
fn get_channel(&self, clientversion: ClientVersion) -> Option<Channel> {
Some(self.build_channel(clientversion))
}
}

294
src/providers/omgxxx.rs
View File

@@ -1,6 +1,6 @@
use crate::DbPool;
use crate::api::ClientVersion;
use crate::providers::Provider;
use crate::providers::{Provider, report_provider_error, report_provider_error_background};
use crate::util::cache::VideoCache;
use crate::util::parse_abbreviated_number;
use crate::util::time::parse_time_to_seconds;
@@ -58,10 +58,20 @@ impl OmgxxxProvider {
thread::spawn(move || {
// Create a tiny runtime just for these async tasks
let rt = tokio::runtime::Builder::new_current_thread()
let rt = match tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("build tokio runtime");
{
Ok(rt) => rt,
Err(e) => {
report_provider_error_background(
"omgxxx",
"spawn_initial_load.runtime_build",
&e.to_string(),
);
return;
}
};
rt.block_on(async move {
// If you have a streaming sites loader, call it here too
@@ -83,13 +93,23 @@ impl OmgxxxProvider {
async fn load_stars(base_url: &str, stars: Arc<RwLock<Vec<FilterOption>>>) -> Result<()> {
let mut requester = util::requester::Requester::new();
for page in [1..10].into_iter().flatten() {
let text = requester
let text = match requester
.get(
format!("{}/models/total-videos/{}/?gender_id=0", &base_url, page).as_str(),
None,
)
.await
.unwrap();
{
Ok(text) => text,
Err(e) => {
report_provider_error_background(
"omgxxx",
"load_stars.request",
&format!("url={base_url}; page={page}; error={e}"),
);
break;
}
};
if text.contains("404 Not Found") || text.is_empty() {
break;
}
@@ -97,19 +117,20 @@ impl OmgxxxProvider {
.split("<div class=\"list-models\">")
.collect::<Vec<&str>>()
.last()
.unwrap()
.copied()
.unwrap_or_default()
.split("custom_list_models_models_list_pagination")
.collect::<Vec<&str>>()[0];
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default();
for stars_element in stars_div.split("<a ").collect::<Vec<&str>>()[1..].to_vec() {
let star_url = stars_element.split("href=\"").collect::<Vec<&str>>()[1]
let star_url = stars_element.split("href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0];
let star_id = star_url.split("/").collect::<Vec<&str>>()[4].to_string();
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default();
let star_id = star_url.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string();
let star_name = stars_element
.split("<strong class=\"title\">")
.collect::<Vec<&str>>()[1]
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
Self::push_unique(
&stars,
@@ -130,26 +151,36 @@ impl OmgxxxProvider {
page += 1;
let text = requester
.get(format!("{}/sites/{}/", &base_url, page).as_str(), None)
.await
.unwrap();
.await;
let text = match text {
Ok(text) => text,
Err(e) => {
report_provider_error_background(
"omgxxx",
"load_sites.request",
&format!("url={base_url}; page={page}; error={e}"),
);
break;
}
};
if text.contains("404 Not Found") || text.is_empty() {
break;
}
let sites_div = text
.split("id=\"list_content_sources_sponsors_list_items\"")
.collect::<Vec<&str>>()[1]
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("class=\"pagination\"")
.collect::<Vec<&str>>()[0];
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default();
for sites_element in
sites_div.split("class=\"headline\"").collect::<Vec<&str>>()[1..].to_vec()
{
let site_url = sites_element.split("href=\"").collect::<Vec<&str>>()[1]
let site_url = sites_element.split("href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0];
let site_id = site_url.split("/").collect::<Vec<&str>>()[4].to_string();
let site_name = sites_element.split("<h2>").collect::<Vec<&str>>()[1]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default();
let site_id = site_url.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string();
let site_name = sites_element.split("<h2>").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
Self::push_unique(
&sites,
@@ -165,23 +196,33 @@ impl OmgxxxProvider {
async fn load_networks(base_url: &str, networks: Arc<RwLock<Vec<FilterOption>>>) -> Result<()> {
let mut requester = util::requester::Requester::new();
let text = requester.get(&base_url, None).await.unwrap();
let networks_div = text.split("class=\"sites__list\"").collect::<Vec<&str>>()[1]
let text = match requester.get(&base_url, None).await {
Ok(text) => text,
Err(e) => {
report_provider_error_background(
"omgxxx",
"load_networks.request",
&format!("url={base_url}; error={e}"),
);
return Ok(());
}
};
let networks_div = text.split("class=\"sites__list\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("</div>")
.collect::<Vec<&str>>()[0];
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default();
for network_element in
networks_div.split("sites__item").collect::<Vec<&str>>()[1..].to_vec()
{
if network_element.contains("sites__all") {
continue;
}
let network_url = network_element.split("href=\"").collect::<Vec<&str>>()[1]
let network_url = network_element.split("href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0];
let network_id = network_url.split("/").collect::<Vec<&str>>()[4].to_string();
let network_name = network_element.split(">").collect::<Vec<&str>>()[1]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default();
let network_id = network_url.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string();
let network_name = network_element.split(">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
Self::push_unique(
&networks,
@@ -306,35 +347,20 @@ impl OmgxxxProvider {
"most-popular" => "/most-popular".to_string(),
_ => "".to_string(),
};
if options.network.is_some()
&& !options.network.as_ref().unwrap().is_empty()
&& options.network.as_ref().unwrap() != "all"
{
sort_string = format!(
"networks/{}{}",
options.network.as_ref().unwrap(),
alt_sort_string
);
if let Some(network) = options.network.as_deref() {
if !network.is_empty() && network != "all" {
sort_string = format!("networks/{}{}", network, alt_sort_string);
}
}
if options.sites.is_some()
&& !options.sites.as_ref().unwrap().is_empty()
&& options.sites.as_ref().unwrap() != "all"
{
sort_string = format!(
"sites/{}{}",
options.sites.as_ref().unwrap(),
alt_sort_string
);
if let Some(site) = options.sites.as_deref() {
if !site.is_empty() && site != "all" {
sort_string = format!("sites/{}{}", site, alt_sort_string);
}
}
if options.stars.is_some()
&& !options.stars.as_ref().unwrap().is_empty()
&& options.stars.as_ref().unwrap() != "all"
{
sort_string = format!(
"models/{}{}",
options.stars.as_ref().unwrap(),
alt_sort_string
);
if let Some(star) = options.stars.as_deref() {
if !star.is_empty() && star != "all" {
sort_string = format!("models/{}{}", star, alt_sort_string);
}
}
let video_url = format!("{}/{}/{}/", self.url, sort_string, page);
let old_items = match cache.get(&video_url) {
@@ -350,8 +376,20 @@ impl OmgxxxProvider {
}
};
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url, None).await.unwrap();
let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let text = match requester.get(&video_url, None).await {
Ok(text) => text,
Err(e) => {
report_provider_error(
"omgxxx",
"get.request",
&format!("url={video_url}; error={e}"),
)
.await;
return Ok(old_items);
}
};
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);
@@ -371,31 +409,41 @@ impl OmgxxxProvider {
) -> Result<Vec<VideoItem>> {
let mut search_type = "search";
let mut search_string = query.to_string().to_ascii_lowercase().trim().to_string();
match self
.stars
.read()
.unwrap()
.iter()
.find(|s| s.title.to_ascii_lowercase() == search_string)
{
Some(star) => {
search_type = "models";
search_string = star.id.clone();
match self.stars.read() {
Ok(stars) => {
if let Some(star) = stars
.iter()
.find(|s| s.title.to_ascii_lowercase() == search_string)
{
search_type = "models";
search_string = star.id.clone();
}
}
Err(e) => {
report_provider_error_background(
"omgxxx",
"query.stars_read",
&e.to_string(),
);
}
_ => {}
}
match self
.sites
.read()
.unwrap()
.iter()
.find(|s| s.title.to_ascii_lowercase() == search_string)
{
Some(site) => {
search_type = "sites";
search_string = site.id.clone();
match self.sites.read() {
Ok(sites) => {
if let Some(site) = sites
.iter()
.find(|s| s.title.to_ascii_lowercase() == search_string)
{
search_type = "sites";
search_string = site.id.clone();
}
}
Err(e) => {
report_provider_error_background(
"omgxxx",
"query.sites_read",
&e.to_string(),
);
}
_ => {}
}
let mut video_url = format!("{}/{}/{}/{}/", self.url, search_type, search_string, page);
video_url = video_url.replace(" ", "+");
@@ -414,9 +462,20 @@ impl OmgxxxProvider {
}
};
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url, None).await.unwrap();
let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let text = match requester.get(&video_url, None).await {
Ok(text) => text,
Err(e) => {
report_provider_error(
"omgxxx",
"query.request",
&format!("url={video_url}; error={e}"),
)
.await;
return Ok(old_items);
}
};
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);
@@ -429,7 +488,18 @@ impl OmgxxxProvider {
fn get_site_id_from_name(&self, site_name: &str) -> Option<String> {
// site_name.to_lowercase().replace(" ", "")
for site in self.sites.read().unwrap().iter() {
let sites_guard = match self.sites.read() {
Ok(guard) => guard,
Err(e) => {
report_provider_error_background(
"omgxxx",
"get_site_id_from_name.sites_read",
&e.to_string(),
);
return None;
}
};
for site in sites_guard.iter() {
if site
.title
.to_lowercase()
@@ -452,11 +522,11 @@ impl OmgxxxProvider {
if !html.contains("class=\"item\"") {
return items;
}
let raw_videos = html.split("videos_list_pagination").collect::<Vec<&str>>()[0]
let raw_videos = html.split("videos_list_pagination").collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.split(" class=\"pagination\" ")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.split("class=\"list-videos\"")
.collect::<Vec<&str>>()[1]
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("class=\"item\"")
.collect::<Vec<&str>>()[1..]
.to_vec();
@@ -465,39 +535,39 @@ impl OmgxxxProvider {
// for (index, line) in vid.iter().enumerate() {
// println!("Line {}: {}", index, line);
// }
let video_url: String = video_segment.split("<a href=\"").collect::<Vec<&str>>()[1]
let video_url: String = video_segment.split("<a href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
let mut title = video_segment.split(" title=\"").collect::<Vec<&str>>()[1]
let mut title = video_segment.split(" title=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
// html decode
title = decode(title.as_bytes()).to_string().unwrap_or(title);
let id = video_url.split("/").collect::<Vec<&str>>()[4].to_string();
let id = video_url.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string();
let thumb = match video_segment.split("img loading").collect::<Vec<&str>>()[1]
let thumb = match video_segment.split("img loading").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.contains("data-src=\"")
{
true => video_segment.split("img loading").collect::<Vec<&str>>()[1]
true => video_segment.split("img loading").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("data-src=\"")
.collect::<Vec<&str>>()[1]
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string(),
false => video_segment.split("img loading").collect::<Vec<&str>>()[1]
false => video_segment.split("img loading").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("data-original=\"")
.collect::<Vec<&str>>()[1]
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string(),
};
let raw_duration = video_segment
.split("<span class=\"duration\">")
.collect::<Vec<&str>>()[1]
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.split(" ")
.collect::<Vec<&str>>()
.last()
@@ -507,9 +577,9 @@ impl OmgxxxProvider {
let views = parse_abbreviated_number(
video_segment
.split("<div class=\"views\">")
.collect::<Vec<&str>>()[1]
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string()
.as_str(),
)
@@ -517,9 +587,9 @@ impl OmgxxxProvider {
let preview = video_segment
.split("data-preview=\"")
.collect::<Vec<&str>>()[1]
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
let site_name = title
.split("]")
@@ -533,9 +603,9 @@ impl OmgxxxProvider {
let mut tags = match video_segment.contains("class=\"models\">") {
true => video_segment
.split("class=\"models\">")
.collect::<Vec<&str>>()[1]
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("</div>")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.split("href=\"")
.collect::<Vec<&str>>()[1..]
.into_iter()
@@ -543,17 +613,17 @@ impl OmgxxxProvider {
Self::push_unique(
&self.stars,
FilterOption {
id: s.split("/").collect::<Vec<&str>>()[4].to_string(),
title: s.split(">").collect::<Vec<&str>>()[1]
id: s.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string(),
title: s.split(">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.trim()
.to_string(),
},
);
s.split(">").collect::<Vec<&str>>()[1]
s.split(">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.trim()
.to_string()
})

161
src/providers/paradisehill.rs
View File

@@ -1,5 +1,7 @@
use crate::api::ClientVersion;
use crate::DbPool;
use crate::providers::Provider;
use crate::status::*;
use crate::util::cache::VideoCache;
use crate::util::requester::Requester;
use crate::videos::VideoItem;
@@ -28,13 +30,28 @@ impl ParadisehillProvider {
url: "https://en.paradisehill.cc".to_string(),
}
}
fn build_channel(&self, _clientversion: ClientVersion) -> Channel {
Channel {
id: "paradisehill".to_string(),
name: "Paradisehill".to_string(),
description: "Porn Movies on Paradise Hill".to_string(),
premium: false,
favicon: "https://www.google.com/s2/favicons?sz=64&domain=en.paradisehill.cc".to_string(),
status: "active".to_string(),
categories: vec![],
options: vec![],
nsfw: true,
cacheDuration: None,
}
}
async fn get(
&self,
cache: VideoCache,
page: u8,
options: ServerOptions,
) -> Result<Vec<VideoItem>> {
let mut requester = options.requester.clone().unwrap();
let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let url_str = format!("{}/all/?sort=created_at&page={}", self.url, page);
@@ -51,7 +68,18 @@ impl ParadisehillProvider {
}
};
let text = requester.get(&url_str, None).await.unwrap();
let text = match requester.get(&url_str, None).await {
Ok(text) => text,
Err(e) => {
crate::providers::report_provider_error(
"paradisehill",
"get.request",
&format!("url={url_str}; error={e}"),
)
.await;
return Ok(old_items);
}
};
// Pass a reference to options if needed, or reconstruct as needed
let video_items: Vec<VideoItem> = self
.get_video_items_from_html(text.clone(), requester)
@@ -73,7 +101,7 @@ impl ParadisehillProvider {
options: ServerOptions,
) -> Result<Vec<VideoItem>> {
// Extract needed fields from options at the start
let mut requester = options.requester.clone().unwrap();
let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let search_string = query.replace(" ", "+");
let url_str = format!(
"{}/search/?pattern={}&page={}",
@@ -93,7 +121,18 @@ impl ParadisehillProvider {
vec![]
}
};
let text = requester.get(&url_str, None).await.unwrap();
let text = match requester.get(&url_str, None).await {
Ok(text) => text,
Err(e) => {
crate::providers::report_provider_error(
"paradisehill",
"query.request",
&format!("url={url_str}; error={e}"),
)
.await;
return Ok(old_items);
}
};
let video_items: Vec<VideoItem> = self
.get_video_items_from_html(text.clone(), requester)
.await;
@@ -109,46 +148,90 @@ impl ParadisehillProvider {
async fn get_video_items_from_html(
&self,
html: String,
requester: Requester,
_requester: Requester,
) -> Vec<VideoItem> {
if html.is_empty() {
println!("HTML is empty");
return vec![];
}
let raw_videos = html.split("item list-film-item").collect::<Vec<&str>>()[1..].to_vec();
let mut urls: Vec<String> = vec![];
for video_segment in &raw_videos {
// let vid = video_segment.split("\n").collect::<Vec<&str>>();
// for (index, line) in vid.iter().enumerate() {
// println!("Line {}: {}", index, line.to_string().trim());
// }
let mut items: Vec<VideoItem> = Vec::new();
for video_segment in html.split("item list-film-item").skip(1) {
let href = video_segment
.split("<a href=\"")
.nth(1)
.and_then(|s| s.split('"').next())
.unwrap_or_default();
if href.is_empty() {
continue;
}
let url_str = format!(
"{}{}",
self.url,
video_segment.split("<a href=\"").collect::<Vec<&str>>()[1]
.split("\"")
.collect::<Vec<&str>>()[0]
.to_string()
let video_url = format!("{}{}", self.url, href);
let id = href
.trim_matches('/')
.split('/')
.next()
.unwrap_or_default()
.to_string();
if id.is_empty() {
continue;
}
let mut title = video_segment
.split("itemprop=\"name\">")
.nth(1)
.and_then(|s| s.split('<').next())
.unwrap_or_default()
.trim()
.to_string();
title = decode(title.as_bytes()).to_string().unwrap_or(title);
let mut thumb = video_segment
.split("itemprop=\"image\" src=\"")
.nth(1)
.and_then(|s| s.split('"').next())
.unwrap_or_default()
.to_string();
if thumb.starts_with('/') {
thumb = format!("{}{}", self.url, thumb);
}
let genre = video_segment
.split("itemprop=\"genre\">")
.nth(1)
.and_then(|s| s.split('<').next())
.unwrap_or_default()
.trim()
.to_string();
let tags = if genre.is_empty() { vec![] } else { vec![genre] };
items.push(
VideoItem::new(id, title, video_url, "paradisehill".to_string(), thumb, 0)
.aspect_ratio(0.697674419 as f32)
.tags(tags),
);
urls.push(url_str.clone());
}
let futures = urls
.into_iter()
.map(|el| self.get_video_item(el.clone(), requester.clone()));
let results: Vec<Result<VideoItem>> = join_all(futures).await;
let video_items: Vec<VideoItem> = results.into_iter().filter_map(Result::ok).collect();
return video_items;
items
}
async fn get_video_item(&self, url_str: String, mut requester: Requester) -> Result<VideoItem> {
let vid = requester.get(&url_str, None).await.unwrap();
let vid = match requester.get(&url_str, None).await {
Ok(vid) => vid,
Err(e) => {
crate::providers::report_provider_error(
"paradisehill",
"get_video_item.request",
&format!("url={url_str}; error={e}"),
)
.await;
return Err(Error::from(e.to_string()));
}
};
let mut title = vid
.split("<meta property=\"og:title\" content=\"")
.collect::<Vec<&str>>()[1]
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.trim()
.to_string();
title = decode(title.as_bytes()).to_string().unwrap_or(title);
@@ -156,27 +239,27 @@ impl ParadisehillProvider {
"{}{}",
self.url,
vid.split("<meta property=\"og:image\" content=\"")
.collect::<Vec<&str>>()[1]
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string()
);
let video_urls = vid.split("var videoList = ").collect::<Vec<&str>>()[1]
let video_urls = vid.split("var videoList = ").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"src\":\"")
.collect::<Vec<&str>>()[1..].to_vec();
let mut formats = vec![];
for url in video_urls {
let video_url = url
.split("\"")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.replace("\\", "")
.to_string();
let format =
videos::VideoFormat::new(video_url.clone(), "1080".to_string(), "mp4".to_string())
// .protocol("https".to_string())
.format_id(video_url.split("/").last().unwrap().to_string())
.format_note(format!("{}", video_url.split("_").last().unwrap().replace(".mp4", "").to_string()))
.format_id(video_url.split("/").last().unwrap_or_default().to_string())
.format_note(video_url.split("_").last().unwrap_or_default().replace(".mp4", ""))
;
formats.push(format);
}
@@ -184,9 +267,9 @@ impl ParadisehillProvider {
formats.reverse();
let id = url_str
.split("/")
.collect::<Vec<&str>>()[3]
.collect::<Vec<&str>>().get(3).copied().unwrap_or_default()
.split("_")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
let video_item = VideoItem::new(
@@ -237,4 +320,8 @@ impl Provider for ParadisehillProvider {
}
}
}
fn get_channel(&self, clientversion: ClientVersion) -> Option<Channel> {
Some(self.build_channel(clientversion))
}
}

157
src/providers/perfectgirls.rs
View File

@@ -1,6 +1,8 @@
use crate::api::ClientVersion;
use crate::util::parse_abbreviated_number;
use crate::DbPool;
use crate::providers::Provider;
use crate::providers::{Provider, report_provider_error};
use crate::status::*;
use crate::util::cache::VideoCache;
use crate::util::flaresolverr::{FlareSolverrRequest, Flaresolverr};
use crate::util::time::parse_time_to_seconds;
@@ -30,6 +32,42 @@ impl PerfectgirlsProvider {
url: "https://www.perfectgirls.xxx".to_string(),
}
}
fn build_channel(&self, _clientversion: ClientVersion) -> Channel {
Channel {
id: "perfectgirls".to_string(),
name: "Perfectgirls".to_string(),
description: "Perfect Girls Tube".to_string(),
premium: false,
favicon: "https://www.google.com/s2/favicons?sz=64&domain=perfectgirls.xxx".to_string(),
status: "active".to_string(),
categories: vec![],
options: vec![ChannelOption {
id: "sort".to_string(),
title: "Sort".to_string(),
description: "Sort the Videos".to_string(),
systemImage: "list.number".to_string(),
colorName: "blue".to_string(),
options: vec![
FilterOption {
id: "new".to_string(),
title: "New".to_string(),
},
FilterOption {
id: "popular".to_string(),
title: "Popular".to_string(),
},
FilterOption {
id: "trending".to_string(),
title: "Trending".to_string(),
},
],
multiSelect: false,
}],
nsfw: true,
cacheDuration: Some(1800),
}
}
async fn get(
&self,
cache: VideoCache,
@@ -61,11 +99,25 @@ impl PerfectgirlsProvider {
let mut response = client.get(video_url.clone())
// .proxy(proxy.clone())
.send().await?;
if response.status().is_redirection(){
println!("Redirection detected, following to: {}", response.headers()["Location"].to_str().unwrap());
response = client.get(response.headers()["Location"].to_str().unwrap())
// .proxy(proxy.clone())
.send().await?;
if response.status().is_redirection() {
let location = match response.headers().get("Location").and_then(|h| h.to_str().ok()) {
Some(location) => location,
None => {
report_provider_error(
"perfectgirls",
"get.redirect_location",
&format!("url={video_url}; missing/invalid Location header"),
)
.await;
return Ok(old_items);
}
};
println!("Redirection detected, following to: {}", location);
response = client
.get(location)
// .proxy(proxy.clone())
.send()
.await?;
}
if response.status().is_success() {
let text = response.text().await?;
@@ -78,7 +130,18 @@ impl PerfectgirlsProvider {
}
Ok(video_items)
} else {
let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set");
let flare_url = match env::var("FLARE_URL") {
Ok(url) => url,
Err(e) => {
report_provider_error(
"perfectgirls",
"get.flare_url",
&e.to_string(),
)
.await;
return Ok(old_items);
}
};
let flare = Flaresolverr::new(flare_url);
let result = flare
.solve(FlareSolverrRequest {
@@ -116,7 +179,7 @@ impl PerfectgirlsProvider {
let mut video_url = format!("{}/search/{}/{}/", self.url, search_string, page);
if search_string.starts_with("@"){
let url_part = search_string.split("@").collect::<Vec<&str>>()[1].replace(":", "/");
let url_part = search_string.split("@").collect::<Vec<&str>>().get(1).copied().unwrap_or_default().replace(":", "/");
video_url = format!("{}/{}/", self.url, url_part);
}
// Check our Video Cache. If the result is younger than 1 hour, we return it.
@@ -141,11 +204,24 @@ impl PerfectgirlsProvider {
// .proxy(proxy.clone())
.send().await?;
if response.status().is_redirection(){
response = client.get(self.url.clone() + response.headers()["Location"].to_str().unwrap())
// .proxy(proxy.clone())
.send().await?;
if response.status().is_redirection() {
let location = match response.headers().get("Location").and_then(|h| h.to_str().ok()) {
Some(location) => location,
None => {
report_provider_error(
"perfectgirls",
"query.redirect_location",
&format!("url={video_url}; missing/invalid Location header"),
)
.await;
return Ok(old_items);
}
};
response = client
.get(self.url.clone() + location)
// .proxy(proxy.clone())
.send()
.await?;
}
if response.status().is_success() {
@@ -159,7 +235,18 @@ impl PerfectgirlsProvider {
}
Ok(video_items)
} else {
let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set");
let flare_url = match env::var("FLARE_URL") {
Ok(url) => url,
Err(e) => {
report_provider_error(
"perfectgirls",
"query.flare_url",
&e.to_string(),
)
.await;
return Ok(old_items);
}
};
let flare = Flaresolverr::new(flare_url);
let result = flare
.solve(FlareSolverrRequest {
@@ -191,7 +278,7 @@ impl PerfectgirlsProvider {
return vec![];
}
let mut items: Vec<VideoItem> = Vec::new();
let raw_videos = html.split("<div class=\"pagination\"").collect::<Vec<&str>>()[0]
let raw_videos = html.split("<div class=\"pagination\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.split("item thumb-bl thumb-bl-video video_")
.collect::<Vec<&str>>()[1..]
.to_vec();
@@ -200,31 +287,31 @@ impl PerfectgirlsProvider {
// for (index, line) in vid.iter().enumerate() {
// println!("Line {}: {}", index, line);
// }
let video_url: String = format!("{}{}", self.url, video_segment.split("<a href=\"").collect::<Vec<&str>>()[1]
let video_url: String = format!("{}{}", self.url, video_segment.split("<a href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0]);
let preview_url = video_segment.split("data-preview-custom=\"").collect::<Vec<&str>>()[1]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default());
let preview_url = video_segment.split("data-preview-custom=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
let mut title = video_segment.split("\" title=\"").collect::<Vec<&str>>()[1]
let mut title = video_segment.split("\" title=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
// html decode
title = decode(title.as_bytes()).to_string().unwrap_or(title);
let id = video_url.split("/").collect::<Vec<&str>>()[4].to_string();
let raw_duration = video_segment.split("fa fa-clock-o").collect::<Vec<&str>>()[1]
.split("<span>").collect::<Vec<&str>>()[1]
let id = video_url.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string();
let raw_duration = video_segment.split("fa fa-clock-o").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<span>").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
let mut thumb = video_segment.split(" class=\"thumb lazy-load\"").collect::<Vec<&str>>()[1]
.split("data-original=\"").collect::<Vec<&str>>()[1]
let mut thumb = video_segment.split(" class=\"thumb lazy-load\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("data-original=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
if thumb.starts_with("//"){
thumb = format!("https:{}",thumb);
@@ -234,7 +321,7 @@ impl PerfectgirlsProvider {
if video_segment.contains("href=\"/channels/"){
let raw_tags = video_segment.split("href=\"/channels/").collect::<Vec<&str>>()[1..]
.iter()
.map(|s| s.split("/\"").collect::<Vec<&str>>()[0].to_string())
.map(|s| s.split("/\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default().to_string())
.collect::<Vec<String>>();
for tag in raw_tags {
if !tag.is_empty() {
@@ -245,7 +332,7 @@ impl PerfectgirlsProvider {
if video_segment.contains("href=\"/pornstars/"){
let raw_tags = video_segment.split("href=\"/pornstars/").collect::<Vec<&str>>()[1..]
.iter()
.map(|s| s.split("/\"").collect::<Vec<&str>>()[0].to_string())
.map(|s| s.split("/\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default().to_string())
.collect::<Vec<String>>();
for tag in raw_tags {
if !tag.is_empty() {
@@ -254,10 +341,10 @@ impl PerfectgirlsProvider {
}
}
let views_part = video_segment.split("fa fa-eye").collect::<Vec<&str>>()[1]
.split("<span>").collect::<Vec<&str>>()[1]
let views_part = video_segment.split("fa fa-eye").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<span>").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
let views = parse_abbreviated_number(&views_part).unwrap_or(0) as u32;
@@ -314,4 +401,8 @@ impl Provider for PerfectgirlsProvider {
}
}
}
fn get_channel(&self, clientversion: ClientVersion) -> Option<Channel> {
Some(self.build_channel(clientversion))
}
}

253
src/providers/perverzija.rs
View File

@@ -1,6 +1,8 @@
use crate::DbPool;
use crate::api::ClientVersion;
use crate::db;
use crate::providers::Provider;
use crate::providers::{Provider, report_provider_error, report_provider_error_background};
use crate::status::*;
use crate::util::cache::VideoCache;
use crate::util::time::parse_time_to_seconds;
use crate::videos::ServerOptions;
@@ -40,6 +42,42 @@ impl PerverzijaProvider {
url: "https://tube.perverzija.com/".to_string(),
}
}
fn build_channel(&self, clientversion: ClientVersion) -> Channel {
let _ = clientversion;
Channel {
id: "perverzija".to_string(),
name: "Perverzija".to_string(),
description: "Free videos from Perverzija".to_string(),
premium: false,
favicon: "https://www.google.com/s2/favicons?sz=64&domain=tube.perverzija.com"
.to_string(),
status: "active".to_string(),
categories: vec![],
options: vec![ChannelOption {
id: "featured".to_string(),
title: "Featured".to_string(),
description: "Filter Featured Videos.".to_string(),
systemImage: "star".to_string(),
colorName: "red".to_string(),
options: vec![
FilterOption {
id: "all".to_string(),
title: "No".to_string(),
},
FilterOption {
id: "featured".to_string(),
title: "Yes".to_string(),
},
],
multiSelect: false,
}],
nsfw: true,
cacheDuration: None,
}
}
async fn get(
&self,
cache: VideoCache,
@@ -47,7 +85,7 @@ impl PerverzijaProvider {
page: u8,
options: ServerOptions,
) -> Result<Vec<VideoItem>> {
let featured = options.featured.unwrap_or("".to_string());
let featured = options.featured.clone().unwrap_or("".to_string());
let mut prefix_uri = "".to_string();
if featured == "featured" {
prefix_uri = "featured-scenes/".to_string();
@@ -71,8 +109,20 @@ impl PerverzijaProvider {
}
};
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&url_str, Some(Version::HTTP_2)).await.unwrap();
let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let text = match requester.get(&url_str, Some(Version::HTTP_2)).await {
Ok(text) => text,
Err(e) => {
report_provider_error(
"perverzija",
"get.request",
&format!("url={url_str}; error={e}"),
)
.await;
return Ok(old_items);
}
};
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone(), pool);
if !video_items.is_empty() {
cache.remove(&url_str);
@@ -122,8 +172,20 @@ impl PerverzijaProvider {
}
};
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&url_str, Some(Version::HTTP_2)).await.unwrap();
let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let text = match requester.get(&url_str, Some(Version::HTTP_2)).await {
Ok(text) => text,
Err(e) => {
report_provider_error(
"perverzija",
"query.request",
&format!("url={url_str}; error={e}"),
)
.await;
return Ok(old_items);
}
};
let video_items: Vec<VideoItem> = match query_parse {
true => {
self.get_video_items_from_html_query(text.clone(), pool)
@@ -146,51 +208,61 @@ impl PerverzijaProvider {
return vec![];
}
let mut items: Vec<VideoItem> = Vec::new();
let video_listing_content = html.split("video-listing-content").collect::<Vec<&str>>()[1];
let video_listing_content = html.split("video-listing-content").collect::<Vec<&str>>().get(1).copied().unwrap_or_default();
let raw_videos = video_listing_content
.split("video-item post")
.collect::<Vec<&str>>()[1..]
.to_vec();
for video_segment in &raw_videos {
let vid = video_segment.split("\n").collect::<Vec<&str>>();
if vid.len() > 20 {
if vid.len() > 20 || vid.len() < 8 {
report_provider_error_background(
"perverzija",
"get_video_items_from_html.snippet_shape",
&format!("unexpected snippet length={}", vid.len()),
);
continue;
}
let line0 = vid.get(0).copied().unwrap_or_default();
let line1 = vid.get(1).copied().unwrap_or_default();
let line4 = vid.get(4).copied().unwrap_or_default();
let line6 = vid.get(6).copied().unwrap_or_default();
let line7 = vid.get(7).copied().unwrap_or_default();
// for (index, line) in vid.iter().enumerate() {
// println!("Line {}: {}", index, line.to_string().trim());
// }
let mut title = vid[1].split(">").collect::<Vec<&str>>()[1]
let mut title = line1.split(">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
// html decode
title = decode(title.as_bytes()).to_string().unwrap_or(title);
if !vid[1].contains("iframe src=&quot;") {
if !line1.contains("iframe src=&quot;") {
continue;
}
let url_str = vid[1].split("iframe src=&quot;").collect::<Vec<&str>>()[1]
let url_str = line1.split("iframe src=&quot;").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("&quot;")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string()
.replace("index.php", "xs1.php");
if url_str.starts_with("https://streamtape.com/") {
continue; // Skip Streamtape links
}
let id = url_str.split("data=").collect::<Vec<&str>>()[1]
let id = url_str.split("data=").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("&")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
let raw_duration = match vid.len() {
10 => vid[6].split("time_dur\">").collect::<Vec<&str>>()[1]
10 => line6.split("time_dur\">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string(),
_ => "00:00".to_string(),
};
let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
if !vid[4].contains("srcset=")
&& vid[4].split("src=\"").collect::<Vec<&str>>().len() == 1
if !line4.contains("srcset=")
&& line4.split("src=\"").collect::<Vec<&str>>().len() == 1
{
for (index, line) in vid.iter().enumerate() {
println!("Line {}: {}\n\n", index, line);
@@ -201,45 +273,54 @@ impl PerverzijaProvider {
for v in vid.clone() {
let line = v.trim();
if line.starts_with("<img ") {
thumb = line.split(" src=\"").collect::<Vec<&str>>()[1]
thumb = line.split(" src=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
}
}
let embed_html = vid[1].split("data-embed='").collect::<Vec<&str>>()[1]
let embed_html = line1.split("data-embed='").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("'")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
let id_url = vid[1].split("data-url='").collect::<Vec<&str>>()[1]
let id_url = line1.split("data-url='").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("'")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
let mut conn = pool.get().expect("couldn't get db connection from pool");
let _ = db::insert_video(&mut conn, &id_url, &url_str);
drop(conn);
match pool.get() {
Ok(mut conn) => {
let _ = db::insert_video(&mut conn, &id_url, &url_str);
}
Err(e) => {
report_provider_error_background(
"perverzija",
"get_video_items_from_html.insert_video.pool_get",
&e.to_string(),
);
}
}
let referer_url = "https://xtremestream.xyz/".to_string();
let embed = VideoEmbed::new(embed_html, url_str.clone());
let mut tags: Vec<String> = Vec::new(); // Placeholder for tags, adjust as needed
let studios_parts = vid[7].split("a href=\"").collect::<Vec<&str>>();
let studios_parts = line7.split("a href=\"").collect::<Vec<&str>>();
for studio in studios_parts.iter().skip(1) {
if studio.starts_with("https://tube.perverzija.com/studio/") {
tags.push(
studio.split("/\"").collect::<Vec<&str>>()[0]
studio.split("/\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.replace("https://tube.perverzija.com/studio/", "@studio:")
.to_string(),
);
}
}
for tag in vid[0].split(" ").collect::<Vec<&str>>() {
for tag in line0.split(" ").collect::<Vec<&str>>() {
if tag.starts_with("stars-") {
let tag_name = tag.split("stars-").collect::<Vec<&str>>()[1]
let tag_name = tag.split("stars-").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
if !tag_name.is_empty() {
tags.push(format!("@stars:{}", tag_name));
@@ -247,9 +328,9 @@ impl PerverzijaProvider {
}
}
for tag in vid[0].split(" ").collect::<Vec<&str>>() {
for tag in line0.split(" ").collect::<Vec<&str>>() {
if tag.starts_with("tag-") {
let tag_name = tag.split("tag-").collect::<Vec<&str>>()[1].to_string();
let tag_name = tag.split("tag-").collect::<Vec<&str>>().get(1).copied().unwrap_or_default().to_string();
if !tag_name.is_empty() {
tags.push(tag_name.replace("-", " ").to_string());
}
@@ -292,37 +373,55 @@ impl PerverzijaProvider {
async fn get_video_item(&self, snippet: &str, pool: DbPool) -> Result<VideoItem> {
let vid = snippet.split("\n").collect::<Vec<&str>>();
if vid.len() > 30 {
if vid.len() > 30 || vid.len() < 7 {
report_provider_error_background(
"perverzija",
"get_video_item.snippet_shape",
&format!("unexpected snippet length={}", vid.len()),
);
return Err("Unexpected video snippet length".into());
}
let line5 = vid.get(5).copied().unwrap_or_default();
let line6 = vid.get(6).copied().unwrap_or_default();
let mut title = vid[5].split(" title=\"").collect::<Vec<&str>>()[1]
let mut title = line5.split(" title=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
title = decode(title.as_bytes()).to_string().unwrap_or(title);
let thumb = match vid[6].split(" src=\"").collect::<Vec<&str>>().len() {
let thumb = match line6.split(" src=\"").collect::<Vec<&str>>().len() {
1 => {
for (index, line) in vid.iter().enumerate() {
println!("Line {}: {}", index, line.to_string().trim());
}
return Err("Failed to parse thumbnail URL".into());
}
_ => vid[6].split(" src=\"").collect::<Vec<&str>>()[1]
_ => line6.split(" src=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string(),
};
let duration = 0;
let lookup_url = vid[5].split(" href=\"").collect::<Vec<&str>>()[1]
let lookup_url = line5.split(" href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
let referer_url = "https://xtremestream.xyz/".to_string();
let mut conn = pool.get().expect("couldn't get db connection from pool");
let mut conn = match pool.get() {
Ok(conn) => conn,
Err(e) => {
report_provider_error(
"perverzija",
"get_video_item.pool_get",
&e.to_string(),
)
.await;
return Err("couldn't get db connection from pool".into());
}
};
let db_result = db::get_video(&mut conn, lookup_url.clone());
match db_result {
Ok(Some(entry)) => {
@@ -334,9 +433,9 @@ impl PerverzijaProvider {
if url_str.starts_with("!") {
return Err("Video was removed".into());
}
let mut id = url_str.split("data=").collect::<Vec<&str>>()[1].to_string();
let mut id = url_str.split("data=").collect::<Vec<&str>>().get(1).copied().unwrap_or_default().to_string();
if id.contains("&") {
id = id.split("&").collect::<Vec<&str>>()[0].to_string()
id = id.split("&").collect::<Vec<&str>>().get(0).copied().unwrap_or_default().to_string()
}
let mut video_item = VideoItem::new(
id,
@@ -382,9 +481,9 @@ impl PerverzijaProvider {
}
};
let mut url_str = text.split("<iframe src=\"").collect::<Vec<&str>>()[1]
let mut url_str = text.split("<iframe src=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string()
.replace("index.php", "xs1.php");
if !url_str.contains("xtremestream.xyz") {
@@ -395,15 +494,15 @@ impl PerverzijaProvider {
let studios_parts = text
.split("<strong>Studio: </strong>")
.collect::<Vec<&str>>()[1]
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("</div>")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.split("<a href=\"")
.collect::<Vec<&str>>();
for studio in studios_parts.iter().skip(1) {
if studio.starts_with("https://tube.perverzija.com/studio/") {
tags.push(
studio.split("/\"").collect::<Vec<&str>>()[0]
studio.split("/\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.replace("https://tube.perverzija.com/studio/", "@studio:")
.to_string(),
);
@@ -412,15 +511,15 @@ impl PerverzijaProvider {
if text.contains("<strong>Stars: </strong>") {
let stars_parts: Vec<&str> = text
.split("<strong>Stars: </strong>")
.collect::<Vec<&str>>()[1]
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("</div>")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.split("<a href=\"")
.collect::<Vec<&str>>();
for star in stars_parts.iter().skip(1) {
if star.starts_with("https://tube.perverzija.com/stars/") {
tags.push(
star.split("/\"").collect::<Vec<&str>>()[0]
star.split("/\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.replace("https://tube.perverzija.com/stars/", "@stars:")
.to_string(),
);
@@ -428,15 +527,15 @@ impl PerverzijaProvider {
}
}
let tags_parts: Vec<&str> = text.split("<strong>Tags: </strong>").collect::<Vec<&str>>()[1]
let tags_parts: Vec<&str> = text.split("<strong>Tags: </strong>").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("</div>")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.split("<a href=\"")
.collect::<Vec<&str>>();
for star in tags_parts.iter().skip(1) {
if star.starts_with("https://tube.perverzija.com/stars/") {
tags.push(
star.split("/\"").collect::<Vec<&str>>()[0]
star.split("/\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.replace("https://tube.perverzija.com/stars/", "@stars:")
.to_string(),
);
@@ -447,25 +546,37 @@ impl PerverzijaProvider {
url_string: url_str.clone(),
tags_strings: tags.clone(),
};
let mut conn = pool.get().expect("couldn't get db connection from pool");
let insert_result = db::insert_video(
&mut conn,
&lookup_url,
&serde_json::to_string(&perverzija_db_entry)?,
);
match insert_result {
Ok(_) => (),
match pool.get() {
Ok(mut conn) => {
let insert_result = db::insert_video(
&mut conn,
&lookup_url,
&serde_json::to_string(&perverzija_db_entry)?,
);
if let Err(e) = insert_result {
report_provider_error(
"perverzija",
"get_video_item.insert_video",
&e.to_string(),
)
.await;
}
}
Err(e) => {
println!("{:?}", e);
report_provider_error(
"perverzija",
"get_video_item.insert_video.pool_get",
&e.to_string(),
)
.await;
}
}
drop(conn);
if !url_str.contains("xtremestream.xyz") {
return Err("Video URL does not contain xtremestream.xyz".into());
}
let mut id = url_str.split("data=").collect::<Vec<&str>>()[1].to_string();
let mut id = url_str.split("data=").collect::<Vec<&str>>().get(1).copied().unwrap_or_default().to_string();
if id.contains("&") {
id = id.split("&").collect::<Vec<&str>>()[0].to_string()
id = id.split("&").collect::<Vec<&str>>().get(0).copied().unwrap_or_default().to_string()
}
// if !vid[6].contains(" src=\""){
// for (index,line) in vid.iter().enumerate() {
@@ -530,4 +641,8 @@ impl Provider for PerverzijaProvider {
}
}
}
fn get_channel(&self, clientversion: ClientVersion) -> Option<Channel> {
Some(self.build_channel(clientversion))
}
}

95
src/providers/pimpbunny.rs
View File

@@ -61,10 +61,15 @@ impl PimpbunnyProvider {
categories: self
.categories
.read()
.unwrap()
.iter()
.map(|c| c.title.clone())
.collect(),
.map(|categories| categories.iter().map(|c| c.title.clone()).collect())
.unwrap_or_else(|e| {
crate::providers::report_provider_error_background(
"pimpbunny",
"build_channel.categories_read",
&e.to_string(),
);
vec![]
}),
options: vec![ChannelOption {
id: "sort".to_string(),
title: "Sort".to_string(),
@@ -267,8 +272,20 @@ impl PimpbunnyProvider {
vec![]
}
};
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url, Some(Version::HTTP_11)).await.unwrap();
let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let text = match requester.get(&video_url, Some(Version::HTTP_11)).await {
Ok(text) => text,
Err(e) => {
crate::providers::report_provider_error(
"pimpbunny",
"get.request",
&format!("url={video_url}; error={e}"),
)
.await;
return Ok(old_items);
}
};
let video_items: Vec<VideoItem> = self
.get_video_items_from_html(text.clone(), &mut requester)
.await;
@@ -300,28 +317,38 @@ impl PimpbunnyProvider {
"most viewed" => "&sort_by=video_viewed",
_ => "&sort_by=post_date",
};
if let Some(star) = self
.stars
.read()
.unwrap()
.iter()
.find(|s| s.title.to_ascii_lowercase() == search_string.to_ascii_lowercase())
{
video_url = format!(
"{}/onlyfans-models/{}/{}/?videos_per_page=20{}",
self.url, star.id, page, sort_string
if let Ok(stars) = self.stars.read() {
if let Some(star) = stars
.iter()
.find(|s| s.title.to_ascii_lowercase() == search_string.to_ascii_lowercase())
{
video_url = format!(
"{}/onlyfans-models/{}/{}/?videos_per_page=20{}",
self.url, star.id, page, sort_string
);
}
} else {
crate::providers::report_provider_error_background(
"pimpbunny",
"query.stars_read",
"failed to lock stars",
);
}
if let Some(cat) = self
.categories
.read()
.unwrap()
.iter()
.find(|c| c.title.to_ascii_lowercase() == search_string.to_ascii_lowercase())
{
video_url = format!(
"{}/categories/{}/{}/?videos_per_page=20{}",
self.url, cat.id, page, sort_string
if let Ok(categories) = self.categories.read() {
if let Some(cat) = categories
.iter()
.find(|c| c.title.to_ascii_lowercase() == search_string.to_ascii_lowercase())
{
video_url = format!(
"{}/categories/{}/{}/?videos_per_page=20{}",
self.url, cat.id, page, sort_string
);
}
} else {
crate::providers::report_provider_error_background(
"pimpbunny",
"query.categories_read",
"failed to lock categories",
);
}
// Check our Video Cache. If the result is younger than 1 hour, we return it.
@@ -339,9 +366,21 @@ impl PimpbunnyProvider {
}
};
let mut requester = options.requester.clone().unwrap();
let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
println!("Fetching URL: {}", video_url);
let text = requester.get(&video_url, Some(Version::HTTP_2)).await.unwrap();
let text = match requester.get(&video_url, Some(Version::HTTP_2)).await {
Ok(text) => text,
Err(e) => {
crate::providers::report_provider_error(
"pimpbunny",
"query.request",
&format!("url={video_url}; error={e}"),
)
.await;
return Ok(old_items);
}
};
let video_items: Vec<VideoItem> = self
.get_video_items_from_html(text.clone(), &mut requester)
.await;

55
src/providers/pmvhaven.rs
View File

@@ -5,7 +5,7 @@ use crate::status::*;
use crate::util::cache::VideoCache;
use crate::util::discord::send_discord_error_report;
use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoItem};
use crate::videos::{ServerOptions, VideoFormat, VideoItem};
use async_trait::async_trait;
use error_chain::error_chain;
use htmlentity::entity::{decode, ICodedDataTrait};
@@ -132,6 +132,40 @@ impl PmvhavenProvider {
}
}
fn is_direct_media_url(url: &str) -> bool {
let lower = url.to_ascii_lowercase();
(lower.starts_with("http://") || lower.starts_with("https://"))
&& (lower.contains("/videos/")
|| lower.contains(".mp4")
|| lower.contains(".m3u8"))
}
fn pick_downloadable_media_url(&self, video: &serde_json::Value) -> Option<String> {
let video_url = video
.get("videoUrl")
.and_then(|v| v.as_str())
.unwrap_or("")
.trim();
if Self::is_direct_media_url(video_url) {
return Some(video_url.replace(' ', "%20"));
}
// Fallback: derive direct media URL from object key.
let key = video
.get("key")
.and_then(|v| v.as_str())
.unwrap_or("")
.trim_matches('/');
if !key.is_empty() {
let rebuilt = format!("https://video.pmvhaven.com/{key}");
if Self::is_direct_media_url(&rebuilt) {
return Some(rebuilt.replace(' ', "%20"));
}
}
None
}
async fn query(
&self,
cache: VideoCache,
@@ -226,7 +260,12 @@ impl PmvhavenProvider {
.unwrap_or(&title)
.to_string();
let video_url = video.get("videoUrl").and_then(|v| v.as_str()).unwrap_or("").to_string();
let video_url = match self.pick_downloadable_media_url(&video) {
Some(url) => url,
None => {
continue;
}
};
let thumb = video.get("thumbnailUrl").and_then(|v| v.as_str()).unwrap_or("").to_string();
let preview = video.get("previewUrl").and_then(|v| v.as_str()).unwrap_or("").to_string();
@@ -248,9 +287,19 @@ impl PmvhavenProvider {
}
}
let format_type = if video_url.to_ascii_lowercase().contains(".m3u8") {
"m3u8".to_string()
} else {
"mp4".to_string()
};
items.push(
VideoItem::new(id, title, video_url.replace(' ', "%20"), "pmvhaven".into(), thumb, duration as u32)
VideoItem::new(id, title, video_url.clone(), "pmvhaven".into(), thumb, duration as u32)
.views(views as u32)
.formats(vec![VideoFormat::new(
video_url,
"1080".to_string(),
format_type,
)])
.preview(preview)
);
}

102
src/providers/porn00.rs
View File

@@ -1,6 +1,8 @@
use crate::api::ClientVersion;
use crate::util::parse_abbreviated_number;
use crate::DbPool;
use crate::providers::Provider;
use crate::status::*;
use crate::util::cache::VideoCache;
use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoItem};
@@ -26,6 +28,42 @@ impl Porn00Provider {
url: "https://www.porn00.org".to_string(),
}
}
fn build_channel(&self, _clientversion: ClientVersion) -> Channel {
Channel {
id: "porn00".to_string(),
name: "Porn00".to_string(),
description: "HD Porn".to_string(),
premium: false,
favicon: "https://www.google.com/s2/favicons?sz=64&domain=www.porn00.org".to_string(),
status: "active".to_string(),
categories: vec![],
options: vec![ChannelOption {
id: "sort".to_string(),
title: "Sort".to_string(),
description: "Sort the Videos".to_string(),
systemImage: "list.number".to_string(),
colorName: "blue".to_string(),
options: vec![
FilterOption {
id: "new".to_string(),
title: "New".to_string(),
},
FilterOption {
id: "popular".to_string(),
title: "Popular".to_string(),
},
FilterOption {
id: "top-rated".to_string(),
title: "Top Rated".to_string(),
},
],
multiSelect: false,
}],
nsfw: true,
cacheDuration: Some(1800),
}
}
async fn get(
&self,
cache: VideoCache,
@@ -59,9 +97,20 @@ impl Porn00Provider {
}
};
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url, None).await.unwrap();
let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let text = match requester.get(&video_url, None).await {
Ok(text) => text,
Err(e) => {
crate::providers::report_provider_error(
"porn00",
"get.request",
&format!("url={video_url}; error={e}"),
)
.await;
return Ok(old_items);
}
};
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);
@@ -96,9 +145,20 @@ impl Porn00Provider {
}
};
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url, None).await.unwrap();
let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let text = match requester.get(&video_url, None).await {
Ok(text) => text,
Err(e) => {
crate::providers::report_provider_error(
"porn00",
"query.request",
&format!("url={video_url}; error={e}"),
)
.await;
return Ok(old_items);
}
};
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);
@@ -115,7 +175,7 @@ impl Porn00Provider {
return vec![];
}
let mut items: Vec<VideoItem> = Vec::new();
let raw_videos = html.split("<div class=\"pagination\"").collect::<Vec<&str>>()[0]
let raw_videos = html.split("<div class=\"pagination\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.split("<div class=\"item \">")
.collect::<Vec<&str>>()[1..]
.to_vec();
@@ -124,31 +184,31 @@ impl Porn00Provider {
// for (index, line) in vid.iter().enumerate() {
// println!("Line {}: {}", index, line);
// }
let video_url: String = video_segment.split("<a href=\"").collect::<Vec<&str>>()[1]
let video_url: String = video_segment.split("<a href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0].to_string();
let mut title = video_segment.split("\" title=\"").collect::<Vec<&str>>()[1]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default().to_string();
let mut title = video_segment.split("\" title=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
// html decode
title = decode(title.as_bytes()).to_string().unwrap_or(title);
let id = video_url.split("/").collect::<Vec<&str>>()[4].to_string();
let raw_duration = video_segment.split("<div class=\"duration\">").collect::<Vec<&str>>()[1]
let id = video_url.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string();
let raw_duration = video_segment.split("<div class=\"duration\">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
let thumb = video_segment.split("<img class=\"thumb ").collect::<Vec<&str>>()[1]
.split("data-original=\"").collect::<Vec<&str>>()[1]
let thumb = video_segment.split("<img class=\"thumb ").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("data-original=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
let views_part = video_segment.split("<div class=\"views\">").collect::<Vec<&str>>()[1]
let views_part = video_segment.split("<div class=\"views\">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
let views = parse_abbreviated_number(&views_part).unwrap_or(0) as u32;
@@ -202,4 +262,8 @@ impl Provider for Porn00Provider {
}
}
}
fn get_channel(&self, clientversion: ClientVersion) -> Option<Channel> {
Some(self.build_channel(clientversion))
}
}

126
src/providers/pornhat.rs
View File

@@ -1,6 +1,8 @@
use crate::api::ClientVersion;
use crate::util::parse_abbreviated_number;
use crate::DbPool;
use crate::providers::Provider;
use crate::status::*;
use crate::util::cache::VideoCache;
use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoItem};
@@ -26,6 +28,42 @@ impl PornhatProvider {
url: "https://www.pornhat.com".to_string(),
}
}
fn build_channel(&self, _clientversion: ClientVersion) -> Channel {
Channel {
id: "pornhat".to_string(),
name: "Pornhat".to_string(),
description: "free HD porn videos".to_string(),
premium: false,
favicon: "https://www.google.com/s2/favicons?sz=64&domain=pornhat.com".to_string(),
status: "active".to_string(),
categories: vec![],
options: vec![ChannelOption {
id: "sort".to_string(),
title: "Sort".to_string(),
description: "Sort the Videos".to_string(),
systemImage: "list.number".to_string(),
colorName: "blue".to_string(),
options: vec![
FilterOption {
id: "new".to_string(),
title: "New".to_string(),
},
FilterOption {
id: "popular".to_string(),
title: "Popular".to_string(),
},
FilterOption {
id: "trending".to_string(),
title: "Trending".to_string(),
},
],
multiSelect: false,
}],
nsfw: true,
cacheDuration: Some(1800),
}
}
async fn get(
&self,
cache: VideoCache,
@@ -51,8 +89,20 @@ impl PornhatProvider {
vec![]
}
};
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url, None).await.unwrap();
let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let text = match requester.get(&video_url, None).await {
Ok(text) => text,
Err(e) => {
crate::providers::report_provider_error(
"pornhat",
"get.request",
&format!("url={video_url}; error={e}"),
)
.await;
return Ok(old_items);
}
};
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);
@@ -73,7 +123,7 @@ impl PornhatProvider {
let mut video_url = format!("{}/search/{}/{}/", self.url, search_string, page);
if search_string.starts_with("@"){
let url_part = search_string.split("@").collect::<Vec<&str>>()[1].replace(":", "/");
let url_part = search_string.split("@").collect::<Vec<&str>>().get(1).copied().unwrap_or_default().replace(":", "/");
video_url = format!("{}/{}/", self.url, url_part);
}
// Check our Video Cache. If the result is younger than 1 hour, we return it.
@@ -90,16 +140,28 @@ impl PornhatProvider {
vec![]
}
};
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);
cache.insert(video_url.clone(), video_items.clone());
} else {
let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let text = match requester.get(&video_url, None).await {
Ok(text) => text,
Err(e) => {
crate::providers::report_provider_error(
"pornhat",
"query.request",
&format!("url={video_url}; error={e}"),
)
.await;
return Ok(old_items);
}
Ok(video_items)
};
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);
cache.insert(video_url.clone(), video_items.clone());
} else {
return Ok(old_items);
}
Ok(video_items)
}
fn get_video_items_from_html(&self, html: String) -> Vec<VideoItem> {
@@ -108,7 +170,7 @@ impl PornhatProvider {
return vec![];
}
let mut items: Vec<VideoItem> = Vec::new();
let raw_videos = html.split("<div class=\"pagination\"").collect::<Vec<&str>>()[0]
let raw_videos = html.split("<div class=\"pagination\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.split("item thumb-bl thumb-bl-video video_")
.collect::<Vec<&str>>()[1..]
.to_vec();
@@ -117,38 +179,38 @@ impl PornhatProvider {
// for (index, line) in vid.iter().enumerate() {
// println!("Line {}: {}", index, line);
// }
let video_url: String = format!("{}{}", self.url, video_segment.split("<a href=\"").collect::<Vec<&str>>()[1]
let video_url: String = format!("{}{}", self.url, video_segment.split("<a href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0]);
let preview_url = video_segment.split("data-preview-custom=\"").collect::<Vec<&str>>()[1]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default());
let preview_url = video_segment.split("data-preview-custom=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
let mut title = video_segment.split("\" title=\"").collect::<Vec<&str>>()[1]
let mut title = video_segment.split("\" title=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
// html decode
title = decode(title.as_bytes()).to_string().unwrap_or(title);
let id = video_url.split("/").collect::<Vec<&str>>()[4].to_string();
let raw_duration = video_segment.split("fa fa-clock-o").collect::<Vec<&str>>()[1]
.split("<span>").collect::<Vec<&str>>()[1]
let id = video_url.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string();
let raw_duration = video_segment.split("fa fa-clock-o").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<span>").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
let thumb = video_segment.split("<img class=\"thumb lazy-load\"").collect::<Vec<&str>>()[1]
.split("data-original=\"").collect::<Vec<&str>>()[1]
let thumb = video_segment.split("<img class=\"thumb lazy-load\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("data-original=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
let mut tags = vec![];
if video_segment.contains("href=\"/sites/"){
let raw_tags = video_segment.split("href=\"/sites/").collect::<Vec<&str>>()[1..]
.iter()
.map(|s| s.split("/\"").collect::<Vec<&str>>()[0].to_string())
.map(|s| s.split("/\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default().to_string())
.collect::<Vec<String>>();
for tag in raw_tags {
if !tag.is_empty() {
@@ -159,7 +221,7 @@ impl PornhatProvider {
if video_segment.contains("href=\"/models/"){
let raw_tags = video_segment.split("href=\"/models/").collect::<Vec<&str>>()[1..]
.iter()
.map(|s| s.split("/\"").collect::<Vec<&str>>()[0].to_string())
.map(|s| s.split("/\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default().to_string())
.collect::<Vec<String>>();
for tag in raw_tags {
if !tag.is_empty() {
@@ -168,10 +230,10 @@ impl PornhatProvider {
}
}
let views_part = video_segment.split("fa fa-eye").collect::<Vec<&str>>()[1]
.split("<span>").collect::<Vec<&str>>()[1]
let views_part = video_segment.split("fa fa-eye").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<span>").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
let views = parse_abbreviated_number(&views_part).unwrap_or(0) as u32;
@@ -227,4 +289,8 @@ impl Provider for PornhatProvider {
}
}
}
fn get_channel(&self, clientversion: ClientVersion) -> Option<Channel> {
Some(self.build_channel(clientversion))
}
}

56
src/providers/pornhub.rs
View File

@@ -1,6 +1,8 @@
use crate::api::ClientVersion;
use crate::util::parse_abbreviated_number;
use crate::DbPool;
use crate::providers::Provider;
use crate::status::*;
use crate::util::cache::VideoCache;
use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoItem};
@@ -35,6 +37,50 @@ impl PornhubProvider {
}
}
fn build_channel(&self, _clientversion: ClientVersion) -> Channel {
Channel {
id: "pornhub".to_string(),
name: "Pornhub".to_string(),
description: "Pornhub Free Videos".to_string(),
premium: false,
favicon: "https://www.google.com/s2/favicons?sz=64&domain=pornhub.com".to_string(),
status: "active".to_string(),
categories: vec![],
options: vec![ChannelOption {
id: "sort".to_string(),
title: "Sort".to_string(),
description: "Sort the Videos".to_string(),
systemImage: "list.number".to_string(),
colorName: "blue".to_string(),
options: vec![
FilterOption {
id: "mr".to_string(),
title: "Most Recent".to_string(),
},
FilterOption {
id: "mv".to_string(),
title: "Most Viewed".to_string(),
},
FilterOption {
id: "tr".to_string(),
title: "Top Rated".to_string(),
},
FilterOption {
id: "lg".to_string(),
title: "Longest".to_string(),
},
FilterOption {
id: "cm".to_string(),
title: "Newest".to_string(),
},
],
multiSelect: false,
}],
nsfw: true,
cacheDuration: Some(1800),
}
}
async fn get(
&self,
cache: VideoCache,
@@ -216,10 +262,10 @@ impl PornhubProvider {
.unwrap_or("");
let parts: Vec<&str> = href.split('/').collect();
if parts.len() >= 3 {
if let (Some(kind), Some(name)) = (parts.get(1), parts.get(2)) {
(
Some(format!("@{}:{}", parts[1], parts[2].replace('-', " "))),
Some(parts[2].to_string()),
Some(format!("@{}:{}", kind, name.replace('-', " "))),
Some((*name).to_string()),
)
} else {
(None, None)
@@ -291,4 +337,8 @@ impl Provider for PornhubProvider {
vec![]
})
}
fn get_channel(&self, clientversion: ClientVersion) -> Option<Channel> {
Some(self.build_channel(clientversion))
}
}

52
src/providers/pornzog.rs
View File

@@ -1,5 +1,7 @@
use crate::api::ClientVersion;
use crate::DbPool;
use crate::providers::Provider;
use crate::status::*;
use crate::util::cache::VideoCache;
use crate::util::discord::{format_error_chain, send_discord_error_report};
use crate::util::time::parse_time_to_seconds;
@@ -28,6 +30,50 @@ impl PornzogProvider {
}
}
fn build_channel(&self, _clientversion: ClientVersion) -> Channel {
Channel {
id: "pornzog".to_string(),
name: "Pornzog".to_string(),
description: "Watch free porn videos at PornZog Free Porn Clips. More than 1 million videos, watch for free now!".to_string(),
premium: false,
favicon: "https://www.google.com/s2/favicons?sz=64&domain=pornzog.com".to_string(),
status: "active".to_string(),
categories: vec![],
options: vec![ChannelOption {
id: "sort".to_string(),
title: "Sort".to_string(),
description: "Sort the Videos".to_string(),
systemImage: "list.number".to_string(),
colorName: "blue".to_string(),
options: vec![
FilterOption {
id: "recent".to_string(),
title: "Recent".to_string(),
},
FilterOption {
id: "relevance".to_string(),
title: "Relevance".to_string(),
},
FilterOption {
id: "viewed".to_string(),
title: "Most Viewed".to_string(),
},
FilterOption {
id: "rated".to_string(),
title: "Most Rated".to_string(),
},
FilterOption {
id: "longest".to_string(),
title: "Longest".to_string(),
},
],
multiSelect: false,
}],
nsfw: true,
cacheDuration: None,
}
}
async fn query(
&self,
cache: VideoCache,
@@ -206,4 +252,8 @@ impl Provider for PornzogProvider {
}
}
}
}
fn get_channel(&self, clientversion: ClientVersion) -> Option<Channel> {
Some(self.build_channel(clientversion))
}
}

115
src/providers/redtube.rs
View File

@@ -1,5 +1,7 @@
use crate::api::ClientVersion;
use crate::DbPool;
use crate::providers::Provider;
use crate::providers::{Provider, report_provider_error};
use crate::status::*;
use crate::util::cache::VideoCache;
use crate::util::parse_abbreviated_number;
use crate::util::time::parse_time_to_seconds;
@@ -27,6 +29,21 @@ impl RedtubeProvider {
url: "https://www.redtube.com".to_string(),
}
}
fn build_channel(&self, _clientversion: ClientVersion) -> Channel {
Channel {
id: "redtube".to_string(),
name: "Redtube".to_string(),
description: "Redtube brings you NEW porn videos every day for free".to_string(),
premium: false,
favicon: "https://www.google.com/s2/favicons?sz=64&domain=www.redtube.com".to_string(),
status: "active".to_string(),
categories: vec![],
options: vec![],
nsfw: true,
cacheDuration: Some(1800),
}
}
async fn get(
&self,
cache: VideoCache,
@@ -48,8 +65,20 @@ impl RedtubeProvider {
vec![]
}
};
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url, None).await.unwrap();
let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let text = match requester.get(&video_url, None).await {
Ok(text) => text,
Err(e) => {
report_provider_error(
"redtube",
"get.request",
&format!("url={video_url}; error={e}"),
)
.await;
return Ok(old_items);
}
};
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);
@@ -85,8 +114,20 @@ impl RedtubeProvider {
vec![]
}
};
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url, None).await.unwrap();
let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let text = match requester.get(&video_url, None).await {
Ok(text) => text,
Err(e) => {
report_provider_error(
"redtube",
"query.request",
&format!("url={video_url}; error={e}"),
)
.await;
return Ok(old_items);
}
};
let video_items: Vec<VideoItem> = self.get_video_items_from_html_query(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);
@@ -105,22 +146,40 @@ impl RedtubeProvider {
let mut items: Vec<VideoItem> = Vec::new();
let video_listing_content = html
.split("<script type=\"application/ld+json\">")
.collect::<Vec<&str>>()[1]
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("</script>")
.collect::<Vec<&str>>()[0];
let mut videos: Value = serde_json::from_str(video_listing_content).unwrap();
for vid in videos.as_array_mut().unwrap() {
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default();
let mut videos: Value = match serde_json::from_str(video_listing_content) {
Ok(videos) => videos,
Err(e) => {
crate::providers::report_provider_error_background(
"redtube",
"get_video_items_from_html.json_parse",
&e.to_string(),
);
return items;
}
};
let Some(video_list) = videos.as_array_mut() else {
crate::providers::report_provider_error_background(
"redtube",
"get_video_items_from_html.json_not_array",
"expected array",
);
return items;
};
for vid in video_list {
let video_url: String = vid["embedUrl"].as_str().unwrap_or("").to_string();
let mut title: String = vid["name"].as_str().unwrap_or("").to_string();
// html decode
title = decode(title.as_bytes()).to_string().unwrap_or(title);
let id = video_url.split("=").collect::<Vec<&str>>()[1].to_string();
let id = video_url.split("=").collect::<Vec<&str>>().get(1).copied().unwrap_or_default().to_string();
let raw_duration = vid["duration"].as_str().unwrap_or("0");
let duration = raw_duration
.replace("PT", "")
.replace("S", "")
.parse::<u32>()
.unwrap();
.unwrap_or(0);
let views: u64 = vid["interactionCount"].as_u64().unwrap_or(0);
let thumb = vid["thumbnailUrl"].as_str().unwrap_or("").to_string();
@@ -144,7 +203,7 @@ impl RedtubeProvider {
return vec![];
}
let mut items: Vec<VideoItem> = Vec::new();
let video_listing_content = html.split("videos_grid").collect::<Vec<&str>>()[1];
let video_listing_content = html.split("videos_grid").collect::<Vec<&str>>().get(1).copied().unwrap_or_default();
let videos = video_listing_content
.split("<li id=\"tags_videos_")
.collect::<Vec<&str>>()[1..]
@@ -153,43 +212,43 @@ impl RedtubeProvider {
// for (i, c) in vid.split("\n").enumerate() {
// println!("{}: {}", i, c);
// }
let id = vid.split("data-video-id=\"").collect::<Vec<&str>>()[1]
let id = vid.split("data-video-id=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
let video_url = format!("{}/{}", self.url, id);
let title = vid.split(" <a title=\"").collect::<Vec<&str>>()[1]
let title = vid.split(" <a title=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.trim()
.to_string();
let thumb = vid.split("<img").collect::<Vec<&str>>()[1]
let thumb = vid.split("<img").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split(" data-src=\"")
.collect::<Vec<&str>>()[1]
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
let raw_duration = vid
.split("<span class=\"video-properties tm_video_duration\">")
.collect::<Vec<&str>>()[1]
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("</span>")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.trim()
.to_string();
let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
let views_str = vid
.split("<span class='info-views'>")
.collect::<Vec<&str>>()[1]
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("</span>")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.trim()
.to_string();
let views = parse_abbreviated_number(&views_str).unwrap_or(0) as u32;
let preview = vid.split("<img").collect::<Vec<&str>>()[1]
let preview = vid.split("<img").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split(" data-mediabook=\"")
.collect::<Vec<&str>>()[1]
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
let video_item =
@@ -239,4 +298,8 @@ impl Provider for RedtubeProvider {
}
}
}
fn get_channel(&self, clientversion: ClientVersion) -> Option<Channel> {
Some(self.build_channel(clientversion))
}
}

187
src/providers/rule34gen.rs
View File

@@ -2,14 +2,13 @@ use crate::api::*;
use crate::status::*;
use crate::util::parse_abbreviated_number;
use crate::DbPool;
use crate::providers::Provider;
use crate::providers::{Provider, report_provider_error};
use crate::util::cache::VideoCache;
use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoItem};
use error_chain::error_chain;
use htmlentity::entity::{ICodedDataTrait, decode};
use std::vec;
use std::time::{SystemTime, UNIX_EPOCH};
use async_trait::async_trait;
error_chain! {
@@ -82,11 +81,6 @@ fn build_channel(&self, clientversion: ClientVersion) -> Channel {
sort: &str,
options: ServerOptions
) -> Result<Vec<VideoItem>> {
let now = SystemTime::now()
.duration_since(UNIX_EPOCH)
.expect("Time went backwards");
let timestamp_millis = now.as_millis(); // u128
let expected_sorts = vec!["post_date", "video_viewed", "rating", "duration", "pseudo_random"];
let sort = if expected_sorts.contains(&sort) {
sort
@@ -95,8 +89,11 @@ fn build_channel(&self, clientversion: ClientVersion) -> Channel {
};
let index = format!("rule34gen:{}:{}", page, sort);
let url = format!("{}/?mode=async&function=get_block&block_id=custom_list_videos_most_recent_videos&tag_ids=&sort_by={}&from={}&_={}", self.url, sort, page, timestamp_millis);
let url = if page <= 1 {
format!("{}/?sort_by={}", self.url, sort)
} else {
format!("{}/{}/?sort_by={}", self.url, page, sort)
};
let mut old_items: Vec<VideoItem> = vec![];
if !(sort == "pseudo_random") {
@@ -114,8 +111,20 @@ fn build_channel(&self, clientversion: ClientVersion) -> Channel {
}
};
}
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&url, None).await.unwrap();
let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let text = match requester.get(&url, None).await {
Ok(text) => text,
Err(e) => {
report_provider_error(
"rule34gen",
"get.request",
&format!("url={url}; error={e}"),
)
.await;
return Ok(old_items);
}
};
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&url);
@@ -133,10 +142,6 @@ fn build_channel(&self, clientversion: ClientVersion) -> Channel {
sort: &str,
options: ServerOptions
) -> Result<Vec<VideoItem>> {
let now = SystemTime::now()
.duration_since(UNIX_EPOCH)
.expect("Time went backwards");
let timestamp_millis = now.as_millis(); // u128
let expected_sorts = vec!["post_date", "video_viewed", "rating", "duration", "pseudo_random"];
let sort = if expected_sorts.contains(&sort) {
sort
@@ -145,8 +150,12 @@ fn build_channel(&self, clientversion: ClientVersion) -> Channel {
};
let index = format!("rule34gen:{}:{}:{}", page, sort, query);
let url = format!("{}/search/{}/?mode=async&function=get_block&block_id=custom_list_videos_videos_list_search&tag_ids=&sort_by={}&from_videos={}&from_albums={}&_={}", self.url, query.replace(" ","-"), sort, page, page, timestamp_millis);
let search_slug = query.replace(" ", "-");
let url = if page <= 1 {
format!("{}/search/{}/?sort_by={}", self.url, search_slug, sort)
} else {
format!("{}/search/{}/{}/?sort_by={}", self.url, search_slug, page, sort)
};
// Check our Video Cache. If the result is younger than 1 hour, we return it.
let old_items = match cache.get(&index) {
@@ -162,16 +171,28 @@ fn build_channel(&self, clientversion: ClientVersion) -> Channel {
vec![]
}
};
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&url, None).await.unwrap();
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&url);
cache.insert(url.clone(), video_items.clone());
} else {
let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let text = match requester.get(&url, None).await {
Ok(text) => text,
Err(e) => {
report_provider_error(
"rule34gen",
"query.request",
&format!("url={url}; error={e}"),
)
.await;
return Ok(old_items);
}
Ok(video_items)
};
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&url);
cache.insert(url.clone(), video_items.clone());
} else {
return Ok(old_items);
}
Ok(video_items)
}
fn get_video_items_from_html(&self, html: String) -> Vec<VideoItem> {
@@ -180,13 +201,99 @@ fn build_channel(&self, clientversion: ClientVersion) -> Channel {
return vec![];
}
let mut items: Vec<VideoItem> = Vec::new();
let video_listing_content = html.split("<div class=\"thumbs clearfix\" id=\"custom_list_videos").collect::<Vec<&str>>()[1].split("<div class=\"pagination\"").collect::<Vec<&str>>()[0].to_string();
if html.contains("cards__item") {
for video_segment in html.split("<div class=\"cards__item\"").skip(1) {
let video_url = video_segment
.split("href=\"")
.nth(1)
.and_then(|s| s.split('"').next())
.unwrap_or_default()
.to_string();
if !video_url.contains("/video/") {
continue;
}
let id = video_url
.split("/video/")
.nth(1)
.and_then(|s| s.split('/').next())
.unwrap_or_default()
.to_string();
if id.is_empty() {
continue;
}
let mut title = video_segment
.split("title=\"")
.nth(1)
.and_then(|s| s.split('"').next())
.unwrap_or_default()
.to_string();
if title.is_empty() {
title = video_segment
.split("<span class=\"card__title\">")
.nth(1)
.and_then(|s| s.split('<').next())
.unwrap_or_default()
.trim()
.to_string();
}
title = decode(title.as_bytes()).to_string().unwrap_or(title);
let thumb = video_segment
.split("data-original=\"")
.nth(1)
.and_then(|s| s.split('"').next())
.unwrap_or_default()
.to_string();
let preview = video_segment
.split("data-preview=\"")
.nth(1)
.and_then(|s| s.split('"').next())
.unwrap_or_default()
.to_string();
let raw_duration = video_segment
.split("card__label card__label--primary\">")
.nth(1)
.and_then(|s| s.split('<').next())
.unwrap_or_default()
.trim()
.to_string();
let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
let views_text = video_segment
.split("<small class=\"card__text\">")
.nth(2)
.and_then(|s| s.split("</small>").next())
.unwrap_or_default()
.replace("views", "")
.replace(' ', "")
.trim()
.to_string();
let views = parse_abbreviated_number(views_text.as_str()).unwrap_or(0) as u32;
items.push(
VideoItem::new(
id,
title,
video_url,
"rule34gen".to_string(),
thumb,
duration,
)
.views(views)
.preview(preview),
);
}
return items;
}
let video_listing_content = html.split("<div class=\"thumbs clearfix\" id=\"custom_list_videos").collect::<Vec<&str>>().get(1).copied().unwrap_or_default().split("<div class=\"pagination\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default().to_string();
let raw_videos = video_listing_content
.split("<div class=\"item thumb video_")
.collect::<Vec<&str>>()[1..]
.to_vec();
for video_segment in &raw_videos {
// let vid = video_segment.split("\n").collect::<Vec<&str>>()[1]
// let vid = video_segment.split("\n").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
// for (index, line) in vid.iter().enumerate() {
// println!("Line {}: {}", index, line);
// }
@@ -195,35 +302,35 @@ fn build_channel(&self, clientversion: ClientVersion) -> Channel {
continue;
}
let mut title = video_segment.split("<div class=\"thumb_title\">").collect::<Vec<&str>>()[1]
let mut title = video_segment.split("<div class=\"thumb_title\">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
// html decode
title = decode(title.as_bytes()).to_string().unwrap_or(title);
let id = video_segment.split("https://rule34gen.com/video/").collect::<Vec<&str>>()[1].split("/").collect::<Vec<&str>>()[0].to_string();
let raw_duration = video_segment.split("<div class=\"time\">").collect::<Vec<&str>>()[1]
let id = video_segment.split("https://rule34gen.com/video/").collect::<Vec<&str>>().get(1).copied().unwrap_or_default().split("/").collect::<Vec<&str>>().get(0).copied().unwrap_or_default().to_string();
let raw_duration = video_segment.split("<div class=\"time\">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
let views = parse_abbreviated_number(&video_segment
.split("<div class=\"views\">").collect::<Vec<&str>>()[1].split("</svg>").collect::<Vec<&str>>()[1]
.split("<div class=\"views\">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default().split("</svg>").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<")
.collect::<Vec<&str>>()[0]).unwrap_or(0);
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()).unwrap_or(0);
//https://rule34gen.com/get_file/47/5e71602b7642f9b997f90c979a368c99b8aad90d89/3942000/3942353/3942353_preview.mp4/
//https://rule34gen.com/get_file/47/5e71602b7642f9b997f90c979a368c99b8aad90d89/3942000/3942353/3942353_preview.mp4/
let thumb = video_segment.split("<img class=\"thumb lazy-load\" src=\"").collect::<Vec<&str>>()[1].split("data-original=\"").collect::<Vec<&str>>()[1]
let thumb = video_segment.split("<img class=\"thumb lazy-load\" src=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default().split("data-original=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
let url = video_segment.split("<a class=\"th js-open-popup\" href=\"").collect::<Vec<&str>>()[1]
let url = video_segment.split("<a class=\"th js-open-popup\" href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
// let preview = video_segment.split("<div class=\"img wrap_image\" data-preview=\"").collect::<Vec<&str>>()[1]
// let preview = video_segment.split("<div class=\"img wrap_image\" data-preview=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
// .split("\"")
// .collect::<Vec<&str>>()[0]
// .collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
// .to_string();

50
src/providers/rule34video.rs
View File

@@ -1,5 +1,7 @@
use crate::api::ClientVersion;
use crate::DbPool;
use crate::providers::Provider;
use crate::status::*;
use crate::util::cache::VideoCache;
use crate::util::discord::send_discord_error_report;
use crate::util::parse_abbreviated_number;
@@ -36,6 +38,50 @@ impl Rule34videoProvider {
}
}
fn build_channel(&self, _clientversion: ClientVersion) -> Channel {
Channel {
id: "rule34video".to_string(),
name: "Rule34Video".to_string(),
description: "If it exists, there is porn".to_string(),
premium: false,
favicon: "https://www.google.com/s2/favicons?sz=64&domain=rule34video.com".to_string(),
status: "active".to_string(),
categories: vec![],
options: vec![ChannelOption {
id: "sort".to_string(),
title: "Sort".to_string(),
description: "Sort the Videos".to_string(),
systemImage: "list.number".to_string(),
colorName: "blue".to_string(),
options: vec![
FilterOption {
id: "post_date".to_string(),
title: "Newest".to_string(),
},
FilterOption {
id: "video_viewed".to_string(),
title: "Most Viewed".to_string(),
},
FilterOption {
id: "rating".to_string(),
title: "Top Rated".to_string(),
},
FilterOption {
id: "duration".to_string(),
title: "Longest".to_string(),
},
FilterOption {
id: "pseudo_random".to_string(),
title: "Random".to_string(),
},
],
multiSelect: false,
}],
nsfw: true,
cacheDuration: Some(1800),
}
}
/// Helper to safely extract a string between two delimiters
fn extract_between<'a>(content: &'a str, start_pat: &str, end_pat: &str) -> Option<&'a str> {
let start_idx = content.find(start_pat)? + start_pat.len();
@@ -265,4 +311,8 @@ impl Provider for Rule34videoProvider {
}
}
}
fn get_channel(&self, clientversion: ClientVersion) -> Option<Channel> {
Some(self.build_channel(clientversion))
}
}

102
src/providers/sxyprn.rs
View File

@@ -1,5 +1,7 @@
use crate::api::ClientVersion;
use crate::DbPool;
use crate::providers::Provider;
use crate::status::*;
use crate::util::cache::VideoCache;
use crate::util::discord::format_error_chain;
use crate::util::discord::send_discord_error_report;
@@ -37,6 +39,70 @@ impl SxyprnProvider {
url: "https://sxyprn.com".to_string(),
}
}
fn build_channel(&self, _clientversion: ClientVersion) -> Channel {
Channel {
id: "sxyprn".to_string(),
name: "SexyPorn".to_string(),
description: "Free Porn Site".to_string(),
premium: false,
favicon: "https://www.google.com/s2/favicons?sz=64&domain=sxyprn.com".to_string(),
status: "active".to_string(),
categories: vec![],
options: vec![
ChannelOption {
id: "sort".to_string(),
title: "Sort".to_string(),
description: "Sort the Videos".to_string(),
systemImage: "list.number".to_string(),
colorName: "blue".to_string(),
options: vec![
FilterOption {
id: "latest".to_string(),
title: "Latest".to_string(),
},
FilterOption {
id: "views".to_string(),
title: "Views".to_string(),
},
FilterOption {
id: "rating".to_string(),
title: "Rating".to_string(),
},
FilterOption {
id: "orgasmic".to_string(),
title: "Orgasmic".to_string(),
},
],
multiSelect: false,
},
ChannelOption {
id: "filter".to_string(),
title: "Filter".to_string(),
description: "Filter the Videos".to_string(),
systemImage: "line.horizontal.3.decrease.circle".to_string(),
colorName: "green".to_string(),
options: vec![
FilterOption {
id: "top".to_string(),
title: "Top".to_string(),
},
FilterOption {
id: "other".to_string(),
title: "Other".to_string(),
},
FilterOption {
id: "all".to_string(),
title: "All".to_string(),
},
],
multiSelect: false,
},
],
nsfw: true,
cacheDuration: Some(1800),
}
}
async fn get(
&self,
cache: VideoCache,
@@ -52,13 +118,13 @@ impl SxyprnProvider {
_ => "latest",
};
// Extract needed fields from options at the start
let filter = options.filter.clone().unwrap();
let filter = options.filter.clone().unwrap_or_else(|| "top".to_string());
let filter_string = match filter.as_str() {
"other" => "other",
"all" => "all",
_ => "top",
};
let mut requester = options.requester.clone().unwrap();
let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let url_str = format!(
"{}/blog/all/{}.html?fl={}&sm={}",
@@ -81,7 +147,18 @@ impl SxyprnProvider {
}
};
let text = requester.get(&url_str, None).await.unwrap();
let text = match requester.get(&url_str, None).await {
Ok(text) => text,
Err(e) => {
crate::providers::report_provider_error(
"sxyprn",
"get.request",
&format!("url={url_str}; error={e}"),
)
.await;
return Ok(old_items);
}
};
// Pass a reference to options if needed, or reconstruct as needed
let video_items = match self
.get_video_items_from_html(text.clone(), pool, requester)
@@ -130,7 +207,7 @@ impl SxyprnProvider {
_ => "latest",
};
// Extract needed fields from options at the start
let mut requester = options.requester.clone().unwrap();
let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let search_string = query.replace(" ", "-");
let url_str = format!(
"{}/{}.html?page={}&sm={}",
@@ -153,7 +230,18 @@ impl SxyprnProvider {
vec![]
}
};
let text = requester.get(&url_str, None).await.unwrap();
let text = match requester.get(&url_str, None).await {
Ok(text) => text,
Err(e) => {
crate::providers::report_provider_error(
"sxyprn",
"query.request",
&format!("url={url_str}; error={e}"),
)
.await;
return Ok(old_items);
}
};
let video_items = match self
.get_video_items_from_html(text.clone(), pool, requester)
@@ -391,4 +479,8 @@ impl Provider for SxyprnProvider {
}
}
}
fn get_channel(&self, clientversion: ClientVersion) -> Option<Channel> {
Some(self.build_channel(clientversion))
}
}

119
src/providers/xxdbx.rs
View File

@@ -1,6 +1,6 @@
use crate::DbPool;
use crate::api::ClientVersion;
use crate::providers::Provider;
use crate::providers::{Provider, report_provider_error_background};
use crate::status::*;
use crate::util::cache::VideoCache;
use crate::util::time::parse_time_to_seconds;
@@ -21,8 +21,13 @@ error_chain! {
fn is_valid_date(s: &str) -> bool {
// Regex: strict yyyy-mm-dd (no validation of real calendar dates, just format)
let re = Regex::new(r"^\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01])$").unwrap();
re.is_match(s)
match Regex::new(r"^\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01])$") {
Ok(re) => re.is_match(s),
Err(e) => {
report_provider_error_background("xxdbx", "is_valid_date.regex", &e.to_string());
false
}
}
}
#[derive(Debug, Clone)]
@@ -104,8 +109,14 @@ impl XxdbxProvider {
}
};
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url, None).await.unwrap();
let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let text = match requester.get(&video_url, None).await {
Ok(text) => text,
Err(e) => {
report_provider_error_background("xxdbx", "get.request", &e.to_string());
return Ok(old_items);
}
};
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);
@@ -126,9 +137,31 @@ impl XxdbxProvider {
let search_string = query.trim().to_string();
let mut search_type = "search";
if self.channels.read().unwrap().iter().map(|s|s.to_ascii_lowercase()).collect::<Vec<String>>().contains(&search_string.to_ascii_lowercase()) {
if self
.channels
.read()
.map(|channels| {
channels
.iter()
.map(|s| s.to_ascii_lowercase())
.collect::<Vec<String>>()
.contains(&search_string.to_ascii_lowercase())
})
.unwrap_or(false)
{
search_type = "channels";
} else if self.stars.read().unwrap().iter().map(|s|s.to_ascii_lowercase()).collect::<Vec<String>>().contains(&search_string.to_ascii_lowercase()) {
} else if self
.stars
.read()
.map(|stars| {
stars
.iter()
.map(|s| s.to_ascii_lowercase())
.collect::<Vec<String>>()
.contains(&search_string.to_ascii_lowercase())
})
.unwrap_or(false)
{
search_type = "stars";
} else if is_valid_date(&search_string){
search_type = "dates";
@@ -153,9 +186,15 @@ impl XxdbxProvider {
}
};
let mut requester = options.requester.clone().unwrap();
let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let text = requester.get(&video_url, None).await.unwrap();
let text = match requester.get(&video_url, None).await {
Ok(text) => text,
Err(e) => {
report_provider_error_background("xxdbx", "query.request", &e.to_string());
return Ok(old_items);
}
};
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);
@@ -171,9 +210,9 @@ impl XxdbxProvider {
return vec![];
}
let mut items: Vec<VideoItem> = Vec::new();
let raw_videos = html.split("</article>").collect::<Vec<&str>>()[0]
let raw_videos = html.split("</article>").collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.split("<div class=\"vids\">")
.collect::<Vec<&str>>()[1]
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<div class=\"v\">")
.collect::<Vec<&str>>()[1..]
.to_vec();
@@ -182,52 +221,68 @@ impl XxdbxProvider {
// for (index, line) in vid.iter().enumerate() {
// println!("Line {}: {}\n\n", index, line);
// }
let video_url: String = format!("{}{}", self.url, video_segment.split("<a href=\"").collect::<Vec<&str>>()[1]
let video_url: String = format!("{}{}", self.url, video_segment.split("<a href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string());
let mut title = video_segment
.split("<div class=\"v_title\">")
.collect::<Vec<&str>>()[1]
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.trim()
.to_string();
// html decode
title = decode(title.as_bytes()).to_string().unwrap_or(title);
let id = video_url.split("/").collect::<Vec<&str>>()[4].to_string();
let id = video_url.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string();
let thumb = format!("https:{}", video_segment.split("<img ").collect::<Vec<&str>>()[1]
.split("src=\"").collect::<Vec<&str>>().last().unwrap()
let thumb = format!("https:{}", video_segment.split("<img ").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("src=\"").collect::<Vec<&str>>().last().copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string());
let raw_duration = video_segment
.split("<div class=\"v_dur\">")
.collect::<Vec<&str>>()[1]
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
let duration = parse_time_to_seconds(raw_duration.as_str()).unwrap_or(0) as u32;
let preview = format!("https:{}",video_segment
.split("data-preview=\"")
.collect::<Vec<&str>>()[1]
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string());
let tags = video_segment.split("<div class=\"v_tags\">").collect::<Vec<&str>>()[1]
.split("</div>").collect::<Vec<&str>>()[0]
let tags = video_segment.split("<div class=\"v_tags\">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("</div>").collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.split("<a href=\"")
.collect::<Vec<&str>>()[1..]
.into_iter().map(|s| s.split("\"").collect::<Vec<&str>>()[0].replace("%20"," ").to_string()).collect::<Vec<String>>();
.into_iter().map(|s| s.split("\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default().replace("%20"," ").to_string()).collect::<Vec<String>>();
for tag in tags.clone() {
let shorted_tag = tag.split("/").collect::<Vec<&str>>()[2].to_string();
if tag.contains("channels") && self.channels.read().unwrap().contains(&shorted_tag) == false {
self.channels.write().unwrap().push(shorted_tag.clone());
let shorted_tag = tag.split("/").collect::<Vec<&str>>().get(2).copied().unwrap_or_default().to_string();
if tag.contains("channels")
&& self
.channels
.read()
.map(|channels| !channels.contains(&shorted_tag))
.unwrap_or(false)
{
if let Ok(mut channels) = self.channels.write() {
channels.push(shorted_tag.clone());
}
}
if tag.contains("stars") && self.stars.read().unwrap().contains(&shorted_tag) == false {
self.stars.write().unwrap().push(shorted_tag.clone());
if tag.contains("stars")
&& self
.stars
.read()
.map(|stars| !stars.contains(&shorted_tag))
.unwrap_or(false)
{
if let Ok(mut stars) = self.stars.write() {
stars.push(shorted_tag.clone());
}
}
}
let video_item = VideoItem::new(
@@ -238,7 +293,7 @@ impl XxdbxProvider {
thumb,
duration,
)
.tags(tags.into_iter().map(|s| s.split("/").collect::<Vec<&str>>().last().unwrap().to_string()).collect::<Vec<String>>())
.tags(tags.into_iter().map(|s| s.split("/").collect::<Vec<&str>>().last().copied().unwrap_or_default().to_string()).collect::<Vec<String>>())
.preview(preview);
items.push(video_item);
}

102
src/providers/xxthots.rs
View File

@@ -1,5 +1,7 @@
use crate::api::ClientVersion;
use crate::DbPool;
use crate::providers::Provider;
use crate::status::*;
use crate::util::cache::VideoCache;
use crate::util::parse_abbreviated_number;
use crate::util::time::parse_time_to_seconds;
@@ -26,6 +28,42 @@ impl XxthotsProvider {
url: "https://xxthots.com".to_string(),
}
}
fn build_channel(&self, _clientversion: ClientVersion) -> Channel {
Channel {
id: "xxthots".to_string(),
name: "XXTHOTS".to_string(),
description: "Free XXX Onlyfans Leaks Videos".to_string(),
premium: false,
favicon: "https://www.google.com/s2/favicons?sz=64&domain=xxthots.com".to_string(),
status: "active".to_string(),
categories: vec![],
options: vec![ChannelOption {
id: "sort".to_string(),
title: "Sort".to_string(),
description: "Sort the Videos".to_string(),
systemImage: "list.number".to_string(),
colorName: "blue".to_string(),
options: vec![
FilterOption {
id: "new".to_string(),
title: "New".to_string(),
},
FilterOption {
id: "popular".to_string(),
title: "Popular".to_string(),
},
FilterOption {
id: "top-rated".to_string(),
title: "Top Rated".to_string(),
},
],
multiSelect: false,
}],
nsfw: true,
cacheDuration: Some(1800),
}
}
async fn get(
&self,
cache: VideoCache,
@@ -60,8 +98,20 @@ impl XxthotsProvider {
vec![]
}
};
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url, None).await.unwrap();
let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let text = match requester.get(&video_url, None).await {
Ok(text) => text,
Err(e) => {
crate::providers::report_provider_error(
"xxthots",
"get.request",
&format!("url={video_url}; error={e}"),
)
.await;
return Ok(old_items);
}
};
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);
@@ -97,8 +147,20 @@ impl XxthotsProvider {
vec![]
}
};
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url, None).await.unwrap();
let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let text = match requester.get(&video_url, None).await {
Ok(text) => text,
Err(e) => {
crate::providers::report_provider_error(
"xxthots",
"query.request",
&format!("url={video_url}; error={e}"),
)
.await;
return Ok(old_items);
}
};
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);
@@ -117,7 +179,7 @@ impl XxthotsProvider {
let mut items: Vec<VideoItem> = Vec::new();
let raw_videos = html
.split("<div class=\"pagination\"")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.split("<div class=\"thumb thumb_rel item \">")
.collect::<Vec<&str>>()[1..]
.to_vec();
@@ -126,41 +188,41 @@ impl XxthotsProvider {
// for (index, line) in vid.iter().enumerate() {
// println!("Line {}: {}", index, line);
// }
let video_url: String = video_segment.split("<a href=\"").collect::<Vec<&str>>()[1]
let video_url: String = video_segment.split("<a href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
let mut title = video_segment.split("\" title=\"").collect::<Vec<&str>>()[1]
let mut title = video_segment.split("\" title=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
// html decode
title = decode(title.as_bytes()).to_string().unwrap_or(title);
let id = video_url.split("/").collect::<Vec<&str>>()[4].to_string();
let id = video_url.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string();
let raw_duration = video_segment
.split("<div class=\"time\">")
.collect::<Vec<&str>>()[1]
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
let thumb = video_segment
.split("<img class=\"lazy-load")
.collect::<Vec<&str>>()[1]
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("data-original=\"")
.collect::<Vec<&str>>()[1]
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
let views_part = video_segment
.split("svg-icon icon-eye")
.collect::<Vec<&str>>()[1]
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("</i>")
.collect::<Vec<&str>>()[1]
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
let views = parse_abbreviated_number(&views_part).unwrap_or(0) as u32;
@@ -212,4 +274,8 @@ impl Provider for XxthotsProvider {
}
}
}
fn get_channel(&self, clientversion: ClientVersion) -> Option<Channel> {
Some(self.build_channel(clientversion))
}
}

120
src/providers/youjizz.rs
View File

@@ -1,6 +1,8 @@
use crate::api::ClientVersion;
use crate::util::parse_abbreviated_number;
use crate::DbPool;
use crate::providers::Provider;
use crate::status::*;
use crate::util::cache::VideoCache;
use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoItem};
@@ -26,6 +28,58 @@ impl YoujizzProvider {
url: "https://www.youjizz.com".to_string(),
}
}
fn build_channel(&self, _clientversion: ClientVersion) -> Channel {
Channel {
id: "youjizz".to_string(),
name: "YouJizz".to_string(),
description: "YouJizz Porntube".to_string(),
premium: false,
favicon: "https://www.google.com/s2/favicons?sz=64&domain=www.youjizz.com".to_string(),
status: "active".to_string(),
categories: vec![],
options: vec![ChannelOption {
id: "sort".to_string(),
title: "Sort".to_string(),
description: "Sort the Videos".to_string(),
systemImage: "list.number".to_string(),
colorName: "blue".to_string(),
options: vec![
FilterOption {
id: "new".to_string(),
title: "New".to_string(),
},
FilterOption {
id: "popular".to_string(),
title: "Popular".to_string(),
},
FilterOption {
id: "top-rated".to_string(),
title: "Top Rated".to_string(),
},
FilterOption {
id: "top-rated-week".to_string(),
title: "Top Rated (Week)".to_string(),
},
FilterOption {
id: "top-rated-month".to_string(),
title: "Top Rated (Month)".to_string(),
},
FilterOption {
id: "trending".to_string(),
title: "Trending".to_string(),
},
FilterOption {
id: "random".to_string(),
title: "Random".to_string(),
},
],
multiSelect: false,
}],
nsfw: true,
cacheDuration: None,
}
}
async fn get(
&self,
cache: VideoCache,
@@ -56,9 +110,20 @@ impl YoujizzProvider {
}
};
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url, None).await.unwrap();
let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let text = match requester.get(&video_url, None).await {
Ok(text) => text,
Err(e) => {
crate::providers::report_provider_error(
"youjizz",
"get.request",
&format!("url={video_url}; error={e}"),
)
.await;
return Ok(old_items);
}
};
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);
@@ -92,9 +157,20 @@ impl YoujizzProvider {
}
};
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url, None).await.unwrap();
let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let text = match requester.get(&video_url, None).await {
Ok(text) => text,
Err(e) => {
crate::providers::report_provider_error(
"youjizz",
"query.request",
&format!("url={video_url}; error={e}"),
)
.await;
return Ok(old_items);
}
};
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);
@@ -111,7 +187,7 @@ impl YoujizzProvider {
return vec![];
}
let mut items: Vec<VideoItem> = Vec::new();
let raw_videos = html.split("class=\"mobile-only\"").collect::<Vec<&str>>()[0]
let raw_videos = html.split("class=\"mobile-only\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.split("class=\"default video-item\"")
.collect::<Vec<&str>>()[1..]
.to_vec();
@@ -124,31 +200,31 @@ impl YoujizzProvider {
// println!("Skipping video segment due to placeholder thumbnail");
// continue;
// }
let video_url: String = format!("{}{}",self.url, video_segment.split("href=\"").collect::<Vec<&str>>()[1]
let video_url: String = format!("{}{}",self.url, video_segment.split("href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0].to_string());
let mut title = video_segment.split("class=\"video-title\">").collect::<Vec<&str>>()[1]
.split(">").collect::<Vec<&str>>()[1]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default().to_string());
let mut title = video_segment.split("class=\"video-title\">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split(">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
// html decode
title = decode(title.as_bytes()).to_string().unwrap_or(title);
let id = video_url.split("/").collect::<Vec<&str>>()[4].to_string();
let id = video_url.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string();
let thumb = format!("https:{}",video_segment.split("<img ").collect::<Vec<&str>>()[1]
.split("data-original=\"").collect::<Vec<&str>>()[1]
let thumb = format!("https:{}",video_segment.split("<img ").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("data-original=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string());
let raw_duration = video_segment.split("fa fa-clock-o\"></i>&nbsp;").collect::<Vec<&str>>()[1]
let raw_duration = video_segment.split("fa fa-clock-o\"></i>&nbsp;").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
let duration = parse_time_to_seconds(raw_duration.as_str()).unwrap_or(0) as u32;
let views = parse_abbreviated_number(video_segment.split("format-views\">").collect::<Vec<&str>>()[1]
let views = parse_abbreviated_number(video_segment.split("format-views\">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<")
.collect::<Vec<&str>>()[0]
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string().as_str()).unwrap_or(0) as u32;
let video_item = VideoItem::new(
@@ -201,4 +277,8 @@ impl Provider for YoujizzProvider {
}
}
}
fn get_channel(&self, clientversion: ClientVersion) -> Option<Channel> {
Some(self.build_channel(clientversion))
}
}

5
src/util/requester.rs
View File

@@ -245,7 +245,10 @@ pub async fn post_json<S>(
// If direct request failed, try FlareSolverr. Map its error to a Send+Sync error immediately,
// so no non-Send error value lives across later `.await`s.
let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set");
let flare_url = match env::var("FLARE_URL") {
Ok(url) => url,
Err(e) => return Err(format!("FLARE_URL not set: {e}").into()),
};
let mut flare = Flaresolverr::new(flare_url);
if self.proxy && env::var("BURP_URL").is_ok() {
flare.set_proxy(true);