simon/hottub
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fixes and cleanup

Browse Source
This commit is contained in:
Simon
2026-03-05 18:18:48 +00:00
parent 76fd5a4f4f
commit 2627505ade
49 changed files with 3245 additions and 1376 deletions
Download Patch File Download Diff File Expand all files Collapse all files

64
src/api.rs
View File

@@ -1,15 +1,18 @@
use crate::providers::{
ALL_PROVIDERS, DynProvider, panic_payload_to_string, report_provider_error,
run_provider_guarded,
};
use crate::util::cache::VideoCache;
use crate::util::discord::send_discord_error_report;
use crate::util::proxy::{Proxy, all_proxies_snapshot};
use crate::util::requester::Requester;
use crate::{DbPool, db, status::*, videos::*};
use ntex::http::header; use ntex::http::header;
use ntex::web; use ntex::web;
use ntex::web::HttpRequest; use ntex::web::HttpRequest;
use std::cmp::Ordering; use std::cmp::Ordering;
use std::io; use std::io;
use tokio::task; use tokio::task;
use crate::providers::{ALL_PROVIDERS, DynProvider, panic_payload_to_string, report_provider_error, run_provider_guarded};
use crate::util::cache::VideoCache;
use crate::util::discord::send_discord_error_report;
use crate::util::proxy::{Proxy, all_proxies_snapshot};
use crate::util::requester::Requester;
use crate::{DbPool, db, status::*, videos::*};
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct ClientVersion { pub struct ClientVersion {
@@ -119,10 +122,9 @@ async fn status(req: HttpRequest) -> Result<impl web::Responder, web::Error> {
let mut status = Status::new(); let mut status = Status::new();
for (provider_name, provider) in ALL_PROVIDERS.iter() { for (provider_name, provider) in ALL_PROVIDERS.iter() {
let channel_result = let channel_result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { provider.get_channel(clientversion.clone())
provider.get_channel(clientversion.clone()) }));
}));
match channel_result { match channel_result {
Ok(Some(channel)) => status.add_channel(channel), Ok(Some(channel)) => status.add_channel(channel),
Ok(None) => {} Ok(None) => {}
@@ -218,7 +220,16 @@ async fn videos_post(
.as_deref() .as_deref()
.unwrap_or("all") .unwrap_or("all")
.to_string(); .to_string();
let sites = video_request.sites.as_deref().unwrap_or("").to_string(); let sites = if channel == "all" {
video_request
.all_provider_sites
.as_deref()
.or(video_request.sites.as_deref())
.unwrap_or("")
.to_string()
} else {
video_request.sites.as_deref().unwrap_or("").to_string()
};
let filter = video_request.filter.as_deref().unwrap_or("new").to_string(); let filter = video_request.filter.as_deref().unwrap_or("new").to_string();
let language = video_request let language = video_request
.language .language
@@ -261,22 +272,25 @@ async fn videos_post(
) )
.await; .await;
// There is a bug in Hottub38 that makes the client error for a 403-url even though formats work fine // There is a bug in Hottub38 that makes the client error for a 403-url even though formats work fine
if clientversion == ClientVersion::new(38, 0, "Hot%20Tub".to_string()) { if clientversion == ClientVersion::new(38, 0, "Hot%20Tub".to_string()) {
// filter out videos without preview for old clients // filter out videos without preview for old clients
video_items = video_items.into_iter().filter_map(|video| { video_items = video_items
let last_url = video .into_iter()
.formats .filter_map(|video| {
.as_ref() let last_url = video
.and_then(|formats| formats.last().map(|f| f.url.clone())); .formats
if let Some(url) = last_url { .as_ref()
let mut v = video; .and_then(|formats| formats.last().map(|f| f.url.clone()));
v.url = url; if let Some(url) = last_url {
return Some(v); let mut v = video;
} v.url = url;
Some(video) return Some(v);
}).collect(); }
} Some(video)
})
.collect();
}
videos.items = video_items.clone(); videos.items = video_items.clone();
if video_items.len() == 0 { if video_items.len() == 0 {

62
src/main.rs
View File

@@ -1,31 +1,32 @@
#![warn(unused_extern_crates)] #![warn(unused_extern_crates)]
#![allow(non_snake_case)] #![allow(non_snake_case)]
use std::{env, thread}; use std::{env, thread};
use diesel::{r2d2::{self, ConnectionManager}, SqliteConnection}; use diesel::{
SqliteConnection,
r2d2::{self, ConnectionManager},
};
use dotenvy::dotenv; use dotenvy::dotenv;
use ntex_files as fs;
use ntex::web; use ntex::web;
use ntex_files as fs;
mod api; mod api;
mod proxy;
mod db; mod db;
mod models; mod models;
mod providers; mod providers;
mod proxies;
mod proxy;
mod schema; mod schema;
mod status; mod status;
mod util; mod util;
mod videos; mod videos;
mod proxies;
type DbPool = r2d2::Pool<ConnectionManager<SqliteConnection>>; type DbPool = r2d2::Pool<ConnectionManager<SqliteConnection>>;
// #[macro_use(c)] // #[macro_use(c)]
// extern crate cute; // extern crate cute;
#[ntex::main] #[ntex::main]
async fn main() -> std::io::Result<()> { async fn main() -> std::io::Result<()> {
// std::env::set_var("RUST_BACKTRACE", "1"); // std::env::set_var("RUST_BACKTRACE", "1");
@@ -33,7 +34,7 @@ async fn main() -> std::io::Result<()> {
// Enable request logging // Enable request logging
if std::env::var("RUST_LOG").is_err() { if std::env::var("RUST_LOG").is_err() {
unsafe{ unsafe {
std::env::set_var("RUST_LOG", "warn"); std::env::set_var("RUST_LOG", "warn");
} }
} }
@@ -49,20 +50,22 @@ async fn main() -> std::io::Result<()> {
let mut requester = util::requester::Requester::new(); let mut requester = util::requester::Requester::new();
requester.set_proxy(env::var("PROXY").unwrap_or("0".to_string()) != "0".to_string()); requester.set_proxy(env::var("PROXY").unwrap_or("0".to_string()) != "0".to_string());
let cache: util::cache::VideoCache = crate::util::cache::VideoCache::new().max_size(100_000).to_owned(); let cache: util::cache::VideoCache = crate::util::cache::VideoCache::new()
.max_size(100_000)
thread::spawn(move || { .to_owned();
// Create a tiny runtime just for these async tasks
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("build tokio runtime");
rt.block_on(async move { thread::spawn(move || {
providers::init_providers_now(); // Create a tiny runtime just for these async tasks
}); let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("build tokio runtime");
rt.block_on(async move {
providers::init_providers_now();
}); });
});
web::HttpServer::new(move || { web::HttpServer::new(move || {
web::App::new() web::App::new()
.state(pool.clone()) .state(pool.clone())
@@ -72,17 +75,16 @@ async fn main() -> std::io::Result<()> {
.service(web::scope("/api").configure(api::config)) .service(web::scope("/api").configure(api::config))
.service(web::scope("/proxy").configure(proxy::config)) .service(web::scope("/proxy").configure(proxy::config))
.service( .service(
web::resource("/") web::resource("/").route(web::get().to(|req: web::HttpRequest| async move {
.route(web::get().to(|req: web::HttpRequest| async move{ let host = match std::env::var("DOMAIN") {
let host = match std::env::var("DOMAIN"){ Ok(d) => d,
Ok(d) => d, Err(_) => req.connection_info().host().to_string(),
Err(_) => req.connection_info().host().to_string() };
}; let source_forward_header = format!("hottub://source?url={}", host);
let source_forward_header = format!("hottub://source?url={}", host); web::HttpResponse::Found()
web::HttpResponse::Found() .header("Location", source_forward_header)
.header("Location", source_forward_header) .finish()
.finish() })),
}))
) )
.service(fs::Files::new("/", "static").index_file("index.html")) .service(fs::Files::new("/", "static").index_file("index.html"))
}) })

2
src/models.rs
View File

@@ -1,5 +1,5 @@
use diesel::prelude::*; use diesel::prelude::*;
use serde::{Serialize}; use serde::Serialize;
#[derive(Debug, Clone, Serialize, Queryable, Insertable)] #[derive(Debug, Clone, Serialize, Queryable, Insertable)]
#[diesel(table_name = crate::schema::videos)] #[diesel(table_name = crate::schema::videos)]

34
src/providers/all.rs
View File

@@ -1,18 +1,18 @@
use std::fs; use crate::DbPool;
use std::time::Duration; use crate::api::{ClientVersion, get_provider};
use crate::providers::{DynProvider, Provider, report_provider_error, run_provider_guarded};
use crate::status::{Channel, ChannelOption, FilterOption};
use crate::util::cache::VideoCache;
use crate::util::interleave;
use crate::videos::{ServerOptions, VideoItem};
use async_trait::async_trait; use async_trait::async_trait;
use capitalize::Capitalize; use capitalize::Capitalize;
use cute::c; use cute::c;
use error_chain::error_chain; use error_chain::error_chain;
use futures::StreamExt; use futures::StreamExt;
use futures::stream::FuturesUnordered; use futures::stream::FuturesUnordered;
use crate::api::{get_provider, ClientVersion}; use std::fs;
use crate::providers::{DynProvider, Provider, report_provider_error, run_provider_guarded}; use std::time::Duration;
use crate::status::{Channel, ChannelOption, FilterOption};
use crate::util::cache::VideoCache;
use crate::util::interleave;
use crate::videos::{ServerOptions, VideoItem};
use crate::DbPool;
error_chain! { error_chain! {
foreign_links { foreign_links {
@@ -21,11 +21,9 @@ error_chain! {
} }
} }
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
#[allow(dead_code)] #[allow(dead_code)]
pub struct AllProvider { pub struct AllProvider {}
}
impl AllProvider { impl AllProvider {
pub fn new() -> Self { pub fn new() -> Self {
@@ -81,7 +79,8 @@ impl Provider for AllProvider {
None => { None => {
// fire-and-forget reporting of missing provider keys // fire-and-forget reporting of missing provider keys
tokio::spawn(async move { tokio::spawn(async move {
report_provider_error("all", "all.get_videos.unknown_provider", &name).await; report_provider_error("all", "all.get_videos.unknown_provider", &name)
.await;
}); });
None None
} }
@@ -128,7 +127,7 @@ impl Provider for AllProvider {
}, },
_ = &mut timeout_timer => { _ = &mut timeout_timer => {
// 55 seconds passed. Stop waiting and return what we have. // 55 seconds passed. Stop waiting and return what we have.
// The tasks remaining in 'futures' will continue running in the // The tasks remaining in 'futures' will continue running in the
// background because they were 'tokio::spawn'ed. // background because they were 'tokio::spawn'ed.
break; break;
}, },
@@ -146,10 +145,7 @@ impl Provider for AllProvider {
.filter_map(|entry| entry.ok()) .filter_map(|entry| entry.ok())
.filter_map(|entry| entry.file_name().into_string().ok()) .filter_map(|entry| entry.file_name().into_string().ok())
.filter(|name| name.ends_with(".rs")) .filter(|name| name.ends_with(".rs"))
.filter(|name| { .filter(|name| !name.contains("mod.rs") && !name.contains("all.rs"))
!name.contains("mod.rs")
&& !name.contains("all.rs")
})
.map(|name| name.replace(".rs", "")) .map(|name| name.replace(".rs", ""))
.collect::<Vec<String>>(); .collect::<Vec<String>>();
let sites = c![FilterOption { let sites = c![FilterOption {
@@ -166,7 +162,7 @@ impl Provider for AllProvider {
status: "active".to_string(), status: "active".to_string(),
categories: vec![], categories: vec![],
options: vec![ChannelOption { options: vec![ChannelOption {
id: "sites".to_string(), id: "all_provider_sites".to_string(),
title: "Sites".to_string(), title: "Sites".to_string(),
description: "What Sites to use".to_string(), description: "What Sites to use".to_string(),
systemImage: "list.number".to_string(), systemImage: "list.number".to_string(),

92
src/providers/beeg.rs
View File

@@ -37,9 +37,18 @@ pub struct BeegProvider {
impl BeegProvider { impl BeegProvider {
pub fn new() -> Self { pub fn new() -> Self {
let provider = BeegProvider { let provider = BeegProvider {
sites: Arc::new(RwLock::new(vec![FilterOption { id: "all".into(), title: "All".into() }])), sites: Arc::new(RwLock::new(vec![FilterOption {
stars: Arc::new(RwLock::new(vec![FilterOption { id: "all".into(), title: "All".into() }])), id: "all".into(),
categories: Arc::new(RwLock::new(vec![FilterOption { id: "all".into(), title: "All".into() }])), title: "All".into(),
}])),
stars: Arc::new(RwLock::new(vec![FilterOption {
id: "all".into(),
title: "All".into(),
}])),
categories: Arc::new(RwLock::new(vec![FilterOption {
id: "all".into(),
title: "All".into(),
}])),
}; };
provider.spawn_initial_load(); provider.spawn_initial_load();
@@ -52,7 +61,10 @@ impl BeegProvider {
let stars = Arc::clone(&self.stars); let stars = Arc::clone(&self.stars);
thread::spawn(move || { thread::spawn(move || {
let rt = match tokio::runtime::Builder::new_current_thread().enable_all().build() { let rt = match tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
{
Ok(rt) => rt, Ok(rt) => rt,
Err(e) => { Err(e) => {
eprintln!("beeg runtime init failed: {}", e); eprintln!("beeg runtime init failed: {}", e);
@@ -77,14 +89,18 @@ impl BeegProvider {
async fn fetch_tags() -> Result<Value> { async fn fetch_tags() -> Result<Value> {
let mut requester = util::requester::Requester::new(); let mut requester = util::requester::Requester::new();
let text = match requester let text = match requester
.get("https://store.externulls.com/tag/facts/tags?get_original=true&slug=index", None) .get(
.await { "https://store.externulls.com/tag/facts/tags?get_original=true&slug=index",
Ok(text) => text, None,
Err(e) => { )
eprintln!("beeg fetch_tags failed: {}", e); .await
return Err(ErrorKind::Parse("failed to fetch tags".into()).into()); {
} Ok(text) => text,
}; Err(e) => {
eprintln!("beeg fetch_tags failed: {}", e);
return Err(ErrorKind::Parse("failed to fetch tags".into()).into());
}
};
Ok(serde_json::from_str(&text)?) Ok(serde_json::from_str(&text)?)
} }
@@ -99,7 +115,13 @@ impl BeegProvider {
s.get("tg_name").and_then(|v| v.as_str()), s.get("tg_name").and_then(|v| v.as_str()),
s.get("tg_slug").and_then(|v| v.as_str()), s.get("tg_slug").and_then(|v| v.as_str()),
) { ) {
Self::push_unique(&stars, FilterOption { id: id.into(), title: name.into() }); Self::push_unique(
&stars,
FilterOption {
id: id.into(),
title: name.into(),
},
);
} }
} }
Ok(()) Ok(())
@@ -139,7 +161,13 @@ impl BeegProvider {
s.get("tg_name").and_then(|v| v.as_str()), s.get("tg_name").and_then(|v| v.as_str()),
s.get("tg_slug").and_then(|v| v.as_str()), s.get("tg_slug").and_then(|v| v.as_str()),
) { ) {
Self::push_unique(&sites, FilterOption { id: id.into(), title: name.into() }); Self::push_unique(
&sites,
FilterOption {
id: id.into(),
title: name.into(),
},
);
} }
} }
Ok(()) Ok(())
@@ -178,7 +206,11 @@ impl BeegProvider {
description: "Filter for different Networks".into(), description: "Filter for different Networks".into(),
systemImage: "list.dash".into(), systemImage: "list.dash".into(),
colorName: "purple".into(), colorName: "purple".into(),
options: self.categories.read().map(|v| v.clone()).unwrap_or_default(), options: self
.categories
.read()
.map(|v| v.clone())
.unwrap_or_default(),
multiSelect: false, multiSelect: false,
}, },
ChannelOption { ChannelOption {
@@ -220,7 +252,8 @@ impl BeegProvider {
} }
let video_url = format!( let video_url = format!(
"https://store.externulls.com/facts/tag?limit=100&offset={}{}", "https://store.externulls.com/facts/tag?limit=100&offset={}{}",
page - 1, match slug { page - 1,
match slug {
"" => "&id=27173".to_string(), "" => "&id=27173".to_string(),
_ => format!("&slug={}", slug.replace(" ", "")), _ => format!("&slug={}", slug.replace(" ", "")),
} }
@@ -237,7 +270,8 @@ impl BeegProvider {
vec![] vec![]
} }
}; };
let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let text = match requester.get(&video_url, None).await { let text = match requester.get(&video_url, None).await {
Ok(text) => text, Ok(text) => text,
Err(e) => { Err(e) => {
@@ -289,7 +323,8 @@ impl BeegProvider {
} }
}; };
let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let text = match requester.get(&video_url, None).await { let text = match requester.get(&video_url, None).await {
Ok(text) => text, Ok(text) => text,
@@ -323,14 +358,24 @@ impl BeegProvider {
}; };
for video in array { for video in array {
let file = match video.get("file") { Some(v) => v, None => continue }; let file = match video.get("file") {
let hls = match file.get("hls_resources") { Some(v) => v, None => continue }; Some(v) => v,
None => continue,
};
let hls = match file.get("hls_resources") {
Some(v) => v,
None => continue,
};
let key = match hls.get("fl_cdn_multi").and_then(|v| v.as_str()) { let key = match hls.get("fl_cdn_multi").and_then(|v| v.as_str()) {
Some(v) => v, Some(v) => v,
None => continue, None => continue,
}; };
let id = file.get("id").and_then(|v| v.as_i64()).unwrap_or(0).to_string(); let id = file
.get("id")
.and_then(|v| v.as_i64())
.unwrap_or(0)
.to_string();
let title = file let title = file
.get("data") .get("data")
.and_then(|v| v.get(0)) .and_then(|v| v.get(0))
@@ -352,7 +397,10 @@ impl BeegProvider {
.and_then(|s| parse_abbreviated_number(s)) .and_then(|s| parse_abbreviated_number(s))
.unwrap_or(0); .unwrap_or(0);
let thumb = format!("https://thumbs.externulls.com/videos/{}/0.webp?size=480x270", id); let thumb = format!(
"https://thumbs.externulls.com/videos/{}/0.webp?size=480x270",
id
);
let mut item = VideoItem::new( let mut item = VideoItem::new(
id, id,

9
src/providers/chaturbate.rs
View File

@@ -252,7 +252,8 @@ impl ChaturbateProvider {
let mut title = video_segment let mut title = video_segment
.get("room_subject") .get("room_subject")
.and_then(|v| v.as_str()) .and_then(|v| v.as_str())
.map(String::from).unwrap_or("".to_string()); .map(String::from)
.unwrap_or("".to_string());
// html decode // html decode
title = decode(title.as_bytes()).to_string().unwrap_or(title); title = decode(title.as_bytes()).to_string().unwrap_or(title);
let id = username.clone(); let id = username.clone();
@@ -262,7 +263,11 @@ impl ChaturbateProvider {
.unwrap_or(&serde_json::Value::String("".to_string())) .unwrap_or(&serde_json::Value::String("".to_string()))
.as_str() .as_str()
.unwrap_or("") .unwrap_or("")
.split("?").collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .split("?")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(); .to_string();
let views = video_segment let views = video_segment
.get("viewers") .get("viewers")

285
src/providers/freepornvideosxxx.rs
View File

@@ -120,17 +120,40 @@ impl FreepornvideosxxxProvider {
.copied() .copied()
.unwrap_or_default() .unwrap_or_default()
.split("custom_list_models_models_list_pagination") .split("custom_list_models_models_list_pagination")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default(); .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default();
for stars_element in stars_div.split("<a ").collect::<Vec<&str>>()[1..].to_vec() { for stars_element in stars_div.split("<a ").collect::<Vec<&str>>()[1..].to_vec() {
let star_url = stars_element.split("href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() let star_url = stars_element
.split("href=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"") .split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default(); .collect::<Vec<&str>>()
let star_id = star_url.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string(); .get(0)
.copied()
.unwrap_or_default();
let star_id = star_url
.split("/")
.collect::<Vec<&str>>()
.get(4)
.copied()
.unwrap_or_default()
.to_string();
let star_name = stars_element let star_name = stars_element
.split("<strong class=\"title\">") .split("<strong class=\"title\">")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<") .split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(); .to_string();
Self::push_unique( Self::push_unique(
&stars, &stars,
@@ -168,19 +191,47 @@ impl FreepornvideosxxxProvider {
} }
let sites_div = text let sites_div = text
.split("id=\"list_content_sources_sponsors_list_items\"") .split("id=\"list_content_sources_sponsors_list_items\"")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("class=\"pagination\"") .split("class=\"pagination\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default(); .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default();
for sites_element in for sites_element in
sites_div.split("class=\"headline\"").collect::<Vec<&str>>()[1..].to_vec() sites_div.split("class=\"headline\"").collect::<Vec<&str>>()[1..].to_vec()
{ {
let site_url = sites_element.split("href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() let site_url = sites_element
.split("href=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"") .split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default(); .collect::<Vec<&str>>()
let site_id = site_url.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string(); .get(0)
let site_name = sites_element.split("<h2>").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .copied()
.unwrap_or_default();
let site_id = site_url
.split("/")
.collect::<Vec<&str>>()
.get(4)
.copied()
.unwrap_or_default()
.to_string();
let site_name = sites_element
.split("<h2>")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<") .split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(); .to_string();
Self::push_unique( Self::push_unique(
&sites, &sites,
@@ -207,22 +258,52 @@ impl FreepornvideosxxxProvider {
return Ok(()); return Ok(());
} }
}; };
let networks_div = text.split("class=\"sites__list\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() let networks_div = text
.split("class=\"sites__list\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("</div>") .split("</div>")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default(); .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default();
for network_element in for network_element in
networks_div.split("sites__item").collect::<Vec<&str>>()[1..].to_vec() networks_div.split("sites__item").collect::<Vec<&str>>()[1..].to_vec()
{ {
if network_element.contains("sites__all") { if network_element.contains("sites__all") {
continue; continue;
} }
let network_url = network_element.split("href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() let network_url = network_element
.split("href=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"") .split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default(); .collect::<Vec<&str>>()
let network_id = network_url.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string(); .get(0)
let network_name = network_element.split(">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .copied()
.unwrap_or_default();
let network_id = network_url
.split("/")
.collect::<Vec<&str>>()
.get(4)
.copied()
.unwrap_or_default()
.to_string();
let network_name = network_element
.split(">")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<") .split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(); .to_string();
Self::push_unique( Self::push_unique(
&networks, &networks,
@@ -271,7 +352,8 @@ impl FreepornvideosxxxProvider {
name: "FreePornVideos XXX".to_string(), name: "FreePornVideos XXX".to_string(),
description: "Free Porn Videos".to_string(), description: "Free Porn Videos".to_string(),
premium: false, premium: false,
favicon: "https://www.google.com/s2/favicons?sz=64&domain=www.freepornvideos.xxx".to_string(), favicon: "https://www.google.com/s2/favicons?sz=64&domain=www.freepornvideos.xxx"
.to_string(),
status: "active".to_string(), status: "active".to_string(),
categories: vec![], categories: vec![],
options: vec![ options: vec![
@@ -522,11 +604,22 @@ impl FreepornvideosxxxProvider {
if !html.contains("class=\"item\"") { if !html.contains("class=\"item\"") {
return items; return items;
} }
let raw_videos = html.split("videos_list_pagination").collect::<Vec<&str>>().get(0).copied().unwrap_or_default() let raw_videos = html
.split("videos_list_pagination")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.split(" class=\"pagination\" ") .split(" class=\"pagination\" ")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.split("class=\"list-videos\"") .split("class=\"list-videos\"")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("class=\"item\"") .split("class=\"item\"")
.collect::<Vec<&str>>()[1..] .collect::<Vec<&str>>()[1..]
.to_vec(); .to_vec();
@@ -535,39 +628,94 @@ impl FreepornvideosxxxProvider {
// for (index, line) in vid.iter().enumerate() { // for (index, line) in vid.iter().enumerate() {
// println!("Line {}: {}", index, line); // println!("Line {}: {}", index, line);
// } // }
let video_url: String = video_segment.split("<a href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() let video_url: String = video_segment
.split("<a href=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"") .split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(); .to_string();
let mut title = video_segment.split(" title=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() let mut title = video_segment
.split(" title=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"") .split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(); .to_string();
// html decode // html decode
title = decode(title.as_bytes()).to_string().unwrap_or(title); title = decode(title.as_bytes()).to_string().unwrap_or(title);
let id = video_url.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string(); let id = video_url
.split("/")
.collect::<Vec<&str>>()
.get(4)
.copied()
.unwrap_or_default()
.to_string();
let thumb = match video_segment.split("<img ").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() let thumb = match video_segment
.split("<img ")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.contains("data-src=\"") .contains("data-src=\"")
{ {
true => video_segment.split("<img ").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() true => video_segment
.split("<img ")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("data-src=\"") .split("data-src=\"")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"") .split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(), .to_string(),
false => video_segment.split("<img ").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() false => video_segment
.split("<img ")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("src=\"") .split("src=\"")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"") .split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(), .to_string(),
}; };
let raw_duration = video_segment let raw_duration = video_segment
.split("<span class=\"duration\">") .split("<span class=\"duration\">")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<") .split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.split(" ") .split(" ")
.collect::<Vec<&str>>() .collect::<Vec<&str>>()
.last() .last()
@@ -577,9 +725,15 @@ impl FreepornvideosxxxProvider {
let views = parse_abbreviated_number( let views = parse_abbreviated_number(
video_segment video_segment
.split("<div class=\"views\">") .split("<div class=\"views\">")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<") .split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string() .to_string()
.as_str(), .as_str(),
) )
@@ -587,9 +741,15 @@ impl FreepornvideosxxxProvider {
let preview = video_segment let preview = video_segment
.split("data-preview=\"") .split("data-preview=\"")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"") .split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(); .to_string();
let site_name = title let site_name = title
.split("]") .split("]")
@@ -603,9 +763,15 @@ impl FreepornvideosxxxProvider {
let mut tags = match video_segment.contains("class=\"models\">") { let mut tags = match video_segment.contains("class=\"models\">") {
true => video_segment true => video_segment
.split("class=\"models\">") .split("class=\"models\">")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("</div>") .split("</div>")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.split("href=\"") .split("href=\"")
.collect::<Vec<&str>>()[1..] .collect::<Vec<&str>>()[1..]
.into_iter() .into_iter()
@@ -613,17 +779,38 @@ impl FreepornvideosxxxProvider {
Self::push_unique( Self::push_unique(
&self.stars, &self.stars,
FilterOption { FilterOption {
id: s.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string(), id: s
title: s.split(">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .split("/")
.collect::<Vec<&str>>()
.get(4)
.copied()
.unwrap_or_default()
.to_string(),
title: s
.split(">")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<") .split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.trim() .trim()
.to_string(), .to_string(),
}, },
); );
s.split(">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() s.split(">")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<") .split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.trim() .trim()
.to_string() .to_string()
}) })

33
src/providers/hanime.rs
View File

@@ -4,8 +4,8 @@ use futures::future::join_all;
use serde_json::json; use serde_json::json;
use std::vec; use std::vec;
use crate::api::ClientVersion;
use crate::DbPool; use crate::DbPool;
use crate::api::ClientVersion;
use crate::db; use crate::db;
use crate::providers::{Provider, report_provider_error, report_provider_error_background}; use crate::providers::{Provider, report_provider_error, report_provider_error_background};
use crate::status::*; use crate::status::*;
@@ -272,7 +272,8 @@ impl HanimeProvider {
id id
); );
let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let payload = json!({ let payload = json!({
"width": 571, "height": 703, "ab": "kh" } "width": 571, "height": 703, "ab": "kh" }
); );
@@ -289,7 +290,7 @@ impl HanimeProvider {
], ],
) )
.await; // Initial request to set cookies .await; // Initial request to set cookies
ntex::time::sleep(ntex::time::Seconds(1)).await; ntex::time::sleep(ntex::time::Seconds(1)).await;
let text = requester let text = requester
.get_raw_with_headers( .get_raw_with_headers(
&url, &url,
@@ -328,7 +329,12 @@ impl HanimeProvider {
let mut url_vec = vec![]; let mut url_vec = vec![];
for el in urls.split("\"url\":\"").collect::<Vec<&str>>() { for el in urls.split("\"url\":\"").collect::<Vec<&str>>() {
let url = el.split("\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default(); let url = el
.split("\"")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default();
if !url.is_empty() && url.contains("m3u8") { if !url.is_empty() && url.contains("m3u8") {
url_vec.push(url.to_string()); url_vec.push(url.to_string());
} }
@@ -381,11 +387,23 @@ impl HanimeProvider {
) -> Result<Vec<VideoItem>> { ) -> Result<Vec<VideoItem>> {
let index = format!("hanime:{}:{}:{}", query, page, sort); let index = format!("hanime:{}:{}:{}", query, page, sort);
let order_by = match sort.contains(".") { let order_by = match sort.contains(".") {
true => sort.split(".").collect::<Vec<&str>>().get(0).copied().unwrap_or_default().to_string(), true => sort
.split(".")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(),
false => "created_at_unix".to_string(), false => "created_at_unix".to_string(),
}; };
let ordering = match sort.contains(".") { let ordering = match sort.contains(".") {
true => sort.split(".").collect::<Vec<&str>>().get(1).copied().unwrap_or_default().to_string(), true => sort
.split(".")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.to_string(),
false => "desc".to_string(), false => "desc".to_string(),
}; };
let old_items = match cache.get(&index) { let old_items = match cache.get(&index) {
@@ -408,7 +426,8 @@ impl HanimeProvider {
.order_by(order_by) .order_by(order_by)
.ordering(ordering); .ordering(ordering);
let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let response = match requester let response = match requester
.post_json("https://search.htv-services.com/search", &search, vec![]) .post_json("https://search.htv-services.com/search", &search, vec![])
.await .await

33
src/providers/hentaihaven.rs
View File

@@ -137,11 +137,7 @@ impl HentaihavenProvider {
options: ServerOptions, options: ServerOptions,
pool: DbPool, pool: DbPool,
) -> Result<Vec<VideoItem>> { ) -> Result<Vec<VideoItem>> {
let video_url = format!( let video_url = format!("{}/?s={}", self.url, query.replace(" ", "+"),);
"{}/?s={}",
self.url,
query.replace(" ", "+"),
);
// Check our Video Cache. If the result is younger than 1 hour, we return it. // Check our Video Cache. If the result is younger than 1 hour, we return it.
let old_items = match cache.get(&video_url) { let old_items = match cache.get(&video_url) {
Some((time, items)) => { Some((time, items)) => {
@@ -171,8 +167,7 @@ impl HentaihavenProvider {
return Ok(old_items); return Ok(old_items);
} }
}; };
if page > 1 if page > 1 {
{
return Ok(vec![]); return Ok(vec![]);
} }
let video_items: Vec<VideoItem> = self let video_items: Vec<VideoItem> = self
@@ -438,8 +433,7 @@ impl HentaihavenProvider {
.last() .last()
.and_then(|s| s.split("summary-content\">").nth(1)) .and_then(|s| s.split("summary-content\">").nth(1))
.and_then(|s| s.split(" Total").nth(0)) .and_then(|s| s.split(" Total").nth(0))
.map(|s| .map(|s| s.trim().parse::<u32>().unwrap_or(0))
s.trim().parse::<u32>().unwrap_or(0))
.unwrap_or(0); .unwrap_or(0);
let mut formats = vec![]; let mut formats = vec![];
let episode_block = html let episode_block = html
@@ -470,14 +464,23 @@ impl HentaihavenProvider {
let format = VideoFormat::new(episode_url, "1080p".to_string(), "m3u8".to_string()) let format = VideoFormat::new(episode_url, "1080p".to_string(), "m3u8".to_string())
.format_id(episode_title.clone()) .format_id(episode_title.clone())
.http_header("Connection".to_string(), "keep-alive".to_string()) .http_header("Connection".to_string(), "keep-alive".to_string())
.http_header("User-Agent".to_string(), "Mozilla/5.0 (X11; Linux x86_64; rv:140.0) Gecko/20100101 Firefox/140.0".to_string()) .http_header(
.http_header("Accept".to_string(), "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8".to_string()) "User-Agent".to_string(),
"Mozilla/5.0 (X11; Linux x86_64; rv:140.0) Gecko/20100101 Firefox/140.0"
.to_string(),
)
.http_header(
"Accept".to_string(),
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8".to_string(),
)
.http_header("Accept-Language".to_string(), "en-US,en;q=0.5".to_string()) .http_header("Accept-Language".to_string(), "en-US,en;q=0.5".to_string())
.http_header("Accept-Encoding".to_string(), "gzip, deflate, br".to_string()) .http_header(
"Accept-Encoding".to_string(),
"gzip, deflate, br".to_string(),
)
.http_header("Sec-Fetch-Mode".to_string(), "navigate".to_string()) .http_header("Sec-Fetch-Mode".to_string(), "navigate".to_string())
.http_header("Origin".to_string(), self.url.clone()) .http_header("Origin".to_string(), self.url.clone())
.format_note(episode_title.clone()) .format_note(episode_title.clone());
;
formats.push(format); formats.push(format);
} }
if formats.is_empty() { if formats.is_empty() {

173
src/providers/homoxxx.rs
View File

@@ -1,5 +1,5 @@
use crate::api::ClientVersion;
use crate::DbPool; use crate::DbPool;
use crate::api::ClientVersion;
use crate::providers::{Provider, report_provider_error}; use crate::providers::{Provider, report_provider_error};
use crate::status::*; use crate::status::*;
use crate::util::cache::VideoCache; use crate::util::cache::VideoCache;
@@ -11,7 +11,7 @@ use error_chain::error_chain;
use htmlentity::entity::{ICodedDataTrait, decode}; use htmlentity::entity::{ICodedDataTrait, decode};
use std::env; use std::env;
use std::vec; use std::vec;
use wreq::{Client}; use wreq::Client;
use wreq_util::Emulation; use wreq_util::Emulation;
error_chain! { error_chain! {
@@ -67,12 +67,7 @@ impl HomoxxxProvider {
cacheDuration: Some(1800), cacheDuration: Some(1800),
} }
} }
async fn get( async fn get(&self, cache: VideoCache, page: u8, sort: &str) -> Result<Vec<VideoItem>> {
&self,
cache: VideoCache,
page: u8,
sort: &str,
) -> Result<Vec<VideoItem>> {
let sort_string = match sort { let sort_string = match sort {
"trending" => "/trending", "trending" => "/trending",
"popular" => "/popular", "popular" => "/popular",
@@ -93,13 +88,22 @@ impl HomoxxxProvider {
}; };
// let proxy = Proxy::all("http://192.168.0.103:8081").unwrap(); // let proxy = Proxy::all("http://192.168.0.103:8081").unwrap();
let client = Client::builder().cert_verification(false).emulation(Emulation::Firefox136).build()?; let client = Client::builder()
.cert_verification(false)
.emulation(Emulation::Firefox136)
.build()?;
let mut response = client.get(video_url.clone()) let mut response = client
// .proxy(proxy.clone()) .get(video_url.clone())
.send().await?; // .proxy(proxy.clone())
.send()
.await?;
if response.status().is_redirection() { if response.status().is_redirection() {
let location = match response.headers().get("Location").and_then(|h| h.to_str().ok()) { let location = match response
.headers()
.get("Location")
.and_then(|h| h.to_str().ok())
{
Some(location) => location, Some(location) => location,
None => { None => {
report_provider_error( report_provider_error(
@@ -132,12 +136,7 @@ impl HomoxxxProvider {
let flare_url = match env::var("FLARE_URL") { let flare_url = match env::var("FLARE_URL") {
Ok(url) => url, Ok(url) => url,
Err(e) => { Err(e) => {
report_provider_error( report_provider_error("homoxxx", "get.flare_url", &e.to_string()).await;
"homoxxx",
"get.flare_url",
&e.to_string(),
)
.await;
return Ok(old_items); return Ok(old_items);
} }
}; };
@@ -168,17 +167,18 @@ impl HomoxxxProvider {
Ok(video_items) Ok(video_items)
} }
} }
async fn query( async fn query(&self, cache: VideoCache, page: u8, query: &str) -> Result<Vec<VideoItem>> {
&self,
cache: VideoCache,
page: u8,
query: &str,
) -> Result<Vec<VideoItem>> {
let search_string = query.to_lowercase().trim().replace(" ", "-"); let search_string = query.to_lowercase().trim().replace(" ", "-");
let mut video_url = format!("{}/search/{}/{}/", self.url, search_string, page); let mut video_url = format!("{}/search/{}/{}/", self.url, search_string, page);
if search_string.starts_with("@"){ if search_string.starts_with("@") {
let url_part = search_string.split("@").collect::<Vec<&str>>().get(1).copied().unwrap_or_default().replace(":", "/"); let url_part = search_string
.split("@")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.replace(":", "/");
video_url = format!("{}/{}/", self.url, url_part); video_url = format!("{}/{}/", self.url, url_part);
} }
// Check our Video Cache. If the result is younger than 1 hour, we return it. // Check our Video Cache. If the result is younger than 1 hour, we return it.
@@ -197,14 +197,23 @@ impl HomoxxxProvider {
}; };
// let proxy = Proxy::all("http://192.168.0.103:8081").unwrap(); // let proxy = Proxy::all("http://192.168.0.103:8081").unwrap();
let client = Client::builder().cert_verification(false).emulation(Emulation::Firefox136).build()?; let client = Client::builder()
.cert_verification(false)
.emulation(Emulation::Firefox136)
.build()?;
let mut response = client.get(video_url.clone()) let mut response = client
// .proxy(proxy.clone()) .get(video_url.clone())
.send().await?; // .proxy(proxy.clone())
.send()
.await?;
if response.status().is_redirection() { if response.status().is_redirection() {
let location = match response.headers().get("Location").and_then(|h| h.to_str().ok()) { let location = match response
.headers()
.get("Location")
.and_then(|h| h.to_str().ok())
{
Some(location) => location, Some(location) => location,
None => { None => {
report_provider_error( report_provider_error(
@@ -237,12 +246,7 @@ impl HomoxxxProvider {
let flare_url = match env::var("FLARE_URL") { let flare_url = match env::var("FLARE_URL") {
Ok(url) => url, Ok(url) => url,
Err(e) => { Err(e) => {
report_provider_error( report_provider_error("homoxxx", "query.flare_url", &e.to_string()).await;
"homoxxx",
"query.flare_url",
&e.to_string(),
)
.await;
return Ok(old_items); return Ok(old_items);
} }
}; };
@@ -277,7 +281,12 @@ impl HomoxxxProvider {
return vec![]; return vec![];
} }
let mut items: Vec<VideoItem> = Vec::new(); let mut items: Vec<VideoItem> = Vec::new();
let raw_videos = html.split("pagination").collect::<Vec<&str>>().get(0).copied().unwrap_or_default() let raw_videos = html
.split("pagination")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.split("<div class=\"item \">") .split("<div class=\"item \">")
.collect::<Vec<&str>>()[1..] .collect::<Vec<&str>>()[1..]
.to_vec(); .to_vec();
@@ -286,31 +295,81 @@ impl HomoxxxProvider {
// for (index, line) in vid.iter().enumerate() { // for (index, line) in vid.iter().enumerate() {
// println!("Line {}: {}", index, line); // println!("Line {}: {}", index, line);
// } // }
let video_url: String = video_segment.split("<a href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() let video_url: String = video_segment
.split("\"") .split("<a href=\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default().to_string(); .collect::<Vec<&str>>()
let preview_url = video_segment.split("data-preview-custom=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .get(1)
.copied()
.unwrap_or_default()
.split("\"") .split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(); .to_string();
let mut title = video_segment.split("\" title=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() let preview_url = video_segment
.split("data-preview-custom=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"") .split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
let mut title = video_segment
.split("\" title=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(); .to_string();
// html decode // html decode
title = decode(title.as_bytes()).to_string().unwrap_or(title); title = decode(title.as_bytes()).to_string().unwrap_or(title);
let id = video_url.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string(); let id = video_url
.split("/")
.collect::<Vec<&str>>()
.get(4)
.copied()
.unwrap_or_default()
.to_string();
let raw_duration = video_segment let raw_duration = video_segment
.split("<p class=\"duration_item\">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .split("<p class=\"duration_item\">")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<") .split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(); .to_string();
let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32; let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
let thumb = video_segment.split("thumb lazyload").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() let thumb = video_segment
.split("data-src=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .split("thumb lazyload")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("data-src=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"") .split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(); .to_string();
let video_item = VideoItem::new( let video_item = VideoItem::new(
id, id,
@@ -320,14 +379,11 @@ impl HomoxxxProvider {
thumb, thumb,
duration, duration,
) )
.preview(preview_url) .preview(preview_url);
;
items.push(video_item); items.push(video_item);
} }
return items; return items;
} }
} }
#[async_trait] #[async_trait]
@@ -346,10 +402,7 @@ impl Provider for HomoxxxProvider {
let _ = per_page; let _ = per_page;
let _ = pool; let _ = pool;
let videos: std::result::Result<Vec<VideoItem>, Error> = match query { let videos: std::result::Result<Vec<VideoItem>, Error> = match query {
Some(q) => { Some(q) => self.query(cache, page.parse::<u8>().unwrap_or(1), &q).await,
self.query(cache, page.parse::<u8>().unwrap_or(1), &q,)
.await
}
None => { None => {
self.get(cache, page.parse::<u8>().unwrap_or(1), &sort) self.get(cache, page.parse::<u8>().unwrap_or(1), &sort)
.await .await

181
src/providers/hqporner.rs
View File

@@ -9,10 +9,9 @@ use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoFormat, VideoItem}; use crate::videos::{ServerOptions, VideoFormat, VideoItem};
use async_trait::async_trait; use async_trait::async_trait;
use error_chain::error_chain; use error_chain::error_chain;
use futures::future::join_all; use futures::stream::{FuturesUnordered, StreamExt};
use htmlentity::entity::{ICodedDataTrait, decode}; use htmlentity::entity::{ICodedDataTrait, decode};
use std::sync::{Arc, RwLock}; use std::sync::{Arc, RwLock};
use std::thread::sleep;
use std::{thread, vec}; use std::{thread, vec};
use titlecase::Titlecase; use titlecase::Titlecase;
@@ -263,17 +262,37 @@ impl HqpornerProvider {
}) })
.unwrap_or_default(); .unwrap_or_default();
let futures = raw_videos // Limit concurrent detail-page requests to reduce transient connect errors.
.into_iter() let mut in_flight = FuturesUnordered::new();
.map(|el| self.get_video_item(el, requester.clone())); let mut iter = raw_videos.into_iter();
let mut items = Vec::new();
const MAX_IN_FLIGHT: usize = 6;
join_all(futures) loop {
.await while in_flight.len() < MAX_IN_FLIGHT {
.into_iter() let Some(seg) = iter.next() else {
.inspect(|r| { break;
if let Err(e) = r { };
in_flight.push(self.get_video_item(seg, requester.clone()));
}
let Some(result) = in_flight.next().await else {
break;
};
match result {
Ok(item)
if item
.formats
.as_ref()
.map(|formats| !formats.is_empty())
.unwrap_or(false) =>
{
items.push(item);
}
Ok(_) => {}
Err(e) => {
let msg = e.to_string(); let msg = e.to_string();
let chain = format_error_chain(e); let chain = format_error_chain(&e);
tokio::spawn(async move { tokio::spawn(async move {
let _ = send_discord_error_report( let _ = send_discord_error_report(
msg, msg,
@@ -287,10 +306,10 @@ impl HqpornerProvider {
.await; .await;
}); });
} }
}) }
.filter_map(Result::ok) }
.filter(|item| item.formats.as_ref().map(|formats| !formats.is_empty()).unwrap_or(false))
.collect() items
} }
async fn get_video_item(&self, seg: String, mut requester: Requester) -> Result<VideoItem> { async fn get_video_item(&self, seg: String, mut requester: Requester) -> Result<VideoItem> {
@@ -319,13 +338,22 @@ impl HqpornerProvider {
.and_then(|s| s.split('.').next()) .and_then(|s| s.split('.').next())
.ok_or_else(|| ErrorKind::Parse(format!("id \n{seg}").into()))? .ok_or_else(|| ErrorKind::Parse(format!("id \n{seg}").into()))?
.to_string(); .to_string();
let thumb = format!( let thumb_raw = seg
"https:{}", .split("onmouseleave='defaultImage(\"")
seg.split("onmouseleave='defaultImage(\"") .nth(1)
.nth(1) .and_then(|s| s.split('"').next())
.and_then(|s| s.split('"').next()) .ok_or_else(|| ErrorKind::Parse(format!("thumb \n{seg}").into()))?;
.ok_or_else(|| ErrorKind::Parse(format!("thumb \n{seg}").into()))? let thumb_abs = if thumb_raw.starts_with("//") {
); format!("https:{}", thumb_raw)
} else if thumb_raw.starts_with("http://") || thumb_raw.starts_with("https://") {
thumb_raw.to_string()
} else {
format!("https://{}", thumb_raw.trim_start_matches('/'))
};
let thumb = match thumb_abs.strip_prefix("https://") {
Some(path) => format!("https://hottub.spacemoehre.de/proxy/hqporner-thumb/{path}"),
None => thumb_abs,
};
let raw_duration = seg let raw_duration = seg
.split("<span class=\"icon fa-clock-o meta-data\">") .split("<span class=\"icon fa-clock-o meta-data\">")
.nth(1) .nth(1)
@@ -335,7 +363,7 @@ impl HqpornerProvider {
let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32; let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
let (tags, formats) = self.extract_media(&video_url, &mut requester).await?; let (tags, formats) = self.extract_media(&video_url, &mut requester).await?;
Ok( Ok(
VideoItem::new(id, title, video_url, "hqporner".into(), thumb, duration) VideoItem::new(id, title, video_url, "hqporner".into(), thumb, duration)
.formats(formats) .formats(formats)
@@ -350,17 +378,35 @@ impl HqpornerProvider {
) -> Result<(Vec<String>, Vec<VideoFormat>)> { ) -> Result<(Vec<String>, Vec<VideoFormat>)> {
let mut formats = vec![]; let mut formats = vec![];
let mut tags = vec![]; let mut tags = vec![];
let resp = requester let headers = vec![("Referer".to_string(), "https://hqporner.com/".into())];
.get_raw_with_headers( let mut text = match self
url, .fetch_text_with_retries(requester, url, &headers, 3)
vec![("Referer".to_string(), "https://hqporner.com/".into())],
)
.await .await
.map_err(|e| Error::from(format!("Request failed: {}", e)))?; {
let text = resp Ok(text) => text,
.text() Err(primary_err) => {
.await if url.contains("://hqporner.com/") {
.map_err(|e| Error::from(format!("Text conversion failed: {}", e)))?; let fallback_url = url.replace("://hqporner.com/", "://www.hqporner.com/");
self.fetch_text_with_retries(requester, &fallback_url, &headers, 3)
.await
.map_err(|fallback_err| {
Error::from(format!(
"Request failed: primary={primary_err}; fallback={fallback_err}"
))
})?
} else {
return Err(Error::from(format!("Request failed: {}", primary_err)));
}
}
};
if text.is_empty() && url.contains("://hqporner.com/") {
let fallback_url = url.replace("://hqporner.com/", "://www.hqporner.com/");
text = self
.fetch_text_with_retries(requester, &fallback_url, &headers, 3)
.await
.unwrap_or_default();
}
if text.contains("Why do I see it?") { if text.contains("Why do I see it?") {
return Ok((tags, formats)); return Ok((tags, formats));
@@ -395,22 +441,11 @@ impl HqpornerProvider {
.and_then(|s| s.split('\'').next()) .and_then(|s| s.split('\'').next())
.ok_or("No player link")? .ok_or("No player link")?
); );
let mut r = requester let response_text = match self
.get_raw_with_headers( .fetch_text_with_retries(requester, &player_url, &headers, 2)
&player_url, .await
vec![("Referer".to_string(), "https://hqporner.com/".into())], {
).await; Ok(text) => text,
if let Err(_e) = &r {
sleep(std::time::Duration::from_secs(1));
r = requester
.get_raw_with_headers(
&player_url,
vec![("Referer".to_string(), "https://hqporner.com/".into())],
).await;
}
let response = match r {
Ok(response) => response,
Err(e) => { Err(e) => {
let err = format!("altplayer request failed: {e}"); let err = format!("altplayer request failed: {e}");
send_discord_error_report( send_discord_error_report(
@@ -426,13 +461,10 @@ impl HqpornerProvider {
return Ok((tags, formats)); return Ok((tags, formats));
} }
}; };
let text2 = response let text2 = response_text;
.text()
.await
.map_err(|e| Error::from(format!("Text conversion failed: {}", e)))?;
// Check for error response // Check for error response
if text2.starts_with("ERR:"){ if text2.starts_with("ERR:") {
return Ok((tags, formats)); return Ok((tags, formats));
} }
@@ -467,6 +499,37 @@ impl HqpornerProvider {
Ok((tags, formats)) Ok((tags, formats))
} }
async fn fetch_text_with_retries(
&self,
requester: &mut Requester,
url: &str,
headers: &[(String, String)],
max_attempts: u8,
) -> std::result::Result<String, String> {
let mut last_err = String::new();
for attempt in 1..=max_attempts {
match requester.get_raw_with_headers(url, headers.to_vec()).await {
Ok(resp) => match resp.text().await {
Ok(text) => return Ok(text),
Err(e) => {
last_err =
format!("text read failed (attempt {attempt}/{max_attempts}): {e}");
}
},
Err(e) => {
last_err = format!("request failed (attempt {attempt}/{max_attempts}): {e}");
}
}
if attempt < max_attempts {
tokio::time::sleep(std::time::Duration::from_millis(250 * attempt as u64)).await;
}
}
Err(last_err)
}
} }
#[async_trait] #[async_trait]
@@ -488,7 +551,15 @@ impl Provider for HqpornerProvider {
}; };
res.unwrap_or_else(|e| { res.unwrap_or_else(|e| {
eprintln!("Hqporner error: {e}"); eprintln!("Hqporner error: {e}");
let _ = send_discord_error_report(e.to_string(), Some(format_error_chain(&e)), None, None, file!(), line!(), module_path!()); let _ = send_discord_error_report(
e.to_string(),
Some(format_error_chain(&e)),
None,
None,
file!(),
line!(),
module_path!(),
);
vec![] vec![]
}) })
} }

6
src/providers/hypnotube.rs
View File

@@ -212,7 +212,8 @@ impl HypnotubeProvider {
vec![] vec![]
} }
}; };
let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let text = match requester.get(&video_url, Some(Version::HTTP_11)).await { let text = match requester.get(&video_url, Some(Version::HTTP_11)).await {
Ok(text) => text, Ok(text) => text,
Err(e) => { Err(e) => {
@@ -273,7 +274,8 @@ impl HypnotubeProvider {
} }
}; };
let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let post_response = match requester let post_response = match requester
.post( .post(
format!("{}/searchgate.php", self.url).as_str(), format!("{}/searchgate.php", self.url).as_str(),

111
src/providers/javtiful.rs
View File

@@ -11,9 +11,9 @@ use crate::videos::{ServerOptions, VideoFormat, VideoItem};
use async_trait::async_trait; use async_trait::async_trait;
use error_chain::error_chain; use error_chain::error_chain;
use futures::future::join_all; use futures::future::join_all;
use htmlentity::entity::{decode, ICodedDataTrait}; use htmlentity::entity::{ICodedDataTrait, decode};
use std::sync::{Arc, RwLock}; use std::sync::{Arc, RwLock};
use std::{vec}; use std::vec;
use titlecase::Titlecase; use titlecase::Titlecase;
use wreq::Version; use wreq::Version;
@@ -118,10 +118,7 @@ impl JavtifulProvider {
"most viewed" => "/sort=most_viewed", "most viewed" => "/sort=most_viewed",
_ => "", _ => "",
}; };
let video_url = format!( let video_url = format!("{}/videos{}?page={}", self.url, sort_string, page);
"{}/videos{}?page={}",
self.url, sort_string, page
);
let old_items = match cache.get(&video_url) { let old_items = match cache.get(&video_url) {
Some((time, items)) => { Some((time, items)) => {
if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
@@ -149,7 +146,12 @@ impl JavtifulProvider {
return Ok(old_items); return Ok(old_items);
} }
}; };
if page > 1 && !text.contains(&format!("<li class=\"page-item active\"><span class=\"page-link\">{}</span>", page)) { if page > 1
&& !text.contains(&format!(
"<li class=\"page-item active\"><span class=\"page-link\">{}</span>",
page
))
{
return Ok(vec![]); return Ok(vec![]);
} }
let video_items: Vec<VideoItem> = self let video_items: Vec<VideoItem> = self
@@ -178,7 +180,10 @@ impl JavtifulProvider {
}; };
let video_url = format!( let video_url = format!(
"{}/search/videos{}?search_query={}&page={}", "{}/search/videos{}?search_query={}&page={}",
self.url, sort_string, query.replace(" ","+"), page self.url,
sort_string,
query.replace(" ", "+"),
page
); );
// Check our Video Cache. If the result is younger than 1 hour, we return it. // Check our Video Cache. If the result is younger than 1 hour, we return it.
let old_items = match cache.get(&video_url) { let old_items = match cache.get(&video_url) {
@@ -209,7 +214,12 @@ impl JavtifulProvider {
return Ok(old_items); return Ok(old_items);
} }
}; };
if page > 1 && !text.contains(&format!("<li class=\"page-item active\"><span class=\"page-link\">{}</span>", page)) { if page > 1
&& !text.contains(&format!(
"<li class=\"page-item active\"><span class=\"page-link\">{}</span>",
page
))
{
return Ok(vec![]); return Ok(vec![]);
} }
let video_items: Vec<VideoItem> = self let video_items: Vec<VideoItem> = self
@@ -233,11 +243,10 @@ impl JavtifulProvider {
return vec![]; return vec![];
} }
let block = match html let block = match html.split("pagination ").next().and_then(|s| {
.split("pagination ") s.split("row row-cols-1 row-cols-sm-2 row-cols-lg-3 row-cols-xl-4")
.next() .nth(1)
.and_then(|s| s.split("row row-cols-1 row-cols-sm-2 row-cols-lg-3 row-cols-xl-4").nth(1)) }) {
{
Some(b) => b, Some(b) => b,
None => { None => {
eprint!("Javtiful Provider: Failed to get block from html"); eprint!("Javtiful Provider: Failed to get block from html");
@@ -250,9 +259,10 @@ impl JavtifulProvider {
file!(), file!(),
line!(), line!(),
module_path!(), module_path!(),
).await; )
return vec![] .await;
}, return vec![];
}
}; };
let futures = block let futures = block
@@ -281,7 +291,8 @@ impl JavtifulProvider {
file!(), // Note: these might report the utility line file!(), // Note: these might report the utility line
line!(), // better to hardcode or pass from outside line!(), // better to hardcode or pass from outside
module_path!(), module_path!(),
).await; )
.await;
}); });
} }
}) })
@@ -289,11 +300,7 @@ impl JavtifulProvider {
.collect() .collect()
} }
async fn get_video_item( async fn get_video_item(&self, seg: String, mut requester: Requester) -> Result<VideoItem> {
&self,
seg: String,
mut requester: Requester,
) -> Result<VideoItem> {
let video_url = seg let video_url = seg
.split(" href=\"") .split(" href=\"")
.nth(1) .nth(1)
@@ -309,7 +316,10 @@ impl JavtifulProvider {
.trim() .trim()
.to_string(); .to_string();
title = decode(title.as_bytes()).to_string().unwrap_or(title).titlecase(); title = decode(title.as_bytes())
.to_string()
.unwrap_or(title)
.titlecase();
let id = video_url let id = video_url
.split('/') .split('/')
.nth(5) .nth(5)
@@ -340,26 +350,17 @@ impl JavtifulProvider {
.unwrap_or("") .unwrap_or("")
.to_string(); .to_string();
let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32; let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
let (tags, formats, views) = let (tags, formats, views) = self.extract_media(&video_url, &mut requester).await?;
self.extract_media(&video_url, &mut requester).await?;
if preview.len() == 0 { if preview.len() == 0 {
preview = format!("https://trailers.jav.si/preview/{id}.mp4"); preview = format!("https://trailers.jav.si/preview/{id}.mp4");
} }
let video_item = VideoItem::new( let video_item = VideoItem::new(id, title, video_url, "javtiful".into(), thumb, duration)
id, .formats(formats)
title, .tags(tags)
video_url, .preview(preview)
"javtiful".into(), .views(views);
thumb,
duration,
)
.formats(formats)
.tags(tags)
.preview(preview)
.views(views);
Ok(video_item) Ok(video_item)
} }
async fn extract_media( async fn extract_media(
@@ -371,7 +372,9 @@ impl JavtifulProvider {
.get(url, Some(Version::HTTP_2)) .get(url, Some(Version::HTTP_2))
.await .await
.map_err(|e| Error::from(format!("{}", e)))?; .map_err(|e| Error::from(format!("{}", e)))?;
let tags = text.split("related-actress").next() let tags = text
.split("related-actress")
.next()
.and_then(|s| s.split("video-comments").next()) .and_then(|s| s.split("video-comments").next())
.and_then(|s| s.split(">Tags<").nth(1)) .and_then(|s| s.split(">Tags<").nth(1))
.map(|tag_block| { .map(|tag_block| {
@@ -383,26 +386,34 @@ impl JavtifulProvider {
.split('>') .split('>')
.nth(1) .nth(1)
.and_then(|s| s.split('<').next()) .and_then(|s| s.split('<').next())
.map(|s| decode(s.as_bytes()).to_string().unwrap_or(s.to_string()).titlecase()) .map(|s| {
decode(s.as_bytes())
.to_string()
.unwrap_or(s.to_string())
.titlecase()
})
}) })
.collect() .collect()
}) })
.unwrap_or_else(|| vec![]); .unwrap_or_else(|| vec![]);
for tag in &tags { for tag in &tags {
Self::push_unique(&self.categories, FilterOption { Self::push_unique(
id: tag.to_ascii_lowercase().replace(" ","+"), &self.categories,
title: tag.to_string(), FilterOption {
}); id: tag.to_ascii_lowercase().replace(" ", "+"),
title: tag.to_string(),
},
);
} }
let views = text.split(" Views ") let views = text
.split(" Views ")
.next() .next()
.and_then(|s| s.split(" ").last()) .and_then(|s| s.split(" ").last())
.and_then(|s| s.replace(".","") .and_then(|s| s.replace(".", "").parse::<u32>().ok())
.parse::<u32>().ok())
.unwrap_or(0); .unwrap_or(0);
let quality="1080p".to_string(); let quality = "1080p".to_string();
let video_url = url.replace("javtiful.com","hottub.spacemoehre.de/proxy/javtiful"); let video_url = url.replace("javtiful.com", "hottub.spacemoehre.de/proxy/javtiful");
Ok(( Ok((
tags, tags,
vec![VideoFormat::new(video_url, quality, "video/mp4".into())], vec![VideoFormat::new(video_url, quality, "video/mp4".into())],

216
src/providers/missav.rs
View File

@@ -1,20 +1,20 @@
use std::vec; use crate::DbPool;
use async_trait::async_trait;
use diesel::r2d2;
use error_chain::error_chain;
use htmlentity::entity::{decode, ICodedDataTrait};
use futures::future::join_all;
use wreq::Version;
use crate::api::ClientVersion; use crate::api::ClientVersion;
use crate::db; use crate::db;
use crate::providers::Provider; use crate::providers::Provider;
use crate::status::*; use crate::status::*;
use crate::util::cache::VideoCache; use crate::util::cache::VideoCache;
use crate::util::discord::{format_error_chain, send_discord_error_report}; use crate::util::discord::{format_error_chain, send_discord_error_report};
use crate::videos::ServerOptions;
use crate::videos::{VideoItem};
use crate::DbPool;
use crate::util::requester::Requester; use crate::util::requester::Requester;
use crate::videos::ServerOptions;
use crate::videos::VideoItem;
use async_trait::async_trait;
use diesel::r2d2;
use error_chain::error_chain;
use futures::future::join_all;
use htmlentity::entity::{ICodedDataTrait, decode};
use std::vec;
use wreq::Version;
error_chain! { error_chain! {
foreign_links { foreign_links {
@@ -39,7 +39,7 @@ pub struct MissavProvider {
impl MissavProvider { impl MissavProvider {
pub fn new() -> Self { pub fn new() -> Self {
MissavProvider { MissavProvider {
url: "https://missav.ws".to_string() url: "https://missav.ws".to_string(),
} }
} }
@@ -177,7 +177,14 @@ impl MissavProvider {
} }
} }
async fn get(&self, cache: VideoCache, pool: DbPool, page: u8, mut sort: String, options: ServerOptions) -> Result<Vec<VideoItem>> { async fn get(
&self,
cache: VideoCache,
pool: DbPool,
page: u8,
mut sort: String,
options: ServerOptions,
) -> Result<Vec<VideoItem>> {
// Use ok_or to avoid unwrapping options // Use ok_or to avoid unwrapping options
let language = options.language.as_ref().ok_or("Missing language")?; let language = options.language.as_ref().ok_or("Missing language")?;
let filter = options.filter.as_ref().ok_or("Missing filter")?; let filter = options.filter.as_ref().ok_or("Missing filter")?;
@@ -187,35 +194,57 @@ impl MissavProvider {
sort = format!("&sort={}", sort); sort = format!("&sort={}", sort);
} }
let url_str = format!("{}/{}/{}?page={}{}", self.url, language, filter, page, sort); let url_str = format!("{}/{}/{}?page={}{}", self.url, language, filter, page, sort);
if let Some((time, items)) = cache.get(&url_str) { if let Some((time, items)) = cache.get(&url_str) {
if time.elapsed().unwrap_or_default().as_secs() < 3600 { if time.elapsed().unwrap_or_default().as_secs() < 3600 {
return Ok(items.clone()); return Ok(items.clone());
} }
} }
let text = requester.get(&url_str, Some(Version::HTTP_2)).await.unwrap_or_else(|e| { let text = requester
eprintln!("Error fetching Missav URL {}: {}", url_str, e); .get(&url_str, Some(Version::HTTP_2))
let _ = send_discord_error_report(e.to_string(), None, Some(&url_str), None, file!(), line!(), module_path!()); .await
"".to_string() .unwrap_or_else(|e| {
}); eprintln!("Error fetching Missav URL {}: {}", url_str, e);
let _ = send_discord_error_report(
e.to_string(),
None,
Some(&url_str),
None,
file!(),
line!(),
module_path!(),
);
"".to_string()
});
let video_items = self.get_video_items_from_html(text, pool, requester).await; let video_items = self.get_video_items_from_html(text, pool, requester).await;
if !video_items.is_empty() { if !video_items.is_empty() {
cache.insert(url_str, video_items.clone()); cache.insert(url_str, video_items.clone());
} }
Ok(video_items) Ok(video_items)
} }
async fn query(&self, cache: VideoCache, pool: DbPool, page: u8, query: &str, mut sort: String, options: ServerOptions) -> Result<Vec<VideoItem>> { async fn query(
&self,
cache: VideoCache,
pool: DbPool,
page: u8,
query: &str,
mut sort: String,
options: ServerOptions,
) -> Result<Vec<VideoItem>> {
let language = options.language.as_ref().ok_or("Missing language")?; let language = options.language.as_ref().ok_or("Missing language")?;
let mut requester = options.requester.clone().ok_or("Missing requester")?; let mut requester = options.requester.clone().ok_or("Missing requester")?;
let search_string = query.replace(" ", "%20"); let search_string = query.replace(" ", "%20");
if !sort.is_empty() { if !sort.is_empty() {
sort = format!("&sort={}", sort); sort = format!("&sort={}", sort);
} }
let url_str = format!("{}/{}/search/{}?page={}{}", self.url, language, search_string, page, sort); let url_str = format!(
"{}/{}/search/{}?page={}{}",
self.url, language, search_string, page, sort
);
if let Some((time, items)) = cache.get(&url_str) { if let Some((time, items)) = cache.get(&url_str) {
if time.elapsed().unwrap_or_default().as_secs() < 3600 { if time.elapsed().unwrap_or_default().as_secs() < 3600 {
@@ -223,24 +252,44 @@ impl MissavProvider {
} }
} }
let text = requester.get(&url_str, Some(Version::HTTP_2)).await.unwrap_or_else(|e| { let text = requester
eprintln!("Error fetching Missav URL {}: {}", url_str, e); .get(&url_str, Some(Version::HTTP_2))
let _ = send_discord_error_report(e.to_string(), None, Some(&url_str), None, file!(), line!(), module_path!()); .await
"".to_string() .unwrap_or_else(|e| {
}); eprintln!("Error fetching Missav URL {}: {}", url_str, e);
let _ = send_discord_error_report(
e.to_string(),
None,
Some(&url_str),
None,
file!(),
line!(),
module_path!(),
);
"".to_string()
});
let video_items = self.get_video_items_from_html(text, pool, requester).await; let video_items = self.get_video_items_from_html(text, pool, requester).await;
if !video_items.is_empty() { if !video_items.is_empty() {
cache.insert(url_str, video_items.clone()); cache.insert(url_str, video_items.clone());
} }
Ok(video_items) Ok(video_items)
} }
async fn get_video_items_from_html(&self, html: String, pool: DbPool, requester: Requester) -> Vec<VideoItem> { async fn get_video_items_from_html(
if html.is_empty() { return vec![]; } &self,
html: String,
pool: DbPool,
requester: Requester,
) -> Vec<VideoItem> {
if html.is_empty() {
return vec![];
}
let segments: Vec<&str> = html.split("@mouseenter=\"setPreview(\'").collect(); let segments: Vec<&str> = html.split("@mouseenter=\"setPreview(\'").collect();
if segments.len() < 2 { return vec![]; } if segments.len() < 2 {
return vec![];
}
let mut urls = vec![]; let mut urls = vec![];
for video_segment in &segments[1..] { for video_segment in &segments[1..] {
@@ -253,14 +302,27 @@ impl MissavProvider {
} }
} }
let futures = urls.into_iter().map(|url| self.get_video_item(url, pool.clone(), requester.clone())); let futures = urls
join_all(futures).await.into_iter().filter_map(Result::ok).collect() .into_iter()
.map(|url| self.get_video_item(url, pool.clone(), requester.clone()));
join_all(futures)
.await
.into_iter()
.filter_map(Result::ok)
.collect()
} }
async fn get_video_item(&self, url_str: String, pool: DbPool, mut requester: Requester) -> Result<VideoItem> { async fn get_video_item(
&self,
url_str: String,
pool: DbPool,
mut requester: Requester,
) -> Result<VideoItem> {
// 1. Database Check // 1. Database Check
{ {
let mut conn = pool.get().map_err(|e| Error::from(format!("Pool error: {}", e)))?; let mut conn = pool
.get()
.map_err(|e| Error::from(format!("Pool error: {}", e)))?;
if let Ok(Some(entry)) = db::get_video(&mut conn, url_str.clone()) { if let Ok(Some(entry)) = db::get_video(&mut conn, url_str.clone()) {
if let Ok(video_item) = serde_json::from_str::<VideoItem>(entry.as_str()) { if let Ok(video_item) = serde_json::from_str::<VideoItem>(entry.as_str()) {
return Ok(video_item); return Ok(video_item);
@@ -269,11 +331,22 @@ impl MissavProvider {
} }
// 2. Fetch Page // 2. Fetch Page
let vid = requester.get(&url_str, Some(Version::HTTP_2)).await.unwrap_or_else(|e| { let vid = requester
eprintln!("Error fetching Missav URL {}: {}", url_str, e); .get(&url_str, Some(Version::HTTP_2))
let _ = send_discord_error_report(e.to_string(), None, Some(&url_str), None, file!(), line!(), module_path!()); .await
"".to_string() .unwrap_or_else(|e| {
}); eprintln!("Error fetching Missav URL {}: {}", url_str, e);
let _ = send_discord_error_report(
e.to_string(),
None,
Some(&url_str),
None,
file!(),
line!(),
module_path!(),
);
"".to_string()
});
// Helper closure to extract content between two strings // Helper closure to extract content between two strings
let extract = |html: &str, start_tag: &str, end_tag: &str| -> Option<String> { let extract = |html: &str, start_tag: &str, end_tag: &str| -> Option<String> {
@@ -285,24 +358,33 @@ impl MissavProvider {
let mut title = extract(&vid, "<meta property=\"og:title\" content=\"", "\"") let mut title = extract(&vid, "<meta property=\"og:title\" content=\"", "\"")
.ok_or_else(|| ErrorKind::ParsingError(format!("title\n{:?}", vid)))?; .ok_or_else(|| ErrorKind::ParsingError(format!("title\n{:?}", vid)))?;
title = decode(title.as_bytes()).to_string().unwrap_or(title); title = decode(title.as_bytes()).to_string().unwrap_or(title);
if url_str.contains("uncensored") { if url_str.contains("uncensored") {
title = format!("[Uncensored] {}", title); title = format!("[Uncensored] {}", title);
} }
let thumb = extract(&vid, "<meta property=\"og:image\" content=\"", "\"") let thumb =
.unwrap_or_default(); extract(&vid, "<meta property=\"og:image\" content=\"", "\"").unwrap_or_default();
let duration = extract(&vid, "<meta property=\"og:video:duration\" content=\"", "\"") let duration = extract(
.and_then(|d| d.parse::<u32>().ok()) &vid,
.unwrap_or(0); "<meta property=\"og:video:duration\" content=\"",
"\"",
)
.and_then(|d| d.parse::<u32>().ok())
.unwrap_or(0);
let id = url_str.split('/').last().ok_or("No ID found")?.to_string(); let id = url_str.split('/').last().ok_or("No ID found")?.to_string();
// 3. Extract Tags (Generic approach to avoid repetitive code) // 3. Extract Tags (Generic approach to avoid repetitive code)
let mut tags = vec![]; let mut tags = vec![];
for (label, prefix) in [("Actress:", "@actress"), ("Actor:", "@actor"), ("Maker:", "@maker"), ("Genre:", "@genre")] { for (label, prefix) in [
("Actress:", "@actress"),
("Actor:", "@actor"),
("Maker:", "@maker"),
("Genre:", "@genre"),
] {
let marker = format!("<span>{}</span>", label); let marker = format!("<span>{}</span>", label);
if let Some(section) = extract(&vid, &marker, "</div>") { if let Some(section) = extract(&vid, &marker, "</div>") {
for part in section.split("class=\"text-nord13 font-medium\">").skip(1) { for part in section.split("class=\"text-nord13 font-medium\">").skip(1) {
@@ -331,15 +413,24 @@ impl MissavProvider {
parts.get(6)?, parts.get(6)?,
parts.get(7)? parts.get(7)?
)) ))
})().ok_or_else(|| ErrorKind::ParsingError(format!("video_url\n{:?}", vid).to_string()))?; })()
.ok_or_else(|| ErrorKind::ParsingError(format!("video_url\n{:?}", vid).to_string()))?;
let video_item = VideoItem::new(id, title, video_url, "missav".to_string(), thumb, duration) let video_item =
.tags(tags) VideoItem::new(id, title, video_url, "missav".to_string(), thumb, duration)
.preview(format!("https://fourhoi.com/{}/preview.mp4", url_str.split('/').last().unwrap_or_default())); .tags(tags)
.preview(format!(
"https://fourhoi.com/{}/preview.mp4",
url_str.split('/').last().unwrap_or_default()
));
// 5. Cache to DB // 5. Cache to DB
if let Ok(mut conn) = pool.get() { if let Ok(mut conn) = pool.get() {
let _ = db::insert_video(&mut conn, &url_str, &serde_json::to_string(&video_item).unwrap_or_default()); let _ = db::insert_video(
&mut conn,
&url_str,
&serde_json::to_string(&video_item).unwrap_or_default(),
);
} }
Ok(video_item) Ok(video_item)
@@ -348,7 +439,16 @@ impl MissavProvider {
#[async_trait] #[async_trait]
impl Provider for MissavProvider { impl Provider for MissavProvider {
async fn get_videos(&self, cache: VideoCache, pool: DbPool, sort: String, query: Option<String>, page: String, _per_page: String, options: ServerOptions) -> Vec<VideoItem> { async fn get_videos(
&self,
cache: VideoCache,
pool: DbPool,
sort: String,
query: Option<String>,
page: String,
_per_page: String,
options: ServerOptions,
) -> Vec<VideoItem> {
let page_num = page.parse::<u8>().unwrap_or(1); let page_num = page.parse::<u8>().unwrap_or(1);
let result = match query { let result = match query {
Some(q) => self.query(cache, pool, page_num, &q, sort, options).await, Some(q) => self.query(cache, pool, page_num, &q, sort, options).await,
@@ -357,7 +457,15 @@ impl Provider for MissavProvider {
result.unwrap_or_else(|e| { result.unwrap_or_else(|e| {
eprintln!("Error fetching videos: {}", e); eprintln!("Error fetching videos: {}", e);
let _ = send_discord_error_report(e.to_string(), Some(format_error_chain(&e)), None, None, file!(), line!(), module_path!()); let _ = send_discord_error_report(
e.to_string(),
Some(format_error_chain(&e)),
None,
None,
file!(),
line!(),
module_path!(),
);
vec![] vec![]
}) })
} }

174
src/providers/mod.rs
View File

@@ -7,7 +7,11 @@ use std::panic::AssertUnwindSafe;
use std::sync::Arc; use std::sync::Arc;
use crate::{ use crate::{
DbPool, api::ClientVersion, status::Channel, util::{cache::VideoCache, discord::send_discord_error_report, requester::Requester}, videos::{ServerOptions, VideoItem} DbPool,
api::ClientVersion,
status::Channel,
util::{cache::VideoCache, discord::send_discord_error_report, requester::Requester},
videos::{ServerOptions, VideoItem},
}; };
pub mod all; pub mod all;
@@ -24,27 +28,27 @@ pub mod pornhat;
pub mod redtube; pub mod redtube;
pub mod rule34video; pub mod rule34video;
// pub mod hentaimoon; // pub mod hentaimoon;
pub mod beeg;
pub mod missav; pub mod missav;
pub mod porn00;
pub mod sxyprn;
pub mod xxthots;
pub mod omgxxx; pub mod omgxxx;
pub mod paradisehill; pub mod paradisehill;
pub mod porn00;
pub mod pornzog; pub mod pornzog;
pub mod youjizz; pub mod sxyprn;
pub mod beeg;
pub mod tnaflix; pub mod tnaflix;
pub mod xxthots;
pub mod youjizz;
// pub mod pornxp; // pub mod pornxp;
pub mod rule34gen; pub mod chaturbate;
pub mod xxdbx;
pub mod hqporner;
pub mod noodlemagazine;
pub mod pimpbunny;
pub mod javtiful;
pub mod hypnotube;
pub mod freepornvideosxxx; pub mod freepornvideosxxx;
pub mod hentaihaven; pub mod hentaihaven;
pub mod chaturbate; pub mod hqporner;
pub mod hypnotube;
pub mod javtiful;
pub mod noodlemagazine;
pub mod pimpbunny;
pub mod rule34gen;
pub mod xxdbx;
// pub mod tube8; // pub mod tube8;
// convenient alias // convenient alias
@@ -53,47 +57,128 @@ pub type DynProvider = Arc<dyn Provider>;
pub static ALL_PROVIDERS: Lazy<HashMap<&'static str, DynProvider>> = Lazy::new(|| { pub static ALL_PROVIDERS: Lazy<HashMap<&'static str, DynProvider>> = Lazy::new(|| {
let mut m = HashMap::default(); let mut m = HashMap::default();
m.insert("all", Arc::new(all::AllProvider::new()) as DynProvider); m.insert("all", Arc::new(all::AllProvider::new()) as DynProvider);
m.insert("perverzija", Arc::new(perverzija::PerverzijaProvider::new()) as DynProvider); m.insert(
m.insert("hanime", Arc::new(hanime::HanimeProvider::new()) as DynProvider); "perverzija",
m.insert("pornhub", Arc::new(pornhub::PornhubProvider::new()) as DynProvider); Arc::new(perverzija::PerverzijaProvider::new()) as DynProvider,
);
m.insert(
"hanime",
Arc::new(hanime::HanimeProvider::new()) as DynProvider,
);
m.insert(
"pornhub",
Arc::new(pornhub::PornhubProvider::new()) as DynProvider,
);
m.insert( m.insert(
"rule34video", "rule34video",
Arc::new(rule34video::Rule34videoProvider::new()) as DynProvider, Arc::new(rule34video::Rule34videoProvider::new()) as DynProvider,
); );
m.insert("redtube", Arc::new(redtube::RedtubeProvider::new()) as DynProvider); m.insert(
m.insert("okporn", Arc::new(okporn::OkpornProvider::new()) as DynProvider); "redtube",
m.insert("pornhat", Arc::new(pornhat::PornhatProvider::new()) as DynProvider); Arc::new(redtube::RedtubeProvider::new()) as DynProvider,
);
m.insert(
"okporn",
Arc::new(okporn::OkpornProvider::new()) as DynProvider,
);
m.insert(
"pornhat",
Arc::new(pornhat::PornhatProvider::new()) as DynProvider,
);
m.insert( m.insert(
"perfectgirls", "perfectgirls",
Arc::new(perfectgirls::PerfectgirlsProvider::new()) as DynProvider, Arc::new(perfectgirls::PerfectgirlsProvider::new()) as DynProvider,
); );
m.insert("okxxx", Arc::new(okxxx::OkxxxProvider::new()) as DynProvider); m.insert(
m.insert("homoxxx", Arc::new(homoxxx::HomoxxxProvider::new()) as DynProvider); "okxxx",
m.insert("missav", Arc::new(missav::MissavProvider::new()) as DynProvider); Arc::new(okxxx::OkxxxProvider::new()) as DynProvider,
m.insert("xxthots", Arc::new(xxthots::XxthotsProvider::new()) as DynProvider); );
m.insert("sxyprn", Arc::new(sxyprn::SxyprnProvider::new()) as DynProvider); m.insert(
m.insert("porn00", Arc::new(porn00::Porn00Provider::new()) as DynProvider); "homoxxx",
m.insert("youjizz", Arc::new(youjizz::YoujizzProvider::new()) as DynProvider); Arc::new(homoxxx::HomoxxxProvider::new()) as DynProvider,
);
m.insert(
"missav",
Arc::new(missav::MissavProvider::new()) as DynProvider,
);
m.insert(
"xxthots",
Arc::new(xxthots::XxthotsProvider::new()) as DynProvider,
);
m.insert(
"sxyprn",
Arc::new(sxyprn::SxyprnProvider::new()) as DynProvider,
);
m.insert(
"porn00",
Arc::new(porn00::Porn00Provider::new()) as DynProvider,
);
m.insert(
"youjizz",
Arc::new(youjizz::YoujizzProvider::new()) as DynProvider,
);
m.insert( m.insert(
"paradisehill", "paradisehill",
Arc::new(paradisehill::ParadisehillProvider::new()) as DynProvider, Arc::new(paradisehill::ParadisehillProvider::new()) as DynProvider,
); );
m.insert("pornzog", Arc::new(pornzog::PornzogProvider::new()) as DynProvider); m.insert(
m.insert("omgxxx", Arc::new(omgxxx::OmgxxxProvider::new()) as DynProvider); "pornzog",
Arc::new(pornzog::PornzogProvider::new()) as DynProvider,
);
m.insert(
"omgxxx",
Arc::new(omgxxx::OmgxxxProvider::new()) as DynProvider,
);
m.insert("beeg", Arc::new(beeg::BeegProvider::new()) as DynProvider); m.insert("beeg", Arc::new(beeg::BeegProvider::new()) as DynProvider);
m.insert("tnaflix", Arc::new(tnaflix::TnaflixProvider::new()) as DynProvider); m.insert(
"tnaflix",
Arc::new(tnaflix::TnaflixProvider::new()) as DynProvider,
);
// m.insert("pornxp", Arc::new(pornxp::PornxpProvider::new()) as DynProvider); // m.insert("pornxp", Arc::new(pornxp::PornxpProvider::new()) as DynProvider);
m.insert("rule34gen", Arc::new(rule34gen::Rule34genProvider::new()) as DynProvider); m.insert(
m.insert("xxdbx", Arc::new(xxdbx::XxdbxProvider::new()) as DynProvider); "rule34gen",
m.insert("hqporner", Arc::new(hqporner::HqpornerProvider::new()) as DynProvider); Arc::new(rule34gen::Rule34genProvider::new()) as DynProvider,
m.insert("pmvhaven", Arc::new(pmvhaven::PmvhavenProvider::new()) as DynProvider); );
m.insert("noodlemagazine", Arc::new(noodlemagazine::NoodlemagazineProvider::new()) as DynProvider); m.insert(
m.insert("pimpbunny", Arc::new(pimpbunny::PimpbunnyProvider::new()) as DynProvider); "xxdbx",
m.insert("javtiful", Arc::new(javtiful::JavtifulProvider::new()) as DynProvider); Arc::new(xxdbx::XxdbxProvider::new()) as DynProvider,
m.insert("hypnotube", Arc::new(hypnotube::HypnotubeProvider::new()) as DynProvider); );
m.insert("freepornvideosxxx", Arc::new(freepornvideosxxx::FreepornvideosxxxProvider::new()) as DynProvider); m.insert(
m.insert("hentaihaven", Arc::new(hentaihaven::HentaihavenProvider::new()) as DynProvider); "hqporner",
m.insert("chaturbate", Arc::new(chaturbate::ChaturbateProvider::new()) as DynProvider); Arc::new(hqporner::HqpornerProvider::new()) as DynProvider,
);
m.insert(
"pmvhaven",
Arc::new(pmvhaven::PmvhavenProvider::new()) as DynProvider,
);
m.insert(
"noodlemagazine",
Arc::new(noodlemagazine::NoodlemagazineProvider::new()) as DynProvider,
);
m.insert(
"pimpbunny",
Arc::new(pimpbunny::PimpbunnyProvider::new()) as DynProvider,
);
m.insert(
"javtiful",
Arc::new(javtiful::JavtifulProvider::new()) as DynProvider,
);
m.insert(
"hypnotube",
Arc::new(hypnotube::HypnotubeProvider::new()) as DynProvider,
);
m.insert(
"freepornvideosxxx",
Arc::new(freepornvideosxxx::FreepornvideosxxxProvider::new()) as DynProvider,
);
m.insert(
"hentaihaven",
Arc::new(hentaihaven::HentaihavenProvider::new()) as DynProvider,
);
m.insert(
"chaturbate",
Arc::new(chaturbate::ChaturbateProvider::new()) as DynProvider,
);
// m.insert("tube8", Arc::new(tube8::Tube8Provider::new()) as DynProvider); // m.insert("tube8", Arc::new(tube8::Tube8Provider::new()) as DynProvider);
// add more here as you migrate them // add more here as you migrate them
m m
@@ -159,7 +244,11 @@ pub fn report_provider_error_background(provider_name: &str, context: &str, msg:
}); });
} }
pub fn requester_or_default(options: &ServerOptions, provider_name: &str, context: &str) -> Requester { pub fn requester_or_default(
options: &ServerOptions,
provider_name: &str,
context: &str,
) -> Requester {
match options.requester.clone() { match options.requester.clone() {
Some(requester) => requester, Some(requester) => requester,
None => { None => {
@@ -173,7 +262,6 @@ pub fn requester_or_default(options: &ServerOptions, provider_name: &str, contex
} }
} }
#[async_trait] #[async_trait]
pub trait Provider: Send + Sync { pub trait Provider: Send + Sync {
async fn get_videos( async fn get_videos(

34
src/providers/noodlemagazine.rs
View File

@@ -3,18 +3,18 @@ use crate::api::ClientVersion;
use crate::providers::Provider; use crate::providers::Provider;
use crate::status::*; use crate::status::*;
use crate::util::cache::VideoCache; use crate::util::cache::VideoCache;
use crate::util::requester::Requester;
use crate::util::parse_abbreviated_number; use crate::util::parse_abbreviated_number;
use crate::util::requester::Requester;
use crate::util::time::parse_time_to_seconds; use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoFormat, VideoItem}; use crate::videos::{ServerOptions, VideoFormat, VideoItem};
use async_trait::async_trait; use async_trait::async_trait;
use error_chain::error_chain; use error_chain::error_chain;
use futures::future::join_all; use futures::future::join_all;
use htmlentity::entity::{decode, ICodedDataTrait}; use htmlentity::entity::{ICodedDataTrait, decode};
use wreq::Version;
use titlecase::Titlecase;
use std::vec; use std::vec;
use titlecase::Titlecase;
use wreq::Version;
error_chain! { error_chain! {
foreign_links { foreign_links {
@@ -223,18 +223,16 @@ impl NoodlemagazineProvider {
.await .await
.ok_or_else(|| Error::from("media extraction failed"))?; .ok_or_else(|| Error::from("media extraction failed"))?;
Ok( Ok(VideoItem::new(
VideoItem::new( id,
id, title,
title, video_url,
video_url, "noodlemagazine".into(),
"noodlemagazine".into(), thumb,
thumb, duration,
duration,
)
.views(views)
.formats(formats),
) )
.views(views)
.formats(formats))
} }
async fn extract_media( async fn extract_media(
@@ -247,11 +245,7 @@ impl NoodlemagazineProvider {
.await .await
.unwrap_or_default(); .unwrap_or_default();
let json_str = text let json_str = text.split("window.playlist = ").nth(1)?.split(';').next()?;
.split("window.playlist = ")
.nth(1)?
.split(';')
.next()?;
let json: serde_json::Value = serde_json::from_str(json_str).ok()?; let json: serde_json::Value = serde_json::from_str(json_str).ok()?;
let sources = json["sources"].as_array()?; let sources = json["sources"].as_array()?;

156
src/providers/okporn.rs
View File

@@ -1,18 +1,18 @@
use crate::api::ClientVersion;
use crate::DbPool; use crate::DbPool;
use crate::api::ClientVersion;
use crate::providers::{Provider, report_provider_error}; use crate::providers::{Provider, report_provider_error};
use crate::status::*; use crate::status::*;
use crate::util::cache::VideoCache; use crate::util::cache::VideoCache;
use crate::util::flaresolverr::{FlareSolverrRequest, Flaresolverr}; use crate::util::flaresolverr::{FlareSolverrRequest, Flaresolverr};
use crate::util::time::parse_time_to_seconds; use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoItem}; use crate::videos::{ServerOptions, VideoItem};
use async_trait::async_trait;
use error_chain::error_chain; use error_chain::error_chain;
use htmlentity::entity::{ICodedDataTrait, decode}; use htmlentity::entity::{ICodedDataTrait, decode};
use std::env; use std::env;
use std::vec; use std::vec;
use wreq::{Client}; use wreq::Client;
use wreq_util::Emulation; use wreq_util::Emulation;
use async_trait::async_trait;
error_chain! { error_chain! {
foreign_links { foreign_links {
@@ -67,12 +67,7 @@ impl OkpornProvider {
cacheDuration: Some(1800), cacheDuration: Some(1800),
} }
} }
async fn get( async fn get(&self, cache: VideoCache, page: u8, sort: &str) -> Result<Vec<VideoItem>> {
&self,
cache: VideoCache,
page: u8,
sort: &str,
) -> Result<Vec<VideoItem>> {
let sort_string = match sort { let sort_string = match sort {
"trending" => "/trending", "trending" => "/trending",
"popular" => "/popular", "popular" => "/popular",
@@ -93,13 +88,22 @@ impl OkpornProvider {
}; };
// let proxy = Proxy::all("http://192.168.0.103:8081").unwrap(); // let proxy = Proxy::all("http://192.168.0.103:8081").unwrap();
let client = Client::builder().cert_verification(false).emulation(Emulation::Firefox136).build()?; let client = Client::builder()
.cert_verification(false)
.emulation(Emulation::Firefox136)
.build()?;
let mut response = client.get(video_url.clone()) let mut response = client
// .proxy(proxy.clone()) .get(video_url.clone())
.send().await?; // .proxy(proxy.clone())
.send()
.await?;
if response.status().is_redirection() { if response.status().is_redirection() {
let location = match response.headers().get("Location").and_then(|h| h.to_str().ok()) { let location = match response
.headers()
.get("Location")
.and_then(|h| h.to_str().ok())
{
Some(location) => location, Some(location) => location,
None => { None => {
report_provider_error( report_provider_error(
@@ -131,12 +135,7 @@ impl OkpornProvider {
let flare_url = match env::var("FLARE_URL") { let flare_url = match env::var("FLARE_URL") {
Ok(url) => url, Ok(url) => url,
Err(e) => { Err(e) => {
report_provider_error( report_provider_error("okporn", "get.flare_url", &e.to_string()).await;
"okporn",
"get.flare_url",
&e.to_string(),
)
.await;
return Ok(old_items); return Ok(old_items);
} }
}; };
@@ -167,12 +166,7 @@ impl OkpornProvider {
Ok(video_items) Ok(video_items)
} }
} }
async fn query( async fn query(&self, cache: VideoCache, page: u8, query: &str) -> Result<Vec<VideoItem>> {
&self,
cache: VideoCache,
page: u8,
query: &str,
) -> Result<Vec<VideoItem>> {
let search_string = query.to_lowercase().trim().replace(" ", "-"); let search_string = query.to_lowercase().trim().replace(" ", "-");
let video_url = format!("{}/search/{}/{}/", self.url, search_string, page); let video_url = format!("{}/search/{}/{}/", self.url, search_string, page);
@@ -192,14 +186,23 @@ impl OkpornProvider {
}; };
// let proxy = Proxy::all("http://192.168.0.103:8081").unwrap(); // let proxy = Proxy::all("http://192.168.0.103:8081").unwrap();
let client = Client::builder().cert_verification(false).emulation(Emulation::Firefox136).build()?; let client = Client::builder()
.cert_verification(false)
.emulation(Emulation::Firefox136)
.build()?;
let mut response = client.get(video_url.clone()) let mut response = client
// .proxy(proxy.clone()) .get(video_url.clone())
.send().await?; // .proxy(proxy.clone())
.send()
.await?;
if response.status().is_redirection() { if response.status().is_redirection() {
let location = match response.headers().get("Location").and_then(|h| h.to_str().ok()) { let location = match response
.headers()
.get("Location")
.and_then(|h| h.to_str().ok())
{
Some(location) => location, Some(location) => location,
None => { None => {
report_provider_error( report_provider_error(
@@ -232,12 +235,7 @@ impl OkpornProvider {
let flare_url = match env::var("FLARE_URL") { let flare_url = match env::var("FLARE_URL") {
Ok(url) => url, Ok(url) => url,
Err(e) => { Err(e) => {
report_provider_error( report_provider_error("okporn", "query.flare_url", &e.to_string()).await;
"okporn",
"query.flare_url",
&e.to_string(),
)
.await;
return Ok(old_items); return Ok(old_items);
} }
}; };
@@ -272,34 +270,78 @@ impl OkpornProvider {
return vec![]; return vec![];
} }
let mut items: Vec<VideoItem> = Vec::new(); let mut items: Vec<VideoItem> = Vec::new();
let raw_videos = html let raw_videos = html.split("<div class=\"item ").collect::<Vec<&str>>()[1..].to_vec();
.split("<div class=\"item ")
.collect::<Vec<&str>>()[1..]
.to_vec();
for video_segment in &raw_videos { for video_segment in &raw_videos {
// let vid = video_segment.split("\n").collect::<Vec<&str>>(); // let vid = video_segment.split("\n").collect::<Vec<&str>>();
// for (index, line) in vid.iter().enumerate() { // for (index, line) in vid.iter().enumerate() {
// println!("Line {}: {}", index, line); // println!("Line {}: {}", index, line);
// } // }
let video_url: String = format!("{}{}", self.url, video_segment.split("<a href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() let video_url: String = format!(
"{}{}",
self.url,
video_segment
.split("<a href=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"") .split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()); .collect::<Vec<&str>>()
let mut title = video_segment.split("\" title=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .get(0)
.copied()
.unwrap_or_default()
);
let mut title = video_segment
.split("\" title=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"") .split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(); .to_string();
// html decode // html decode
title = decode(title.as_bytes()).to_string().unwrap_or(title); title = decode(title.as_bytes()).to_string().unwrap_or(title);
let id = video_url.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string(); let id = video_url
let raw_duration = video_segment.split("<span class=\"duration_item\">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .split("/")
.split("<") .collect::<Vec<&str>>()
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .get(4)
.to_string(); .copied()
.unwrap_or_default()
.to_string();
let raw_duration = video_segment
.split("<span class=\"duration_item\">")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32; let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
let thumb = video_segment.split("<img class=\"thumb lazy-load\" src=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default().split("data-original=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() let thumb = video_segment
.split("<img class=\"thumb lazy-load\" src=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("data-original=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"") .split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(); .to_string();
let video_item = VideoItem::new( let video_item = VideoItem::new(
@@ -309,14 +351,11 @@ impl OkpornProvider {
"okporn".to_string(), "okporn".to_string(),
thumb, thumb,
duration, duration,
) );
;
items.push(video_item); items.push(video_item);
} }
return items; return items;
} }
} }
#[async_trait] #[async_trait]
@@ -335,10 +374,7 @@ impl Provider for OkpornProvider {
let _ = per_page; let _ = per_page;
let _ = pool; let _ = pool;
let videos: std::result::Result<Vec<VideoItem>, Error> = match query { let videos: std::result::Result<Vec<VideoItem>, Error> = match query {
Some(q) => { Some(q) => self.query(cache, page.parse::<u8>().unwrap_or(1), &q).await,
self.query(cache, page.parse::<u8>().unwrap_or(1), &q,)
.await
}
None => { None => {
self.get(cache, page.parse::<u8>().unwrap_or(1), &sort) self.get(cache, page.parse::<u8>().unwrap_or(1), &sort)
.await .await

242
src/providers/okxxx.rs
View File

@@ -1,19 +1,19 @@
use crate::api::ClientVersion;
use crate::util::parse_abbreviated_number;
use crate::DbPool; use crate::DbPool;
use crate::api::ClientVersion;
use crate::providers::{Provider, report_provider_error}; use crate::providers::{Provider, report_provider_error};
use crate::status::*; use crate::status::*;
use crate::util::cache::VideoCache; use crate::util::cache::VideoCache;
use crate::util::flaresolverr::{FlareSolverrRequest, Flaresolverr}; use crate::util::flaresolverr::{FlareSolverrRequest, Flaresolverr};
use crate::util::parse_abbreviated_number;
use crate::util::time::parse_time_to_seconds; use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoItem}; use crate::videos::{ServerOptions, VideoItem};
use async_trait::async_trait;
use error_chain::error_chain; use error_chain::error_chain;
use htmlentity::entity::{ICodedDataTrait, decode}; use htmlentity::entity::{ICodedDataTrait, decode};
use std::env; use std::env;
use std::vec; use std::vec;
use wreq::{Client}; use wreq::Client;
use wreq_util::Emulation; use wreq_util::Emulation;
use async_trait::async_trait;
error_chain! { error_chain! {
foreign_links { foreign_links {
@@ -68,12 +68,7 @@ impl OkxxxProvider {
cacheDuration: Some(1800), cacheDuration: Some(1800),
} }
} }
async fn get( async fn get(&self, cache: VideoCache, page: u8, sort: &str) -> Result<Vec<VideoItem>> {
&self,
cache: VideoCache,
page: u8,
sort: &str,
) -> Result<Vec<VideoItem>> {
let sort_string = match sort { let sort_string = match sort {
"trending" => "/trending", "trending" => "/trending",
"popular" => "/popular", "popular" => "/popular",
@@ -94,13 +89,22 @@ impl OkxxxProvider {
}; };
// let proxy = Proxy::all("http://192.168.0.103:8081").unwrap(); // let proxy = Proxy::all("http://192.168.0.103:8081").unwrap();
let client = Client::builder().cert_verification(false).emulation(Emulation::Firefox136).build()?; let client = Client::builder()
.cert_verification(false)
.emulation(Emulation::Firefox136)
.build()?;
let mut response = client.get(video_url.clone()) let mut response = client
// .proxy(proxy.clone()) .get(video_url.clone())
.send().await?; // .proxy(proxy.clone())
.send()
.await?;
if response.status().is_redirection() { if response.status().is_redirection() {
let location = match response.headers().get("Location").and_then(|h| h.to_str().ok()) { let location = match response
.headers()
.get("Location")
.and_then(|h| h.to_str().ok())
{
Some(location) => location, Some(location) => location,
None => { None => {
report_provider_error( report_provider_error(
@@ -133,12 +137,7 @@ impl OkxxxProvider {
let flare_url = match env::var("FLARE_URL") { let flare_url = match env::var("FLARE_URL") {
Ok(url) => url, Ok(url) => url,
Err(e) => { Err(e) => {
report_provider_error( report_provider_error("okxxx", "get.flare_url", &e.to_string()).await;
"okxxx",
"get.flare_url",
&e.to_string(),
)
.await;
return Ok(old_items); return Ok(old_items);
} }
}; };
@@ -169,17 +168,18 @@ impl OkxxxProvider {
Ok(video_items) Ok(video_items)
} }
} }
async fn query( async fn query(&self, cache: VideoCache, page: u8, query: &str) -> Result<Vec<VideoItem>> {
&self,
cache: VideoCache,
page: u8,
query: &str,
) -> Result<Vec<VideoItem>> {
let search_string = query.to_lowercase().trim().replace(" ", "-"); let search_string = query.to_lowercase().trim().replace(" ", "-");
let mut video_url = format!("{}/search/{}/{}/", self.url, search_string, page); let mut video_url = format!("{}/search/{}/{}/", self.url, search_string, page);
if search_string.starts_with("@"){ if search_string.starts_with("@") {
let url_part = search_string.split("@").collect::<Vec<&str>>().get(1).copied().unwrap_or_default().replace(":", "/"); let url_part = search_string
.split("@")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.replace(":", "/");
video_url = format!("{}/{}/", self.url, url_part); video_url = format!("{}/{}/", self.url, url_part);
} }
// Check our Video Cache. If the result is younger than 1 hour, we return it. // Check our Video Cache. If the result is younger than 1 hour, we return it.
@@ -198,14 +198,23 @@ impl OkxxxProvider {
}; };
// let proxy = Proxy::all("http://192.168.0.103:8081").unwrap(); // let proxy = Proxy::all("http://192.168.0.103:8081").unwrap();
let client = Client::builder().cert_verification(false).emulation(Emulation::Firefox136).build()?; let client = Client::builder()
.cert_verification(false)
.emulation(Emulation::Firefox136)
.build()?;
let mut response = client.get(video_url.clone()) let mut response = client
// .proxy(proxy.clone()) .get(video_url.clone())
.send().await?; // .proxy(proxy.clone())
.send()
.await?;
if response.status().is_redirection() { if response.status().is_redirection() {
let location = match response.headers().get("Location").and_then(|h| h.to_str().ok()) { let location = match response
.headers()
.get("Location")
.and_then(|h| h.to_str().ok())
{
Some(location) => location, Some(location) => location,
None => { None => {
report_provider_error( report_provider_error(
@@ -238,12 +247,7 @@ impl OkxxxProvider {
let flare_url = match env::var("FLARE_URL") { let flare_url = match env::var("FLARE_URL") {
Ok(url) => url, Ok(url) => url,
Err(e) => { Err(e) => {
report_provider_error( report_provider_error("okxxx", "query.flare_url", &e.to_string()).await;
"okxxx",
"query.flare_url",
&e.to_string(),
)
.await;
return Ok(old_items); return Ok(old_items);
} }
}; };
@@ -278,7 +282,12 @@ impl OkxxxProvider {
return vec![]; return vec![];
} }
let mut items: Vec<VideoItem> = Vec::new(); let mut items: Vec<VideoItem> = Vec::new();
let raw_videos = html.split("<div class=\"pagination\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default() let raw_videos = html
.split("<div class=\"pagination\"")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.split("item thumb-bl thumb-bl-video video_") .split("item thumb-bl thumb-bl-video video_")
.collect::<Vec<&str>>()[1..] .collect::<Vec<&str>>()[1..]
.to_vec(); .to_vec();
@@ -287,61 +296,150 @@ impl OkxxxProvider {
// for (index, line) in vid.iter().enumerate() { // for (index, line) in vid.iter().enumerate() {
// println!("Line {}: {}", index, line); // println!("Line {}: {}", index, line);
// } // }
let video_url: String = format!("{}{}", self.url, video_segment.split("<a href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() let video_url: String = format!(
"{}{}",
self.url,
video_segment
.split("<a href=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"") .split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()); .collect::<Vec<&str>>()
let preview_url = video_segment.split("data-preview-custom=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .get(0)
.copied()
.unwrap_or_default()
);
let preview_url = video_segment
.split("data-preview-custom=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"") .split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(); .to_string();
let mut title = video_segment.split("\" title=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() let mut title = video_segment
.split("\" title=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"") .split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(); .to_string();
// html decode // html decode
title = decode(title.as_bytes()).to_string().unwrap_or(title); title = decode(title.as_bytes()).to_string().unwrap_or(title);
let id = video_url.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string(); let id = video_url
let raw_duration = video_segment.split("fa fa-clock-o").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .split("/")
.split("<span>").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(4)
.copied()
.unwrap_or_default()
.to_string();
let raw_duration = video_segment
.split("fa fa-clock-o")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<span>")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<") .split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(); .to_string();
let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32; let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
let thumb = format!("https:{}", video_segment.split(" class=\"thumb lazy-load\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() let thumb = format!(
.split("data-original=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() "https:{}",
.split("\"") video_segment
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .split(" class=\"thumb lazy-load\"")
.to_string()); .collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("data-original=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string()
);
let mut tags = vec![]; let mut tags = vec![];
if video_segment.contains("href=\"/sites/"){ if video_segment.contains("href=\"/sites/") {
let raw_tags = video_segment.split("href=\"/sites/").collect::<Vec<&str>>()[1..] let raw_tags = video_segment.split("href=\"/sites/").collect::<Vec<&str>>()[1..]
.iter() .iter()
.map(|s| s.split("/\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default().to_string()) .map(|s| {
s.split("/\"")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string()
})
.collect::<Vec<String>>(); .collect::<Vec<String>>();
for tag in raw_tags { for tag in raw_tags {
if !tag.is_empty() { if !tag.is_empty() {
tags.push(format!("@sites:{}",tag)); tags.push(format!("@sites:{}", tag));
} }
} }
} }
if video_segment.contains("href=\"/models/"){ if video_segment.contains("href=\"/models/") {
let raw_tags = video_segment.split("href=\"/models/").collect::<Vec<&str>>()[1..] let raw_tags = video_segment
.split("href=\"/models/")
.collect::<Vec<&str>>()[1..]
.iter() .iter()
.map(|s| s.split("/\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default().to_string()) .map(|s| {
s.split("/\"")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string()
})
.collect::<Vec<String>>(); .collect::<Vec<String>>();
for tag in raw_tags { for tag in raw_tags {
if !tag.is_empty() { if !tag.is_empty() {
tags.push(format!("@models:{}",tag)); tags.push(format!("@models:{}", tag));
} }
} }
} }
let views_part = video_segment.split("fa fa-eye").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() let views_part = video_segment
.split("<span>").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .split("fa fa-eye")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<span>")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<") .split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(); .to_string();
let views = parse_abbreviated_number(&views_part).unwrap_or(0) as u32; let views = parse_abbreviated_number(&views_part).unwrap_or(0) as u32;
@@ -355,14 +453,11 @@ impl OkxxxProvider {
) )
.preview(preview_url) .preview(preview_url)
.views(views) .views(views)
.tags(tags) .tags(tags);
;
items.push(video_item); items.push(video_item);
} }
return items; return items;
} }
} }
#[async_trait] #[async_trait]
@@ -381,10 +476,7 @@ impl Provider for OkxxxProvider {
let _ = per_page; let _ = per_page;
let _ = pool; let _ = pool;
let videos: std::result::Result<Vec<VideoItem>, Error> = match query { let videos: std::result::Result<Vec<VideoItem>, Error> = match query {
Some(q) => { Some(q) => self.query(cache, page.parse::<u8>().unwrap_or(1), &q).await,
self.query(cache, page.parse::<u8>().unwrap_or(1), &q,)
.await
}
None => { None => {
self.get(cache, page.parse::<u8>().unwrap_or(1), &sort) self.get(cache, page.parse::<u8>().unwrap_or(1), &sort)
.await .await

294
src/providers/omgxxx.rs
View File

@@ -120,17 +120,40 @@ impl OmgxxxProvider {
.copied() .copied()
.unwrap_or_default() .unwrap_or_default()
.split("custom_list_models_models_list_pagination") .split("custom_list_models_models_list_pagination")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default(); .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default();
for stars_element in stars_div.split("<a ").collect::<Vec<&str>>()[1..].to_vec() { for stars_element in stars_div.split("<a ").collect::<Vec<&str>>()[1..].to_vec() {
let star_url = stars_element.split("href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() let star_url = stars_element
.split("href=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"") .split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default(); .collect::<Vec<&str>>()
let star_id = star_url.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string(); .get(0)
.copied()
.unwrap_or_default();
let star_id = star_url
.split("/")
.collect::<Vec<&str>>()
.get(4)
.copied()
.unwrap_or_default()
.to_string();
let star_name = stars_element let star_name = stars_element
.split("<strong class=\"title\">") .split("<strong class=\"title\">")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<") .split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(); .to_string();
Self::push_unique( Self::push_unique(
&stars, &stars,
@@ -168,19 +191,47 @@ impl OmgxxxProvider {
} }
let sites_div = text let sites_div = text
.split("id=\"list_content_sources_sponsors_list_items\"") .split("id=\"list_content_sources_sponsors_list_items\"")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("class=\"pagination\"") .split("class=\"pagination\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default(); .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default();
for sites_element in for sites_element in
sites_div.split("class=\"headline\"").collect::<Vec<&str>>()[1..].to_vec() sites_div.split("class=\"headline\"").collect::<Vec<&str>>()[1..].to_vec()
{ {
let site_url = sites_element.split("href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() let site_url = sites_element
.split("href=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"") .split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default(); .collect::<Vec<&str>>()
let site_id = site_url.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string(); .get(0)
let site_name = sites_element.split("<h2>").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .copied()
.unwrap_or_default();
let site_id = site_url
.split("/")
.collect::<Vec<&str>>()
.get(4)
.copied()
.unwrap_or_default()
.to_string();
let site_name = sites_element
.split("<h2>")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<") .split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(); .to_string();
Self::push_unique( Self::push_unique(
&sites, &sites,
@@ -207,22 +258,52 @@ impl OmgxxxProvider {
return Ok(()); return Ok(());
} }
}; };
let networks_div = text.split("class=\"sites__list\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() let networks_div = text
.split("class=\"sites__list\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("</div>") .split("</div>")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default(); .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default();
for network_element in for network_element in
networks_div.split("sites__item").collect::<Vec<&str>>()[1..].to_vec() networks_div.split("sites__item").collect::<Vec<&str>>()[1..].to_vec()
{ {
if network_element.contains("sites__all") { if network_element.contains("sites__all") {
continue; continue;
} }
let network_url = network_element.split("href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() let network_url = network_element
.split("href=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"") .split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default(); .collect::<Vec<&str>>()
let network_id = network_url.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string(); .get(0)
let network_name = network_element.split(">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .copied()
.unwrap_or_default();
let network_id = network_url
.split("/")
.collect::<Vec<&str>>()
.get(4)
.copied()
.unwrap_or_default()
.to_string();
let network_name = network_element
.split(">")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<") .split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(); .to_string();
Self::push_unique( Self::push_unique(
&networks, &networks,
@@ -420,11 +501,7 @@ impl OmgxxxProvider {
} }
} }
Err(e) => { Err(e) => {
report_provider_error_background( report_provider_error_background("omgxxx", "query.stars_read", &e.to_string());
"omgxxx",
"query.stars_read",
&e.to_string(),
);
} }
} }
match self.sites.read() { match self.sites.read() {
@@ -438,11 +515,7 @@ impl OmgxxxProvider {
} }
} }
Err(e) => { Err(e) => {
report_provider_error_background( report_provider_error_background("omgxxx", "query.sites_read", &e.to_string());
"omgxxx",
"query.sites_read",
&e.to_string(),
);
} }
} }
let mut video_url = format!("{}/{}/{}/{}/", self.url, search_type, search_string, page); let mut video_url = format!("{}/{}/{}/{}/", self.url, search_type, search_string, page);
@@ -522,11 +595,22 @@ impl OmgxxxProvider {
if !html.contains("class=\"item\"") { if !html.contains("class=\"item\"") {
return items; return items;
} }
let raw_videos = html.split("videos_list_pagination").collect::<Vec<&str>>().get(0).copied().unwrap_or_default() let raw_videos = html
.split("videos_list_pagination")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.split(" class=\"pagination\" ") .split(" class=\"pagination\" ")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.split("class=\"list-videos\"") .split("class=\"list-videos\"")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("class=\"item\"") .split("class=\"item\"")
.collect::<Vec<&str>>()[1..] .collect::<Vec<&str>>()[1..]
.to_vec(); .to_vec();
@@ -535,39 +619,94 @@ impl OmgxxxProvider {
// for (index, line) in vid.iter().enumerate() { // for (index, line) in vid.iter().enumerate() {
// println!("Line {}: {}", index, line); // println!("Line {}: {}", index, line);
// } // }
let video_url: String = video_segment.split("<a href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() let video_url: String = video_segment
.split("<a href=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"") .split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(); .to_string();
let mut title = video_segment.split(" title=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() let mut title = video_segment
.split(" title=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"") .split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(); .to_string();
// html decode // html decode
title = decode(title.as_bytes()).to_string().unwrap_or(title); title = decode(title.as_bytes()).to_string().unwrap_or(title);
let id = video_url.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string(); let id = video_url
.split("/")
.collect::<Vec<&str>>()
.get(4)
.copied()
.unwrap_or_default()
.to_string();
let thumb = match video_segment.split("img loading").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() let thumb = match video_segment
.split("img loading")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.contains("data-src=\"") .contains("data-src=\"")
{ {
true => video_segment.split("img loading").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() true => video_segment
.split("img loading")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("data-src=\"") .split("data-src=\"")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"") .split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(), .to_string(),
false => video_segment.split("img loading").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() false => video_segment
.split("img loading")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("data-original=\"") .split("data-original=\"")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"") .split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(), .to_string(),
}; };
let raw_duration = video_segment let raw_duration = video_segment
.split("<span class=\"duration\">") .split("<span class=\"duration\">")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<") .split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.split(" ") .split(" ")
.collect::<Vec<&str>>() .collect::<Vec<&str>>()
.last() .last()
@@ -577,9 +716,15 @@ impl OmgxxxProvider {
let views = parse_abbreviated_number( let views = parse_abbreviated_number(
video_segment video_segment
.split("<div class=\"views\">") .split("<div class=\"views\">")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<") .split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string() .to_string()
.as_str(), .as_str(),
) )
@@ -587,9 +732,15 @@ impl OmgxxxProvider {
let preview = video_segment let preview = video_segment
.split("data-preview=\"") .split("data-preview=\"")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"") .split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(); .to_string();
let site_name = title let site_name = title
.split("]") .split("]")
@@ -603,9 +754,15 @@ impl OmgxxxProvider {
let mut tags = match video_segment.contains("class=\"models\">") { let mut tags = match video_segment.contains("class=\"models\">") {
true => video_segment true => video_segment
.split("class=\"models\">") .split("class=\"models\">")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("</div>") .split("</div>")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.split("href=\"") .split("href=\"")
.collect::<Vec<&str>>()[1..] .collect::<Vec<&str>>()[1..]
.into_iter() .into_iter()
@@ -613,17 +770,38 @@ impl OmgxxxProvider {
Self::push_unique( Self::push_unique(
&self.stars, &self.stars,
FilterOption { FilterOption {
id: s.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string(), id: s
title: s.split(">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .split("/")
.collect::<Vec<&str>>()
.get(4)
.copied()
.unwrap_or_default()
.to_string(),
title: s
.split(">")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<") .split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.trim() .trim()
.to_string(), .to_string(),
}, },
); );
s.split(">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() s.split(">")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<") .split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.trim() .trim()
.to_string() .to_string()
}) })

22
src/providers/paradisehill.rs
View File

@@ -1,14 +1,14 @@
use crate::api::ClientVersion;
use crate::DbPool; use crate::DbPool;
use crate::api::ClientVersion;
use crate::providers::Provider; use crate::providers::Provider;
use crate::status::*; use crate::status::*;
use crate::util::cache::VideoCache; use crate::util::cache::VideoCache;
use crate::util::requester::Requester; use crate::util::requester::Requester;
use crate::videos::VideoItem;
use crate::videos::ServerOptions; use crate::videos::ServerOptions;
use crate::videos::VideoItem;
use async_trait::async_trait;
use error_chain::error_chain; use error_chain::error_chain;
use htmlentity::entity::{ICodedDataTrait, decode}; use htmlentity::entity::{ICodedDataTrait, decode};
use async_trait::async_trait;
error_chain! { error_chain! {
foreign_links { foreign_links {
@@ -35,7 +35,8 @@ impl ParadisehillProvider {
name: "Paradisehill".to_string(), name: "Paradisehill".to_string(),
description: "Porn Movies on Paradise Hill".to_string(), description: "Porn Movies on Paradise Hill".to_string(),
premium: false, premium: false,
favicon: "https://www.google.com/s2/favicons?sz=64&domain=en.paradisehill.cc".to_string(), favicon: "https://www.google.com/s2/favicons?sz=64&domain=en.paradisehill.cc"
.to_string(),
status: "active".to_string(), status: "active".to_string(),
categories: vec![], categories: vec![],
options: vec![], options: vec![],
@@ -49,7 +50,8 @@ impl ParadisehillProvider {
page: u8, page: u8,
options: ServerOptions, options: ServerOptions,
) -> Result<Vec<VideoItem>> { ) -> Result<Vec<VideoItem>> {
let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let url_str = format!("{}/all/?sort=created_at&page={}", self.url, page); let url_str = format!("{}/all/?sort=created_at&page={}", self.url, page);
@@ -99,7 +101,8 @@ impl ParadisehillProvider {
options: ServerOptions, options: ServerOptions,
) -> Result<Vec<VideoItem>> { ) -> Result<Vec<VideoItem>> {
// Extract needed fields from options at the start // Extract needed fields from options at the start
let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let search_string = query.replace(" ", "+"); let search_string = query.replace(" ", "+");
let url_str = format!( let url_str = format!(
"{}/search/?pattern={}&page={}", "{}/search/?pattern={}&page={}",
@@ -200,7 +203,11 @@ impl ParadisehillProvider {
.unwrap_or_default() .unwrap_or_default()
.trim() .trim()
.to_string(); .to_string();
let tags = if genre.is_empty() { vec![] } else { vec![genre] }; let tags = if genre.is_empty() {
vec![]
} else {
vec![genre]
};
items.push( items.push(
VideoItem::new(id, title, video_url, "paradisehill".to_string(), thumb, 0) VideoItem::new(id, title, video_url, "paradisehill".to_string(), thumb, 0)
@@ -211,7 +218,6 @@ impl ParadisehillProvider {
items items
} }
} }
#[async_trait] #[async_trait]

243
src/providers/perfectgirls.rs
View File

@@ -1,19 +1,19 @@
use crate::api::ClientVersion;
use crate::util::parse_abbreviated_number;
use crate::DbPool; use crate::DbPool;
use crate::api::ClientVersion;
use crate::providers::{Provider, report_provider_error}; use crate::providers::{Provider, report_provider_error};
use crate::status::*; use crate::status::*;
use crate::util::cache::VideoCache; use crate::util::cache::VideoCache;
use crate::util::flaresolverr::{FlareSolverrRequest, Flaresolverr}; use crate::util::flaresolverr::{FlareSolverrRequest, Flaresolverr};
use crate::util::parse_abbreviated_number;
use crate::util::time::parse_time_to_seconds; use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoItem}; use crate::videos::{ServerOptions, VideoItem};
use async_trait::async_trait;
use error_chain::error_chain; use error_chain::error_chain;
use htmlentity::entity::{ICodedDataTrait, decode}; use htmlentity::entity::{ICodedDataTrait, decode};
use std::env; use std::env;
use std::vec; use std::vec;
use wreq::{Client}; use wreq::Client;
use wreq_util::Emulation; use wreq_util::Emulation;
use async_trait::async_trait;
error_chain! { error_chain! {
foreign_links { foreign_links {
@@ -68,12 +68,7 @@ impl PerfectgirlsProvider {
cacheDuration: Some(1800), cacheDuration: Some(1800),
} }
} }
async fn get( async fn get(&self, cache: VideoCache, page: u8, sort: &str) -> Result<Vec<VideoItem>> {
&self,
cache: VideoCache,
page: u8,
sort: &str,
) -> Result<Vec<VideoItem>> {
let sort_string = match sort { let sort_string = match sort {
"trending" => "/trending", "trending" => "/trending",
"popular" => "/popular", "popular" => "/popular",
@@ -94,13 +89,22 @@ impl PerfectgirlsProvider {
}; };
// let proxy = Proxy::all("http://192.168.0.103:8081").unwrap(); // let proxy = Proxy::all("http://192.168.0.103:8081").unwrap();
let client = Client::builder().cert_verification(false).emulation(Emulation::Firefox136).build()?; let client = Client::builder()
.cert_verification(false)
.emulation(Emulation::Firefox136)
.build()?;
let mut response = client.get(video_url.clone()) let mut response = client
// .proxy(proxy.clone()) .get(video_url.clone())
.send().await?; // .proxy(proxy.clone())
.send()
.await?;
if response.status().is_redirection() { if response.status().is_redirection() {
let location = match response.headers().get("Location").and_then(|h| h.to_str().ok()) { let location = match response
.headers()
.get("Location")
.and_then(|h| h.to_str().ok())
{
Some(location) => location, Some(location) => location,
None => { None => {
report_provider_error( report_provider_error(
@@ -133,12 +137,7 @@ impl PerfectgirlsProvider {
let flare_url = match env::var("FLARE_URL") { let flare_url = match env::var("FLARE_URL") {
Ok(url) => url, Ok(url) => url,
Err(e) => { Err(e) => {
report_provider_error( report_provider_error("perfectgirls", "get.flare_url", &e.to_string()).await;
"perfectgirls",
"get.flare_url",
&e.to_string(),
)
.await;
return Ok(old_items); return Ok(old_items);
} }
}; };
@@ -169,17 +168,18 @@ impl PerfectgirlsProvider {
Ok(video_items) Ok(video_items)
} }
} }
async fn query( async fn query(&self, cache: VideoCache, page: u8, query: &str) -> Result<Vec<VideoItem>> {
&self,
cache: VideoCache,
page: u8,
query: &str,
) -> Result<Vec<VideoItem>> {
let search_string = query.to_lowercase().trim().replace(" ", "-"); let search_string = query.to_lowercase().trim().replace(" ", "-");
let mut video_url = format!("{}/search/{}/{}/", self.url, search_string, page); let mut video_url = format!("{}/search/{}/{}/", self.url, search_string, page);
if search_string.starts_with("@"){ if search_string.starts_with("@") {
let url_part = search_string.split("@").collect::<Vec<&str>>().get(1).copied().unwrap_or_default().replace(":", "/"); let url_part = search_string
.split("@")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.replace(":", "/");
video_url = format!("{}/{}/", self.url, url_part); video_url = format!("{}/{}/", self.url, url_part);
} }
// Check our Video Cache. If the result is younger than 1 hour, we return it. // Check our Video Cache. If the result is younger than 1 hour, we return it.
@@ -198,14 +198,23 @@ impl PerfectgirlsProvider {
}; };
// let proxy = Proxy::all("http://192.168.0.103:8081").unwrap(); // let proxy = Proxy::all("http://192.168.0.103:8081").unwrap();
let client = Client::builder().cert_verification(false).emulation(Emulation::Firefox136).build()?; let client = Client::builder()
.cert_verification(false)
.emulation(Emulation::Firefox136)
.build()?;
let mut response = client.get(video_url.clone()) let mut response = client
// .proxy(proxy.clone()) .get(video_url.clone())
.send().await?; // .proxy(proxy.clone())
.send()
.await?;
if response.status().is_redirection() { if response.status().is_redirection() {
let location = match response.headers().get("Location").and_then(|h| h.to_str().ok()) { let location = match response
.headers()
.get("Location")
.and_then(|h| h.to_str().ok())
{
Some(location) => location, Some(location) => location,
None => { None => {
report_provider_error( report_provider_error(
@@ -238,12 +247,7 @@ impl PerfectgirlsProvider {
let flare_url = match env::var("FLARE_URL") { let flare_url = match env::var("FLARE_URL") {
Ok(url) => url, Ok(url) => url,
Err(e) => { Err(e) => {
report_provider_error( report_provider_error("perfectgirls", "query.flare_url", &e.to_string()).await;
"perfectgirls",
"query.flare_url",
&e.to_string(),
)
.await;
return Ok(old_items); return Ok(old_items);
} }
}; };
@@ -278,7 +282,12 @@ impl PerfectgirlsProvider {
return vec![]; return vec![];
} }
let mut items: Vec<VideoItem> = Vec::new(); let mut items: Vec<VideoItem> = Vec::new();
let raw_videos = html.split("<div class=\"pagination\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default() let raw_videos = html
.split("<div class=\"pagination\"")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.split("item thumb-bl thumb-bl-video video_") .split("item thumb-bl thumb-bl-video video_")
.collect::<Vec<&str>>()[1..] .collect::<Vec<&str>>()[1..]
.to_vec(); .to_vec();
@@ -287,64 +296,152 @@ impl PerfectgirlsProvider {
// for (index, line) in vid.iter().enumerate() { // for (index, line) in vid.iter().enumerate() {
// println!("Line {}: {}", index, line); // println!("Line {}: {}", index, line);
// } // }
let video_url: String = format!("{}{}", self.url, video_segment.split("<a href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() let video_url: String = format!(
"{}{}",
self.url,
video_segment
.split("<a href=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"") .split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()); .collect::<Vec<&str>>()
let preview_url = video_segment.split("data-preview-custom=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .get(0)
.copied()
.unwrap_or_default()
);
let preview_url = video_segment
.split("data-preview-custom=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"") .split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(); .to_string();
let mut title = video_segment.split("\" title=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() let mut title = video_segment
.split("\" title=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"") .split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(); .to_string();
// html decode // html decode
title = decode(title.as_bytes()).to_string().unwrap_or(title); title = decode(title.as_bytes()).to_string().unwrap_or(title);
let id = video_url.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string(); let id = video_url
let raw_duration = video_segment.split("fa fa-clock-o").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .split("/")
.split("<span>").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(4)
.copied()
.unwrap_or_default()
.to_string();
let raw_duration = video_segment
.split("fa fa-clock-o")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<span>")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<") .split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(); .to_string();
let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32; let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
let mut thumb = video_segment.split(" class=\"thumb lazy-load\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() let mut thumb = video_segment
.split("data-original=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .split(" class=\"thumb lazy-load\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("data-original=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"") .split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(); .to_string();
if thumb.starts_with("//"){ if thumb.starts_with("//") {
thumb = format!("https:{}",thumb); thumb = format!("https:{}", thumb);
} }
let mut tags = vec![]; let mut tags = vec![];
if video_segment.contains("href=\"/channels/"){ if video_segment.contains("href=\"/channels/") {
let raw_tags = video_segment.split("href=\"/channels/").collect::<Vec<&str>>()[1..] let raw_tags = video_segment
.split("href=\"/channels/")
.collect::<Vec<&str>>()[1..]
.iter() .iter()
.map(|s| s.split("/\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default().to_string()) .map(|s| {
s.split("/\"")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string()
})
.collect::<Vec<String>>(); .collect::<Vec<String>>();
for tag in raw_tags { for tag in raw_tags {
if !tag.is_empty() { if !tag.is_empty() {
tags.push(format!("@channels:{}",tag)); tags.push(format!("@channels:{}", tag));
} }
} }
} }
if video_segment.contains("href=\"/pornstars/"){ if video_segment.contains("href=\"/pornstars/") {
let raw_tags = video_segment.split("href=\"/pornstars/").collect::<Vec<&str>>()[1..] let raw_tags = video_segment
.split("href=\"/pornstars/")
.collect::<Vec<&str>>()[1..]
.iter() .iter()
.map(|s| s.split("/\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default().to_string()) .map(|s| {
s.split("/\"")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string()
})
.collect::<Vec<String>>(); .collect::<Vec<String>>();
for tag in raw_tags { for tag in raw_tags {
if !tag.is_empty() { if !tag.is_empty() {
tags.push(format!("@pornstars:{}",tag)); tags.push(format!("@pornstars:{}", tag));
} }
} }
} }
let views_part = video_segment.split("fa fa-eye").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() let views_part = video_segment
.split("<span>").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .split("fa fa-eye")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<span>")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<") .split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(); .to_string();
let views = parse_abbreviated_number(&views_part).unwrap_or(0) as u32; let views = parse_abbreviated_number(&views_part).unwrap_or(0) as u32;
@@ -358,14 +455,11 @@ impl PerfectgirlsProvider {
) )
.preview(preview_url) .preview(preview_url)
.views(views) .views(views)
.tags(tags) .tags(tags);
;
items.push(video_item); items.push(video_item);
} }
return items; return items;
} }
} }
#[async_trait] #[async_trait]
@@ -384,10 +478,7 @@ impl Provider for PerfectgirlsProvider {
let _ = per_page; let _ = per_page;
let _ = pool; let _ = pool;
let videos: std::result::Result<Vec<VideoItem>, Error> = match query { let videos: std::result::Result<Vec<VideoItem>, Error> = match query {
Some(q) => { Some(q) => self.query(cache, page.parse::<u8>().unwrap_or(1), &q).await,
self.query(cache, page.parse::<u8>().unwrap_or(1), &q,)
.await
}
None => { None => {
self.get(cache, page.parse::<u8>().unwrap_or(1), &sort) self.get(cache, page.parse::<u8>().unwrap_or(1), &sort)
.await .await

398
src/providers/perverzija.rs
View File

@@ -6,16 +6,15 @@ use crate::status::*;
use crate::util::cache::VideoCache; use crate::util::cache::VideoCache;
use crate::util::time::parse_time_to_seconds; use crate::util::time::parse_time_to_seconds;
use crate::videos::ServerOptions; use crate::videos::ServerOptions;
use crate::videos::{self, VideoEmbed, VideoItem}; use crate::videos::{self, VideoItem};
use async_trait::async_trait; use async_trait::async_trait;
use error_chain::error_chain; use error_chain::error_chain;
use futures::future::join_all; use futures::future::join_all;
use htmlentity::entity::{ICodedDataTrait, decode}; use htmlentity::entity::{ICodedDataTrait, decode};
use serde::Deserialize; use serde::Deserialize;
use serde::Serialize; use serde::Serialize;
use wreq::Version;
use std::vec;
use wreq::Client; use wreq::Client;
use wreq::Version;
use wreq_util::Emulation; use wreq_util::Emulation;
error_chain! { error_chain! {
@@ -78,6 +77,66 @@ impl PerverzijaProvider {
} }
} }
fn extract_between<'a>(haystack: &'a str, start: &str, end: &str) -> Option<&'a str> {
let rest = haystack.split(start).nth(1)?;
Some(rest.split(end).next().unwrap_or_default())
}
fn extract_iframe_src(haystack: &str) -> String {
Self::extract_between(haystack, "iframe src=\"", "\"")
.or_else(|| Self::extract_between(haystack, "iframe src=&quot;", "&quot;"))
.unwrap_or_default()
.to_string()
}
fn extract_thumb(haystack: &str) -> String {
let img_segment = haystack.split("<img").nth(1).unwrap_or_default();
let mut thumb = Self::extract_between(img_segment, "data-original=\"", "\"")
.or_else(|| Self::extract_between(img_segment, "data-src=\"", "\""))
.or_else(|| Self::extract_between(img_segment, "src=\"", "\""))
.unwrap_or_default()
.to_string();
if thumb.starts_with("data:image") {
thumb.clear();
} else if thumb.starts_with("//") {
thumb = format!("https:{thumb}");
}
thumb
}
fn extract_title(haystack: &str) -> String {
let mut title = Self::extract_between(haystack, "<h4 class='gv-title'>", "</h4>")
.or_else(|| Self::extract_between(haystack, "<h4 class=\"gv-title\">", "</h4>"))
.or_else(|| Self::extract_between(haystack, " title='", "'"))
.or_else(|| Self::extract_between(haystack, " title=\"", "\""))
.unwrap_or_default()
.to_string();
title = decode(title.as_bytes()).to_string().unwrap_or(title);
if title.contains('<') && title.contains('>') {
let mut plain = String::new();
let mut in_tag = false;
for c in title.chars() {
match c {
'<' => in_tag = true,
'>' => in_tag = false,
_ if !in_tag => plain.push(c),
_ => {}
}
}
let normalized = plain.split_whitespace().collect::<Vec<&str>>().join(" ");
if !normalized.is_empty() {
title = normalized;
}
} else {
title = title.split_whitespace().collect::<Vec<&str>>().join(" ");
}
title.trim().to_string()
}
async fn get( async fn get(
&self, &self,
cache: VideoCache, cache: VideoCache,
@@ -204,93 +263,91 @@ impl PerverzijaProvider {
fn get_video_items_from_html(&self, html: String, pool: DbPool) -> Vec<VideoItem> { fn get_video_items_from_html(&self, html: String, pool: DbPool) -> Vec<VideoItem> {
if html.is_empty() { if html.is_empty() {
println!("HTML is empty"); report_provider_error_background(
"perverzija",
"get_video_items_from_html.empty_html",
"empty html response",
);
return vec![]; return vec![];
} }
let mut items: Vec<VideoItem> = Vec::new(); let mut items: Vec<VideoItem> = Vec::new();
let video_listing_content = html.split("video-listing-content").collect::<Vec<&str>>().get(1).copied().unwrap_or_default(); let video_listing_content = html.split("video-listing-content").nth(1).unwrap_or(&html);
let raw_videos = video_listing_content let raw_videos: Vec<&str> = video_listing_content
.split("video-item post") .split("video-item post")
.collect::<Vec<&str>>()[1..] .skip(1)
.to_vec(); .collect();
for video_segment in &raw_videos {
let vid = video_segment.split("\n").collect::<Vec<&str>>(); if raw_videos.is_empty() {
if vid.len() > 20 || vid.len() < 8 { report_provider_error_background(
"perverzija",
"get_video_items_from_html.no_segments",
&format!("html_len={}", html.len()),
);
return vec![];
}
for video_segment in raw_videos {
let title = Self::extract_title(video_segment);
let embed_html_raw = Self::extract_between(video_segment, "data-embed='", "'")
.or_else(|| Self::extract_between(video_segment, "data-embed=\"", "\""))
.unwrap_or_default()
.to_string();
let embed_html = decode(embed_html_raw.as_bytes())
.to_string()
.unwrap_or(embed_html_raw.clone());
let mut url_str = Self::extract_iframe_src(&embed_html);
if url_str.is_empty() {
url_str = Self::extract_iframe_src(video_segment);
}
if url_str.is_empty() {
report_provider_error_background( report_provider_error_background(
"perverzija", "perverzija",
"get_video_items_from_html.snippet_shape", "get_video_items_from_html.url_missing",
&format!("unexpected snippet length={}", vid.len()), "missing iframe src in segment",
); );
continue; continue;
} }
let line0 = vid.get(0).copied().unwrap_or_default(); url_str = url_str.replace("index.php", "xs1.php");
let line1 = vid.get(1).copied().unwrap_or_default();
let line4 = vid.get(4).copied().unwrap_or_default();
let line6 = vid.get(6).copied().unwrap_or_default();
let line7 = vid.get(7).copied().unwrap_or_default();
// for (index, line) in vid.iter().enumerate() {
// println!("Line {}: {}", index, line.to_string().trim());
// }
let mut title = line1.split(">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
// html decode
title = decode(title.as_bytes()).to_string().unwrap_or(title);
if !line1.contains("iframe src=&quot;") {
continue;
}
let url_str = line1.split("iframe src=&quot;").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("&quot;")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string()
.replace("index.php", "xs1.php");
if url_str.starts_with("https://streamtape.com/") { if url_str.starts_with("https://streamtape.com/") {
continue; // Skip Streamtape links continue; // Skip Streamtape links
} }
let id = url_str.split("data=").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("&") let id_url = Self::extract_between(video_segment, "data-url='", "'")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .or_else(|| Self::extract_between(video_segment, "data-url=\"", "\""))
.unwrap_or_default()
.to_string();
let mut id = url_str
.split("data=")
.nth(1)
.unwrap_or_default()
.split('&')
.next()
.unwrap_or_default()
.to_string();
if id.is_empty() {
id = id_url
.trim_end_matches('/')
.rsplit('/')
.next()
.unwrap_or_default()
.to_string();
}
let raw_duration = Self::extract_between(video_segment, "time_dur\">", "<")
.or_else(|| Self::extract_between(video_segment, "class=\"time\">", "<"))
.unwrap_or("00:00")
.to_string(); .to_string();
let raw_duration = match vid.len() {
10 => line6.split("time_dur\">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string(),
_ => "00:00".to_string(),
};
let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32; let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
let thumb = Self::extract_thumb(video_segment);
if !line4.contains("srcset=")
&& line4.split("src=\"").collect::<Vec<&str>>().len() == 1
{
for (index, line) in vid.iter().enumerate() {
println!("Line {}: {}\n\n", index, line);
}
}
let mut thumb = "".to_string();
for v in vid.clone() {
let line = v.trim();
if line.starts_with("<img ") {
thumb = line.split(" src=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
}
}
let embed_html = line1.split("data-embed='").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("'")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
let id_url = line1.split("data-url='").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("'")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
match pool.get() { match pool.get() {
Ok(mut conn) => { Ok(mut conn) => {
let _ = db::insert_video(&mut conn, &id_url, &url_str); if !id_url.is_empty() {
let _ = db::insert_video(&mut conn, &id_url, &url_str);
}
} }
Err(e) => { Err(e) => {
report_provider_error_background( report_provider_error_background(
@@ -301,26 +358,36 @@ impl PerverzijaProvider {
} }
} }
let referer_url = "https://xtremestream.xyz/".to_string(); let referer_url = "https://xtremestream.xyz/".to_string();
let embed = VideoEmbed::new(embed_html, url_str.clone());
let mut tags: Vec<String> = Vec::new(); // Placeholder for tags, adjust as needed let mut tags: Vec<String> = Vec::new();
let studios_parts = line7.split("a href=\"").collect::<Vec<&str>>(); let studios_parts = video_segment.split("a href=\"").collect::<Vec<&str>>();
for studio in studios_parts.iter().skip(1) { for studio in studios_parts.iter().skip(1) {
if studio.starts_with("https://tube.perverzija.com/studio/") { if studio.starts_with("https://tube.perverzija.com/studio/") {
tags.push( tags.push(
studio.split("/\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default() studio
.split("/\"")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.replace("https://tube.perverzija.com/studio/", "@studio:") .replace("https://tube.perverzija.com/studio/", "@studio:")
.to_string(), .to_string(),
); );
} }
} }
for tag in line0.split(" ").collect::<Vec<&str>>() { for tag in video_segment.split_whitespace() {
if tag.starts_with("stars-") { let token =
let tag_name = tag.split("stars-").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() tag.trim_matches(|c: char| c == '"' || c == '\'' || c == '>' || c == '<');
.split("\"") if token.starts_with("stars-") {
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() let tag_name = token
.split("stars-")
.nth(1)
.unwrap_or_default()
.split('"')
.next()
.unwrap_or_default()
.to_string(); .to_string();
if !tag_name.is_empty() { if !tag_name.is_empty() {
tags.push(format!("@stars:{}", tag_name)); tags.push(format!("@stars:{}", tag_name));
@@ -328,9 +395,11 @@ impl PerverzijaProvider {
} }
} }
for tag in line0.split(" ").collect::<Vec<&str>>() { for tag in video_segment.split_whitespace() {
if tag.starts_with("tag-") { let token =
let tag_name = tag.split("tag-").collect::<Vec<&str>>().get(1).copied().unwrap_or_default().to_string(); tag.trim_matches(|c: char| c == '"' || c == '\'' || c == '>' || c == '<');
if token.starts_with("tag-") {
let tag_name = token.split("tag-").nth(1).unwrap_or_default().to_string();
if !tag_name.is_empty() { if !tag_name.is_empty() {
tags.push(tag_name.replace("-", " ").to_string()); tags.push(tag_name.replace("-", " ").to_string());
} }
@@ -339,7 +408,7 @@ impl PerverzijaProvider {
let mut video_item = VideoItem::new( let mut video_item = VideoItem::new(
id, id,
title, title,
embed.source.clone(), url_str.clone(),
"perverzija".to_string(), "perverzija".to_string(),
thumb, thumb,
duration, duration,
@@ -361,7 +430,15 @@ impl PerverzijaProvider {
} }
async fn get_video_items_from_html_query(&self, html: String, pool: DbPool) -> Vec<VideoItem> { async fn get_video_items_from_html_query(&self, html: String, pool: DbPool) -> Vec<VideoItem> {
let raw_videos = html.split("video-item post").collect::<Vec<&str>>()[1..].to_vec(); let raw_videos: Vec<&str> = html.split("video-item post").skip(1).collect();
if raw_videos.is_empty() {
report_provider_error_background(
"perverzija",
"get_video_items_from_html_query.no_segments",
&format!("html_len={}", html.len()),
);
return vec![];
}
let futures = raw_videos let futures = raw_videos
.into_iter() .into_iter()
.map(|el| self.get_video_item(el, pool.clone())); .map(|el| self.get_video_item(el, pool.clone()));
@@ -372,53 +449,39 @@ impl PerverzijaProvider {
} }
async fn get_video_item(&self, snippet: &str, pool: DbPool) -> Result<VideoItem> { async fn get_video_item(&self, snippet: &str, pool: DbPool) -> Result<VideoItem> {
let vid = snippet.split("\n").collect::<Vec<&str>>(); if snippet.trim().is_empty() {
if vid.len() > 30 || vid.len() < 7 {
report_provider_error_background( report_provider_error_background(
"perverzija", "perverzija",
"get_video_item.snippet_shape", "get_video_item.empty_snippet",
&format!("unexpected snippet length={}", vid.len()), "snippet is empty",
); );
return Err("Unexpected video snippet length".into()); return Err("empty snippet".into());
} }
let line5 = vid.get(5).copied().unwrap_or_default();
let line6 = vid.get(6).copied().unwrap_or_default();
let mut title = line5.split(" title=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() let title = Self::extract_title(snippet);
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
title = decode(title.as_bytes()).to_string().unwrap_or(title);
let thumb = match line6.split(" src=\"").collect::<Vec<&str>>().len() { let thumb = Self::extract_thumb(snippet);
1 => {
for (index, line) in vid.iter().enumerate() {
println!("Line {}: {}", index, line.to_string().trim());
}
return Err("Failed to parse thumbnail URL".into());
}
_ => line6.split(" src=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string(),
};
let duration = 0; let duration = 0;
let lookup_url = line5.split(" href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() let lookup_url = Self::extract_between(snippet, " href=\"", "\"")
.split("\"") .or_else(|| Self::extract_between(snippet, "data-url='", "'"))
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .unwrap_or_default()
.to_string(); .to_string();
if lookup_url.is_empty() {
report_provider_error_background(
"perverzija",
"get_video_item.lookup_url_missing",
"missing lookup url in snippet",
);
return Err("Failed to parse lookup url".into());
}
let referer_url = "https://xtremestream.xyz/".to_string(); let referer_url = "https://xtremestream.xyz/".to_string();
let mut conn = match pool.get() { let mut conn = match pool.get() {
Ok(conn) => conn, Ok(conn) => conn,
Err(e) => { Err(e) => {
report_provider_error( report_provider_error("perverzija", "get_video_item.pool_get", &e.to_string())
"perverzija", .await;
"get_video_item.pool_get",
&e.to_string(),
)
.await;
return Err("couldn't get db connection from pool".into()); return Err("couldn't get db connection from pool".into());
} }
}; };
@@ -433,9 +496,21 @@ impl PerverzijaProvider {
if url_str.starts_with("!") { if url_str.starts_with("!") {
return Err("Video was removed".into()); return Err("Video was removed".into());
} }
let mut id = url_str.split("data=").collect::<Vec<&str>>().get(1).copied().unwrap_or_default().to_string(); let mut id = url_str
.split("data=")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.to_string();
if id.contains("&") { if id.contains("&") {
id = id.split("&").collect::<Vec<&str>>().get(0).copied().unwrap_or_default().to_string() id = id
.split("&")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string()
} }
let mut video_item = VideoItem::new( let mut video_item = VideoItem::new(
id, id,
@@ -481,9 +556,17 @@ impl PerverzijaProvider {
} }
}; };
let mut url_str = text.split("<iframe src=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() let mut url_str = text
.split("<iframe src=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"") .split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string() .to_string()
.replace("index.php", "xs1.php"); .replace("index.php", "xs1.php");
if !url_str.contains("xtremestream.xyz") { if !url_str.contains("xtremestream.xyz") {
@@ -494,15 +577,26 @@ impl PerverzijaProvider {
let studios_parts = text let studios_parts = text
.split("<strong>Studio: </strong>") .split("<strong>Studio: </strong>")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("</div>") .split("</div>")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.split("<a href=\"") .split("<a href=\"")
.collect::<Vec<&str>>(); .collect::<Vec<&str>>();
for studio in studios_parts.iter().skip(1) { for studio in studios_parts.iter().skip(1) {
if studio.starts_with("https://tube.perverzija.com/studio/") { if studio.starts_with("https://tube.perverzija.com/studio/") {
tags.push( tags.push(
studio.split("/\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default() studio
.split("/\"")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.replace("https://tube.perverzija.com/studio/", "@studio:") .replace("https://tube.perverzija.com/studio/", "@studio:")
.to_string(), .to_string(),
); );
@@ -511,15 +605,25 @@ impl PerverzijaProvider {
if text.contains("<strong>Stars: </strong>") { if text.contains("<strong>Stars: </strong>") {
let stars_parts: Vec<&str> = text let stars_parts: Vec<&str> = text
.split("<strong>Stars: </strong>") .split("<strong>Stars: </strong>")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("</div>") .split("</div>")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.split("<a href=\"") .split("<a href=\"")
.collect::<Vec<&str>>(); .collect::<Vec<&str>>();
for star in stars_parts.iter().skip(1) { for star in stars_parts.iter().skip(1) {
if star.starts_with("https://tube.perverzija.com/stars/") { if star.starts_with("https://tube.perverzija.com/stars/") {
tags.push( tags.push(
star.split("/\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default() star.split("/\"")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.replace("https://tube.perverzija.com/stars/", "@stars:") .replace("https://tube.perverzija.com/stars/", "@stars:")
.to_string(), .to_string(),
); );
@@ -527,15 +631,27 @@ impl PerverzijaProvider {
} }
} }
let tags_parts: Vec<&str> = text.split("<strong>Tags: </strong>").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() let tags_parts: Vec<&str> = text
.split("<strong>Tags: </strong>")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("</div>") .split("</div>")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.split("<a href=\"") .split("<a href=\"")
.collect::<Vec<&str>>(); .collect::<Vec<&str>>();
for star in tags_parts.iter().skip(1) { for star in tags_parts.iter().skip(1) {
if star.starts_with("https://tube.perverzija.com/stars/") { if star.starts_with("https://tube.perverzija.com/stars/") {
tags.push( tags.push(
star.split("/\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default() star.split("/\"")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.replace("https://tube.perverzija.com/stars/", "@stars:") .replace("https://tube.perverzija.com/stars/", "@stars:")
.to_string(), .to_string(),
); );
@@ -574,9 +690,21 @@ impl PerverzijaProvider {
if !url_str.contains("xtremestream.xyz") { if !url_str.contains("xtremestream.xyz") {
return Err("Video URL does not contain xtremestream.xyz".into()); return Err("Video URL does not contain xtremestream.xyz".into());
} }
let mut id = url_str.split("data=").collect::<Vec<&str>>().get(1).copied().unwrap_or_default().to_string(); let mut id = url_str
.split("data=")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.to_string();
if id.contains("&") { if id.contains("&") {
id = id.split("&").collect::<Vec<&str>>().get(0).copied().unwrap_or_default().to_string() id = id
.split("&")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string()
} }
// if !vid[6].contains(" src=\""){ // if !vid[6].contains(" src=\""){
// for (index,line) in vid.iter().enumerate() { // for (index,line) in vid.iter().enumerate() {

49
src/providers/pimpbunny.rs
View File

@@ -11,7 +11,7 @@ use crate::videos::{ServerOptions, VideoFormat, VideoItem};
use async_trait::async_trait; use async_trait::async_trait;
use error_chain::error_chain; use error_chain::error_chain;
use futures::future::join_all; use futures::future::join_all;
use htmlentity::entity::{decode, ICodedDataTrait}; use htmlentity::entity::{ICodedDataTrait, decode};
use std::sync::{Arc, RwLock}; use std::sync::{Arc, RwLock};
use std::{thread, vec}; use std::{thread, vec};
use titlecase::Titlecase; use titlecase::Titlecase;
@@ -122,7 +122,8 @@ impl PimpbunnyProvider {
file!(), file!(),
line!(), line!(),
module_path!(), module_path!(),
).await; )
.await;
return; return;
} }
}; };
@@ -138,7 +139,8 @@ impl PimpbunnyProvider {
file!(), file!(),
line!(), line!(),
module_path!(), module_path!(),
).await; )
.await;
} }
if let Err(e) = Self::load_categories(&url, Arc::clone(&categories)).await { if let Err(e) = Self::load_categories(&url, Arc::clone(&categories)).await {
eprintln!("load_categories failed: {e}"); eprintln!("load_categories failed: {e}");
@@ -150,7 +152,8 @@ impl PimpbunnyProvider {
file!(), file!(),
line!(), line!(),
module_path!(), module_path!(),
).await; )
.await;
} }
}); });
}); });
@@ -183,7 +186,9 @@ impl PimpbunnyProvider {
.unwrap_or(""); .unwrap_or("");
for el in block.split("<div class=\"col\">").skip(1) { for el in block.split("<div class=\"col\">").skip(1) {
if el.contains("pb-promoted-link") || !el.contains("href=\"https://pimpbunny.com/onlyfans-models/") { if el.contains("pb-promoted-link")
|| !el.contains("href=\"https://pimpbunny.com/onlyfans-models/")
{
continue; continue;
} }
@@ -309,7 +314,9 @@ impl PimpbunnyProvider {
let mut video_url = format!( let mut video_url = format!(
"{}/search/{}/?mode=async&function=get_block&block_id=list_videos_videos_list_search_result&videos_per_page=32&from_videos={}", "{}/search/{}/?mode=async&function=get_block&block_id=list_videos_videos_list_search_result&videos_per_page=32&from_videos={}",
self.url, search_string.replace(" ","-"), page self.url,
search_string.replace(" ", "-"),
page
); );
let sort_string = match options.sort.as_deref().unwrap_or("") { let sort_string = match options.sort.as_deref().unwrap_or("") {
@@ -423,11 +430,7 @@ impl PimpbunnyProvider {
.collect() .collect()
} }
async fn get_video_item( async fn get_video_item(&self, seg: String, mut requester: Requester) -> Result<VideoItem> {
&self,
seg: String,
mut requester: Requester,
) -> Result<VideoItem> {
let video_url = seg let video_url = seg
.split(" href=\"") .split(" href=\"")
.nth(1) .nth(1)
@@ -443,7 +446,10 @@ impl PimpbunnyProvider {
.ok_or_else(|| ErrorKind::Parse("video title".into()))? .ok_or_else(|| ErrorKind::Parse("video title".into()))?
.trim() .trim()
.to_string(); .to_string();
title = decode(title.as_bytes()).to_string().unwrap_or(title).titlecase(); title = decode(title.as_bytes())
.to_string()
.unwrap_or(title)
.titlecase();
let id = video_url let id = video_url
.split('/') .split('/')
@@ -483,18 +489,13 @@ impl PimpbunnyProvider {
let (tags, formats, views, duration) = let (tags, formats, views, duration) =
self.extract_media(&video_url, &mut requester).await?; self.extract_media(&video_url, &mut requester).await?;
Ok(VideoItem::new( Ok(
id, VideoItem::new(id, title, video_url, "pimpbunny".into(), thumb, duration)
title, .formats(formats)
video_url, .tags(tags)
"pimpbunny".into(), .preview(preview)
thumb, .views(views),
duration,
) )
.formats(formats)
.tags(tags)
.preview(preview)
.views(views))
} }
async fn extract_media( async fn extract_media(
@@ -532,7 +533,7 @@ impl PimpbunnyProvider {
let duration = json["duration"] let duration = json["duration"]
.as_str() .as_str()
.map(|d| parse_time_to_seconds(&d.replace(['P','T','H','M','S'], "")).unwrap_or(0)) .map(|d| parse_time_to_seconds(&d.replace(['P', 'T', 'H', 'M', 'S'], "")).unwrap_or(0))
.unwrap_or(0) as u32; .unwrap_or(0) as u32;
Ok(( Ok((

257
src/providers/pmvhaven.rs
View File

@@ -1,6 +1,6 @@
use crate::DbPool; use crate::DbPool;
use crate::api::ClientVersion; use crate::api::ClientVersion;
use crate::providers::Provider; use crate::providers::{Provider, report_provider_error_background, requester_or_default};
use crate::status::*; use crate::status::*;
use crate::util::cache::VideoCache; use crate::util::cache::VideoCache;
use crate::util::discord::send_discord_error_report; use crate::util::discord::send_discord_error_report;
@@ -8,10 +8,11 @@ use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoFormat, VideoItem}; use crate::videos::{ServerOptions, VideoFormat, VideoItem};
use async_trait::async_trait; use async_trait::async_trait;
use error_chain::error_chain; use error_chain::error_chain;
use htmlentity::entity::{decode, ICodedDataTrait}; use htmlentity::entity::{ICodedDataTrait, decode};
use std::fmt::Write;
use std::sync::{Arc, RwLock}; use std::sync::{Arc, RwLock};
use std::vec; use std::vec;
use std::fmt::Write; use url::form_urlencoded::byte_serialize;
error_chain! { error_chain! {
foreign_links { foreign_links {
@@ -135,9 +136,7 @@ impl PmvhavenProvider {
fn is_direct_media_url(url: &str) -> bool { fn is_direct_media_url(url: &str) -> bool {
let lower = url.to_ascii_lowercase(); let lower = url.to_ascii_lowercase();
(lower.starts_with("http://") || lower.starts_with("https://")) (lower.starts_with("http://") || lower.starts_with("https://"))
&& (lower.contains("/videos/") && (lower.contains("/videos/") || lower.contains(".mp4") || lower.contains(".m3u8"))
|| lower.contains(".mp4")
|| lower.contains(".m3u8"))
} }
fn pick_downloadable_media_url(&self, video: &serde_json::Value) -> Option<String> { fn pick_downloadable_media_url(&self, video: &serde_json::Value) -> Option<String> {
@@ -192,67 +191,119 @@ impl PmvhavenProvider {
_ => "", _ => "",
}; };
let endpoint = if search.is_empty() { let encoded_search: String = byte_serialize(search.as_bytes()).collect();
"api/videos"
} else {
"api/videos/search"
};
let mut url = format!(
"{}/{endpoint}?limit=100&page={page}{duration}{sort}",
self.url
);
let mut extra_filters = String::new();
if let Ok(stars) = self.stars.read() { if let Ok(stars) = self.stars.read() {
if let Some(star) = stars.iter().find(|s| s.eq_ignore_ascii_case(&search)) { if let Some(star) = stars.iter().find(|s| s.eq_ignore_ascii_case(&search)) {
url.push_str(&format!("&stars={star}")); let encoded_star: String = byte_serialize(star.as_bytes()).collect();
extra_filters.push_str(&format!("&stars={encoded_star}"));
} }
} }
if let Ok(cats) = self.categories.read() { if let Ok(cats) = self.categories.read() {
if let Some(cat) = cats.iter().find(|c| c.eq_ignore_ascii_case(&search)) { if let Some(cat) = cats.iter().find(|c| c.eq_ignore_ascii_case(&search)) {
url.push_str(&format!("&tagMode=OR&tags={cat}&expandTags=false")); let encoded_cat: String = byte_serialize(cat.as_bytes()).collect();
extra_filters.push_str(&format!("&tagMode=OR&tags={encoded_cat}&expandTags=false"));
} }
} }
if !search.is_empty() { let mut urls = vec![];
url.push_str(&format!("&q={search}")); if search.is_empty() {
urls.push(format!(
"{}/api/videos?limit=100&page={page}{duration}{sort}{extra_filters}",
self.url
));
} else {
urls.push(format!(
"{}/api/videos/search?limit=100&page={page}{duration}{sort}{extra_filters}&q={encoded_search}",
self.url
));
urls.push(format!(
"{}/api/videos/search?limit=100&page={page}{duration}{sort}{extra_filters}&query={encoded_search}",
self.url
));
urls.push(format!(
"{}/api/videos?limit=100&page={page}{duration}{sort}{extra_filters}&q={encoded_search}",
self.url
));
urls.push(format!(
"{}/api/videos?limit=100&page={page}{duration}{sort}{extra_filters}&search={encoded_search}",
self.url
));
} }
if let Some((time, items)) = cache.get(&url) {
if time.elapsed().unwrap_or_default().as_secs() < 300 { let mut requester = requester_or_default(&options, "pmvhaven", "query");
return Ok(items.clone()); for url in urls {
if let Some((time, items)) = cache.get(&url) {
if time.elapsed().unwrap_or_default().as_secs() < 300 {
return Ok(items.clone());
}
}
let text = match requester.get(&url, None).await {
Ok(text) => text,
Err(err) => {
report_provider_error_background(
"pmvhaven",
"get.request",
&format!("url={url}; error={err}"),
);
continue;
}
};
let json: serde_json::Value = match serde_json::from_str(&text) {
Ok(json) => json,
Err(err) => {
report_provider_error_background(
"pmvhaven",
"parse.json",
&format!("url={url}; error={err}"),
);
continue;
}
};
let items = self.get_video_items_from_json(json).await;
if !items.is_empty() {
cache.remove(&url);
cache.insert(url, items.clone());
return Ok(items);
} }
} }
let mut requester = match options.requester { Ok(vec![])
Some(r) => r,
None => return Ok(vec![]),
};
let text = requester.get(&url, None).await.unwrap_or_default();
let json = serde_json::from_str(&text).unwrap_or(serde_json::Value::Null);
let items = self.get_video_items_from_json(json).await;
if !items.is_empty() {
cache.remove(&url);
cache.insert(url, items.clone());
}
Ok(items)
} }
async fn get_video_items_from_json(&self, json: serde_json::Value) -> Vec<VideoItem> { async fn get_video_items_from_json(&self, json: serde_json::Value) -> Vec<VideoItem> {
let mut items = vec![]; let mut items = vec![];
if !json.get("success").and_then(|v| v.as_bool()).unwrap_or(false) { if !json
.get("success")
.and_then(|v| v.as_bool())
.unwrap_or(false)
{
return items; return items;
} }
let videos = json.get("data").and_then(|v| v.as_array()).cloned().unwrap_or_default(); let videos = json
.get("data")
.and_then(|v| v.as_array())
.or_else(|| json.get("videos").and_then(|v| v.as_array()))
.cloned()
.unwrap_or_default();
for video in videos { for video in videos {
let title = decode(video.get("title").and_then(|v| v.as_str()).unwrap_or("").as_bytes()) let title = decode(
.to_string() video
.unwrap_or_default(); .get("title")
.and_then(|v| v.as_str())
.unwrap_or("")
.as_bytes(),
)
.to_string()
.unwrap_or_default();
let id = video let id = video
.get("_id") .get("_id")
@@ -266,14 +317,36 @@ impl PmvhavenProvider {
continue; continue;
} }
}; };
let thumb = video.get("thumbnailUrl").and_then(|v| v.as_str()).unwrap_or("").to_string(); let thumb = video
let preview = video.get("previewUrl").and_then(|v| v.as_str()).unwrap_or("").to_string(); .get("thumbnailUrl")
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string();
let preview = video
.get("previewUrl")
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string();
let views = video.get("views").and_then(|v| v.as_u64()).unwrap_or(0); let views = video.get("views").and_then(|v| v.as_u64()).unwrap_or(0);
let duration = parse_time_to_seconds(video.get("duration").and_then(|v| v.as_str()).unwrap_or("0")).unwrap_or(0); let duration = parse_time_to_seconds(
video
.get("duration")
.and_then(|v| v.as_str())
.unwrap_or("0"),
)
.unwrap_or(0);
let tags = video.get("tags").and_then(|v| v.as_array()).cloned().unwrap_or_default(); let tags = video
let stars = video.get("starsTags").and_then(|v| v.as_array()).cloned().unwrap_or_default(); .get("tags")
.and_then(|v| v.as_array())
.cloned()
.unwrap_or_default();
let stars = video
.get("starsTags")
.and_then(|v| v.as_array())
.cloned()
.unwrap_or_default();
for t in tags.iter() { for t in tags.iter() {
if let Some(s) = t.as_str() { if let Some(s) = t.as_str() {
let decoded = decode(s.as_bytes()).to_string().unwrap_or_default(); let decoded = decode(s.as_bytes()).to_string().unwrap_or_default();
@@ -293,14 +366,21 @@ impl PmvhavenProvider {
"mp4".to_string() "mp4".to_string()
}; };
items.push( items.push(
VideoItem::new(id, title, video_url.clone(), "pmvhaven".into(), thumb, duration as u32) VideoItem::new(
.views(views as u32) id,
.formats(vec![VideoFormat::new( title,
video_url, video_url.clone(),
"1080".to_string(), "pmvhaven".into(),
format_type, thumb,
)]) duration as u32,
.preview(preview) )
.views(views as u32)
.formats(vec![VideoFormat::new(
video_url,
"1080".to_string(),
format_type,
)])
.preview(preview),
); );
} }
@@ -308,6 +388,56 @@ impl PmvhavenProvider {
} }
} }
#[cfg(test)]
mod tests {
use super::PmvhavenProvider;
use serde_json::json;
#[tokio::test]
async fn parses_videos_from_videos_key() {
let provider = PmvhavenProvider::new();
let payload = json!({
"success": true,
"videos": [{
"_id": "abc123",
"title": "Sample Title",
"videoUrl": "https://video.pmvhaven.com/videos/sample.mp4",
"thumbnailUrl": "https://video.pmvhaven.com/thumbnails/sample.webp",
"previewUrl": "https://video.pmvhaven.com/previews/sample.mp4",
"views": 42,
"duration": "2:11",
"tags": [],
"starsTags": []
}]
});
let items = provider.get_video_items_from_json(payload).await;
assert_eq!(items.len(), 1);
}
#[tokio::test]
async fn parses_videos_from_data_key() {
let provider = PmvhavenProvider::new();
let payload = json!({
"success": true,
"data": [{
"_id": "abc123",
"title": "Sample Title",
"videoUrl": "https://video.pmvhaven.com/videos/sample.mp4",
"thumbnailUrl": "https://video.pmvhaven.com/thumbnails/sample.webp",
"previewUrl": "https://video.pmvhaven.com/previews/sample.mp4",
"views": 42,
"duration": "2:11",
"tags": [],
"starsTags": []
}]
});
let items = provider.get_video_items_from_json(payload).await;
assert_eq!(items.len(), 1);
}
}
#[async_trait] #[async_trait]
impl Provider for PmvhavenProvider { impl Provider for PmvhavenProvider {
async fn get_videos( async fn get_videos(
@@ -332,14 +462,15 @@ impl Provider for PmvhavenProvider {
let _ = writeln!(chain_str, "{}. {}", i + 1, cause); let _ = writeln!(chain_str, "{}. {}", i + 1, cause);
} }
send_discord_error_report( send_discord_error_report(
e.to_string(), e.to_string(),
Some(chain_str), Some(chain_str),
Some("PMVHaven Provider"), Some("PMVHaven Provider"),
Some("Failed to load videos from PMVHaven"), Some("Failed to load videos from PMVHaven"),
file!(), file!(),
line!(), line!(),
module_path!(), module_path!(),
).await; )
.await;
vec![] vec![]
} }
} }

102
src/providers/porn00.rs
View File

@@ -1,15 +1,15 @@
use crate::api::ClientVersion;
use crate::util::parse_abbreviated_number;
use crate::DbPool; use crate::DbPool;
use crate::api::ClientVersion;
use crate::providers::Provider; use crate::providers::Provider;
use crate::status::*; use crate::status::*;
use crate::util::cache::VideoCache; use crate::util::cache::VideoCache;
use crate::util::parse_abbreviated_number;
use crate::util::time::parse_time_to_seconds; use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoItem}; use crate::videos::{ServerOptions, VideoItem};
use async_trait::async_trait;
use error_chain::error_chain; use error_chain::error_chain;
use htmlentity::entity::{ICodedDataTrait, decode}; use htmlentity::entity::{ICodedDataTrait, decode};
use std::vec; use std::vec;
use async_trait::async_trait;
error_chain! { error_chain! {
foreign_links { foreign_links {
@@ -132,7 +132,10 @@ impl Porn00Provider {
options: ServerOptions, options: ServerOptions,
) -> Result<Vec<VideoItem>> { ) -> Result<Vec<VideoItem>> {
let search_string = query.to_lowercase().trim().replace(" ", "-"); let search_string = query.to_lowercase().trim().replace(" ", "-");
let video_url = format!("{}/q/{}/?mode=async&function=get_block&block_id=list_videos_videos_list_search_result&q={}&category_ids=&sort_by=post_date&from_videos={}&from_albums={}&", self.url, search_string, search_string, page, page); let video_url = format!(
"{}/q/{}/?mode=async&function=get_block&block_id=list_videos_videos_list_search_result&q={}&category_ids=&sort_by=post_date&from_videos={}&from_albums={}&",
self.url, search_string, search_string, page, page
);
// Check our Video Cache. If the result is younger than 1 hour, we return it. // Check our Video Cache. If the result is younger than 1 hour, we return it.
let old_items = match cache.get(&video_url) { let old_items = match cache.get(&video_url) {
Some((time, items)) => { Some((time, items)) => {
@@ -178,7 +181,12 @@ impl Porn00Provider {
return vec![]; return vec![];
} }
let mut items: Vec<VideoItem> = Vec::new(); let mut items: Vec<VideoItem> = Vec::new();
let raw_videos = html.split("<div class=\"pagination\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default() let raw_videos = html
.split("<div class=\"pagination\"")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.split("<div class=\"item \">") .split("<div class=\"item \">")
.collect::<Vec<&str>>()[1..] .collect::<Vec<&str>>()[1..]
.to_vec(); .to_vec();
@@ -187,31 +195,82 @@ impl Porn00Provider {
// for (index, line) in vid.iter().enumerate() { // for (index, line) in vid.iter().enumerate() {
// println!("Line {}: {}", index, line); // println!("Line {}: {}", index, line);
// } // }
let video_url: String = video_segment.split("<a href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() let video_url: String = video_segment
.split("\"") .split("<a href=\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default().to_string(); .collect::<Vec<&str>>()
let mut title = video_segment.split("\" title=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .get(1)
.copied()
.unwrap_or_default()
.split("\"") .split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
let mut title = video_segment
.split("\" title=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(); .to_string();
// html decode // html decode
title = decode(title.as_bytes()).to_string().unwrap_or(title); title = decode(title.as_bytes()).to_string().unwrap_or(title);
let id = video_url.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string(); let id = video_url
let raw_duration = video_segment.split("<div class=\"duration\">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .split("/")
.collect::<Vec<&str>>()
.get(4)
.copied()
.unwrap_or_default()
.to_string();
let raw_duration = video_segment
.split("<div class=\"duration\">")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<") .split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(); .to_string();
let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32; let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
let thumb = video_segment.split("<img class=\"thumb ").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() let thumb = video_segment
.split("data-original=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .split("<img class=\"thumb ")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("data-original=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"") .split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(); .to_string();
let views_part = video_segment.split("<div class=\"views\">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() let views_part = video_segment
.split("<div class=\"views\">")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<") .split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(); .to_string();
let views = parse_abbreviated_number(&views_part).unwrap_or(0) as u32; let views = parse_abbreviated_number(&views_part).unwrap_or(0) as u32;
@@ -223,14 +282,11 @@ impl Porn00Provider {
thumb, thumb,
duration, duration,
) )
.views(views) .views(views);
;
items.push(video_item); items.push(video_item);
} }
return items; return items;
} }
} }
#[async_trait] #[async_trait]
@@ -249,7 +305,7 @@ impl Provider for Porn00Provider {
let _ = pool; let _ = pool;
let videos: std::result::Result<Vec<VideoItem>, Error> = match query { let videos: std::result::Result<Vec<VideoItem>, Error> = match query {
Some(q) => { Some(q) => {
self.query(cache, page.parse::<u8>().unwrap_or(1), &q,options) self.query(cache, page.parse::<u8>().unwrap_or(1), &q, options)
.await .await
} }
None => { None => {

166
src/providers/pornhat.rs
View File

@@ -1,15 +1,15 @@
use crate::api::ClientVersion;
use crate::util::parse_abbreviated_number;
use crate::DbPool; use crate::DbPool;
use crate::api::ClientVersion;
use crate::providers::Provider; use crate::providers::Provider;
use crate::status::*; use crate::status::*;
use crate::util::cache::VideoCache; use crate::util::cache::VideoCache;
use crate::util::parse_abbreviated_number;
use crate::util::time::parse_time_to_seconds; use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoItem}; use crate::videos::{ServerOptions, VideoItem};
use async_trait::async_trait;
use error_chain::error_chain; use error_chain::error_chain;
use htmlentity::entity::{ICodedDataTrait, decode}; use htmlentity::entity::{ICodedDataTrait, decode};
use std::vec; use std::vec;
use async_trait::async_trait;
error_chain! { error_chain! {
foreign_links { foreign_links {
@@ -69,7 +69,7 @@ impl PornhatProvider {
cache: VideoCache, cache: VideoCache,
page: u8, page: u8,
sort: &str, sort: &str,
options:ServerOptions options: ServerOptions,
) -> Result<Vec<VideoItem>> { ) -> Result<Vec<VideoItem>> {
let sort_string = match sort { let sort_string = match sort {
"trending" => "/trending", "trending" => "/trending",
@@ -117,13 +117,19 @@ impl PornhatProvider {
cache: VideoCache, cache: VideoCache,
page: u8, page: u8,
query: &str, query: &str,
options:ServerOptions options: ServerOptions,
) -> Result<Vec<VideoItem>> { ) -> Result<Vec<VideoItem>> {
let search_string = query.to_lowercase().trim().replace(" ", "-"); let search_string = query.to_lowercase().trim().replace(" ", "-");
let mut video_url = format!("{}/search/{}/{}/", self.url, search_string, page); let mut video_url = format!("{}/search/{}/{}/", self.url, search_string, page);
if search_string.starts_with("@"){ if search_string.starts_with("@") {
let url_part = search_string.split("@").collect::<Vec<&str>>().get(1).copied().unwrap_or_default().replace(":", "/"); let url_part = search_string
.split("@")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.replace(":", "/");
video_url = format!("{}/{}/", self.url, url_part); video_url = format!("{}/{}/", self.url, url_part);
} }
// Check our Video Cache. If the result is younger than 1 hour, we return it. // Check our Video Cache. If the result is younger than 1 hour, we return it.
@@ -170,7 +176,12 @@ impl PornhatProvider {
return vec![]; return vec![];
} }
let mut items: Vec<VideoItem> = Vec::new(); let mut items: Vec<VideoItem> = Vec::new();
let raw_videos = html.split("<div class=\"pagination\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default() let raw_videos = html
.split("<div class=\"pagination\"")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.split("item thumb-bl thumb-bl-video video_") .split("item thumb-bl thumb-bl-video video_")
.collect::<Vec<&str>>()[1..] .collect::<Vec<&str>>()[1..]
.to_vec(); .to_vec();
@@ -179,61 +190,147 @@ impl PornhatProvider {
// for (index, line) in vid.iter().enumerate() { // for (index, line) in vid.iter().enumerate() {
// println!("Line {}: {}", index, line); // println!("Line {}: {}", index, line);
// } // }
let video_url: String = format!("{}{}", self.url, video_segment.split("<a href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() let video_url: String = format!(
"{}{}",
self.url,
video_segment
.split("<a href=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"") .split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()); .collect::<Vec<&str>>()
let preview_url = video_segment.split("data-preview-custom=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .get(0)
.copied()
.unwrap_or_default()
);
let preview_url = video_segment
.split("data-preview-custom=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"") .split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(); .to_string();
let mut title = video_segment.split("\" title=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() let mut title = video_segment
.split("\" title=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"") .split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(); .to_string();
// html decode // html decode
title = decode(title.as_bytes()).to_string().unwrap_or(title); title = decode(title.as_bytes()).to_string().unwrap_or(title);
let id = video_url.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string(); let id = video_url
let raw_duration = video_segment.split("fa fa-clock-o").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .split("/")
.split("<span>").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(4)
.copied()
.unwrap_or_default()
.to_string();
let raw_duration = video_segment
.split("fa fa-clock-o")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<span>")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<") .split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(); .to_string();
let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32; let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
let thumb = video_segment.split("<img class=\"thumb lazy-load\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() let thumb = video_segment
.split("data-original=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .split("<img class=\"thumb lazy-load\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("data-original=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"") .split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(); .to_string();
let mut tags = vec![]; let mut tags = vec![];
if video_segment.contains("href=\"/sites/"){ if video_segment.contains("href=\"/sites/") {
let raw_tags = video_segment.split("href=\"/sites/").collect::<Vec<&str>>()[1..] let raw_tags = video_segment.split("href=\"/sites/").collect::<Vec<&str>>()[1..]
.iter() .iter()
.map(|s| s.split("/\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default().to_string()) .map(|s| {
s.split("/\"")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string()
})
.collect::<Vec<String>>(); .collect::<Vec<String>>();
for tag in raw_tags { for tag in raw_tags {
if !tag.is_empty() { if !tag.is_empty() {
tags.push(format!("@sites:{}",tag)); tags.push(format!("@sites:{}", tag));
} }
} }
} }
if video_segment.contains("href=\"/models/"){ if video_segment.contains("href=\"/models/") {
let raw_tags = video_segment.split("href=\"/models/").collect::<Vec<&str>>()[1..] let raw_tags = video_segment
.split("href=\"/models/")
.collect::<Vec<&str>>()[1..]
.iter() .iter()
.map(|s| s.split("/\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default().to_string()) .map(|s| {
s.split("/\"")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string()
})
.collect::<Vec<String>>(); .collect::<Vec<String>>();
for tag in raw_tags { for tag in raw_tags {
if !tag.is_empty() { if !tag.is_empty() {
tags.push(format!("@models:{}",tag)); tags.push(format!("@models:{}", tag));
} }
} }
} }
let views_part = video_segment.split("fa fa-eye").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() let views_part = video_segment
.split("<span>").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .split("fa fa-eye")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<span>")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<") .split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(); .to_string();
let views = parse_abbreviated_number(&views_part).unwrap_or(0) as u32; let views = parse_abbreviated_number(&views_part).unwrap_or(0) as u32;
@@ -247,14 +344,11 @@ impl PornhatProvider {
) )
.preview(preview_url) .preview(preview_url)
.views(views) .views(views)
.tags(tags) .tags(tags);
;
items.push(video_item); items.push(video_item);
} }
return items; return items;
} }
} }
#[async_trait] #[async_trait]

35
src/providers/pornhub.rs
View File

@@ -1,15 +1,15 @@
use crate::api::ClientVersion;
use crate::util::parse_abbreviated_number;
use crate::DbPool; use crate::DbPool;
use crate::api::ClientVersion;
use crate::providers::Provider; use crate::providers::Provider;
use crate::status::*; use crate::status::*;
use crate::util::cache::VideoCache; use crate::util::cache::VideoCache;
use crate::util::parse_abbreviated_number;
use crate::util::time::parse_time_to_seconds; use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoItem}; use crate::videos::{ServerOptions, VideoItem};
use error_chain::error_chain;
use htmlentity::entity::{decode, ICodedDataTrait};
use async_trait::async_trait; use async_trait::async_trait;
use error_chain::error_chain;
use htmlentity::entity::{ICodedDataTrait, decode};
use std::vec; use std::vec;
error_chain! { error_chain! {
@@ -130,14 +130,22 @@ impl PornhubProvider {
let mut split_string = "<ul id=\"video"; let mut split_string = "<ul id=\"video";
let search_string = query.to_lowercase().trim().replace(' ', "+"); let search_string = query.to_lowercase().trim().replace(' ', "+");
let mut video_url = let mut video_url = format!(
format!("{}/video/search?search={}&page={}", self.url, search_string, page); "{}/video/search?search={}&page={}",
self.url, search_string, page
);
if query.starts_with('@') { if query.starts_with('@') {
let mut parts = query[1..].split(':'); let mut parts = query[1..].split(':');
let a = parts.next().unwrap_or(""); let a = parts.next().unwrap_or("");
let b = parts.next().unwrap_or(""); let b = parts.next().unwrap_or("");
video_url = format!("{}/{}/{}/videos?page={}", self.url, a, b.replace(' ', "-"), page); video_url = format!(
"{}/{}/{}/videos?page={}",
self.url,
a,
b.replace(' ', "-"),
page
);
if query.contains("@model") || query.contains("@pornstar") { if query.contains("@model") || query.contains("@pornstar") {
split_string = "mostRecentVideosSection"; split_string = "mostRecentVideosSection";
@@ -207,7 +215,9 @@ impl PornhubProvider {
.and_then(|s| s.split('"').next()); .and_then(|s| s.split('"').next());
let video_url = match url_part { let video_url = match url_part {
Some(u) if !u.is_empty() && u != "javascript:void(0)" => format!("{}{}", self.url, u), Some(u) if !u.is_empty() && u != "javascript:void(0)" => {
format!("{}{}", self.url, u)
}
_ => continue, _ => continue,
}; };
@@ -274,14 +284,7 @@ impl PornhubProvider {
(None, None) (None, None)
}; };
let mut item = VideoItem::new( let mut item = VideoItem::new(id, title, video_url, "pornhub".into(), thumb, duration);
id,
title,
video_url,
"pornhub".into(),
thumb,
duration,
);
if views > 0 { if views > 0 {
item = item.views(views); item = item.views(views);

42
src/providers/pornzog.rs
View File

@@ -1,15 +1,15 @@
use crate::api::ClientVersion;
use crate::DbPool; use crate::DbPool;
use crate::api::ClientVersion;
use crate::providers::Provider; use crate::providers::Provider;
use crate::status::*; use crate::status::*;
use crate::util::cache::VideoCache; use crate::util::cache::VideoCache;
use crate::util::discord::{format_error_chain, send_discord_error_report}; use crate::util::discord::{format_error_chain, send_discord_error_report};
use crate::util::time::parse_time_to_seconds; use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoItem}; use crate::videos::{ServerOptions, VideoItem};
use async_trait::async_trait;
use error_chain::error_chain; use error_chain::error_chain;
use htmlentity::entity::{ICodedDataTrait, decode}; use htmlentity::entity::{ICodedDataTrait, decode};
use std::vec; use std::vec;
use async_trait::async_trait;
error_chain! { error_chain! {
foreign_links { foreign_links {
@@ -112,12 +112,15 @@ impl PornzogProvider {
// SAFE: Check if requester exists instead of unwrap() // SAFE: Check if requester exists instead of unwrap()
let mut requester = match options.requester.clone() { let mut requester = match options.requester.clone() {
Some(r) => r, Some(r) => r,
None => return Ok(old_items), None => return Ok(old_items),
}; };
let text = requester.get(&video_url, None).await.map_err(|e| format!("{}", e))?; let text = requester
.get(&video_url, None)
.await
.map_err(|e| format!("{}", e))?;
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone()); let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() { if !video_items.is_empty() {
cache.remove(&video_url); cache.remove(&video_url);
cache.insert(video_url.clone(), video_items.clone()); cache.insert(video_url.clone(), video_items.clone());
@@ -131,7 +134,7 @@ impl PornzogProvider {
if html.is_empty() { if html.is_empty() {
return vec![]; return vec![];
} }
let mut items: Vec<VideoItem> = Vec::new(); let mut items: Vec<VideoItem> = Vec::new();
// Helper for safe splitting: returns Option<&str> // Helper for safe splitting: returns Option<&str>
@@ -149,7 +152,7 @@ impl PornzogProvider {
let raw_videos: Vec<&str> = body.split("class=\"thumb-video ").skip(1).collect(); let raw_videos: Vec<&str> = body.split("class=\"thumb-video ").skip(1).collect();
for (idx, video_segment) in raw_videos.iter().enumerate() { for (idx, video_segment) in raw_videos.iter().enumerate() {
// Attempt to parse each item. If one fails, we log it and continue to the next // Attempt to parse each item. If one fails, we log it and continue to the next
// instead of crashing the whole request. // instead of crashing the whole request.
let result: Option<VideoItem> = (|| { let result: Option<VideoItem> = (|| {
let mut video_url = get_part(video_segment, "href=\"", 1)? let mut video_url = get_part(video_segment, "href=\"", 1)?
@@ -162,7 +165,9 @@ impl PornzogProvider {
} }
let title_raw = get_part(video_segment, "alt=\"", 1)?.split("\"").next()?; let title_raw = get_part(video_segment, "alt=\"", 1)?.split("\"").next()?;
let title = decode(title_raw.as_bytes()).to_string().unwrap_or(title_raw.to_string()); let title = decode(title_raw.as_bytes())
.to_string()
.unwrap_or(title_raw.to_string());
// The ID is the 5th element in a "/" split: e.g., "", "video", "123", "title" // The ID is the 5th element in a "/" split: e.g., "", "video", "123", "title"
let id = video_url.split("/").nth(4)?.to_string(); let id = video_url.split("/").nth(4)?.to_string();
@@ -179,8 +184,9 @@ impl PornzogProvider {
let tags_section = get_part(video_segment, "class=\"tags\"", 1)? let tags_section = get_part(video_segment, "class=\"tags\"", 1)?
.split("</p>") .split("</p>")
.next()?; .next()?;
let tags = tags_section.split("<a href=\"") let tags = tags_section
.split("<a href=\"")
.skip(1) .skip(1)
.filter_map(|el| { .filter_map(|el| {
let name = el.split(">").nth(1)?.split("<").next()?; let name = el.split(">").nth(1)?.split("<").next()?;
@@ -188,7 +194,10 @@ impl PornzogProvider {
}) })
.collect::<Vec<String>>(); .collect::<Vec<String>>();
Some(VideoItem::new(id, title, video_url, "pornzog".to_string(), thumb, duration).tags(tags)) Some(
VideoItem::new(id, title, video_url, "pornzog".to_string(), thumb, duration)
.tags(tags),
)
})(); })();
match result { match result {
@@ -214,7 +223,7 @@ impl Provider for PornzogProvider {
) -> Vec<VideoItem> { ) -> Vec<VideoItem> {
let _ = per_page; let _ = per_page;
let _ = pool; let _ = pool;
let page_num = page.parse::<u8>().unwrap_or(1); let page_num = page.parse::<u8>().unwrap_or(1);
let query_str = query.unwrap_or_default(); let query_str = query.unwrap_or_default();
@@ -225,13 +234,13 @@ impl Provider for PornzogProvider {
// 1. Create a collection of owned data so we don't hold references to `e` // 1. Create a collection of owned data so we don't hold references to `e`
let mut error_reports = Vec::new(); let mut error_reports = Vec::new();
// Iterating through the error chain to collect data into owned Strings // Iterating through the error chain to collect data into owned Strings
for cause in e.iter().skip(1) { for cause in e.iter().skip(1) {
error_reports.push(( error_reports.push((
cause.to_string(), // Title cause.to_string(), // Title
format_error_chain(cause), // Description/Chain format_error_chain(cause), // Description/Chain
format!("caused by: {}", cause) // Message format!("caused by: {}", cause), // Message
)); ));
} }
@@ -245,9 +254,10 @@ impl Provider for PornzogProvider {
file!(), file!(),
line!(), line!(),
module_path!(), module_path!(),
).await; )
.await;
} }
vec![] vec![]
} }
} }

107
src/providers/redtube.rs
View File

@@ -1,5 +1,5 @@
use crate::api::ClientVersion;
use crate::DbPool; use crate::DbPool;
use crate::api::ClientVersion;
use crate::providers::{Provider, report_provider_error}; use crate::providers::{Provider, report_provider_error};
use crate::status::*; use crate::status::*;
use crate::util::cache::VideoCache; use crate::util::cache::VideoCache;
@@ -94,7 +94,7 @@ impl RedtubeProvider {
page: u8, page: u8,
query: &str, query: &str,
sort: &str, sort: &str,
options: ServerOptions options: ServerOptions,
) -> Result<Vec<VideoItem>> { ) -> Result<Vec<VideoItem>> {
let _ = sort; //TODO let _ = sort; //TODO
let search_string = query.to_lowercase().trim().replace(" ", "+"); let search_string = query.to_lowercase().trim().replace(" ", "+");
@@ -146,9 +146,15 @@ impl RedtubeProvider {
let mut items: Vec<VideoItem> = Vec::new(); let mut items: Vec<VideoItem> = Vec::new();
let video_listing_content = html let video_listing_content = html
.split("<script type=\"application/ld+json\">") .split("<script type=\"application/ld+json\">")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("</script>") .split("</script>")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default(); .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default();
let mut videos: Value = match serde_json::from_str(video_listing_content) { let mut videos: Value = match serde_json::from_str(video_listing_content) {
Ok(videos) => videos, Ok(videos) => videos,
Err(e) => { Err(e) => {
@@ -173,7 +179,13 @@ impl RedtubeProvider {
let mut title: String = vid["name"].as_str().unwrap_or("").to_string(); let mut title: String = vid["name"].as_str().unwrap_or("").to_string();
// html decode // html decode
title = decode(title.as_bytes()).to_string().unwrap_or(title); title = decode(title.as_bytes()).to_string().unwrap_or(title);
let id = video_url.split("=").collect::<Vec<&str>>().get(1).copied().unwrap_or_default().to_string(); let id = video_url
.split("=")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.to_string();
let raw_duration = vid["duration"].as_str().unwrap_or("0"); let raw_duration = vid["duration"].as_str().unwrap_or("0");
let duration = raw_duration let duration = raw_duration
.replace("PT", "") .replace("PT", "")
@@ -203,7 +215,12 @@ impl RedtubeProvider {
return vec![]; return vec![];
} }
let mut items: Vec<VideoItem> = Vec::new(); let mut items: Vec<VideoItem> = Vec::new();
let video_listing_content = html.split("videos_grid").collect::<Vec<&str>>().get(1).copied().unwrap_or_default(); let video_listing_content = html
.split("videos_grid")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default();
let videos = video_listing_content let videos = video_listing_content
.split("<li id=\"tags_videos_") .split("<li id=\"tags_videos_")
.collect::<Vec<&str>>()[1..] .collect::<Vec<&str>>()[1..]
@@ -212,43 +229,93 @@ impl RedtubeProvider {
// for (i, c) in vid.split("\n").enumerate() { // for (i, c) in vid.split("\n").enumerate() {
// println!("{}: {}", i, c); // println!("{}: {}", i, c);
// } // }
let id = vid.split("data-video-id=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() let id = vid
.split("data-video-id=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"") .split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(); .to_string();
let video_url = format!("{}/{}", self.url, id); let video_url = format!("{}/{}", self.url, id);
let title = vid.split(" <a title=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() let title = vid
.split(" <a title=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"") .split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.trim() .trim()
.to_string(); .to_string();
let thumb = vid.split("<img").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() let thumb = vid
.split("<img")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split(" data-src=\"") .split(" data-src=\"")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"") .split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(); .to_string();
let raw_duration = vid let raw_duration = vid
.split("<span class=\"video-properties tm_video_duration\">") .split("<span class=\"video-properties tm_video_duration\">")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("</span>") .split("</span>")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.trim() .trim()
.to_string(); .to_string();
let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32; let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
let views_str = vid let views_str = vid
.split("<span class='info-views'>") .split("<span class='info-views'>")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("</span>") .split("</span>")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.trim() .trim()
.to_string(); .to_string();
let views = parse_abbreviated_number(&views_str).unwrap_or(0) as u32; let views = parse_abbreviated_number(&views_str).unwrap_or(0) as u32;
let preview = vid.split("<img").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() let preview = vid
.split("<img")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split(" data-mediabook=\"") .split(" data-mediabook=\"")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"") .split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(); .to_string();
let video_item = let video_item =

247
src/providers/rule34gen.rs
View File

@@ -1,15 +1,15 @@
use crate::api::*;
use crate::status::*;
use crate::util::parse_abbreviated_number;
use crate::DbPool; use crate::DbPool;
use crate::api::*;
use crate::providers::{Provider, report_provider_error}; use crate::providers::{Provider, report_provider_error};
use crate::status::*;
use crate::util::cache::VideoCache; use crate::util::cache::VideoCache;
use crate::util::parse_abbreviated_number;
use crate::util::time::parse_time_to_seconds; use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoItem}; use crate::videos::{ServerOptions, VideoItem};
use async_trait::async_trait;
use error_chain::error_chain; use error_chain::error_chain;
use htmlentity::entity::{ICodedDataTrait, decode}; use htmlentity::entity::{ICodedDataTrait, decode};
use std::vec; use std::vec;
use async_trait::async_trait;
error_chain! { error_chain! {
foreign_links { foreign_links {
@@ -29,49 +29,49 @@ impl Rule34genProvider {
} }
} }
fn build_channel(&self, clientversion: ClientVersion) -> Channel { fn build_channel(&self, clientversion: ClientVersion) -> Channel {
let _ = clientversion; let _ = clientversion;
Channel { Channel {
id: "rule34gen".to_string(), id: "rule34gen".to_string(),
name: "Rule34Gen".to_string(), name: "Rule34Gen".to_string(),
description: "If it exists, here might be an AI generated video of it".to_string(), description: "If it exists, here might be an AI generated video of it".to_string(),
premium: false, premium: false,
favicon: "https://www.google.com/s2/favicons?sz=64&domain=rule34gen.com".to_string(), favicon: "https://www.google.com/s2/favicons?sz=64&domain=rule34gen.com".to_string(),
status: "active".to_string(), status: "active".to_string(),
categories: vec![], categories: vec![],
options: vec![ChannelOption { options: vec![ChannelOption {
id: "sort".to_string(), id: "sort".to_string(),
title: "Sort".to_string(), title: "Sort".to_string(),
description: "Sort the Videos".to_string(), //"Sort the videos by Date or Name.".to_string(), description: "Sort the Videos".to_string(), //"Sort the videos by Date or Name.".to_string(),
systemImage: "list.number".to_string(), systemImage: "list.number".to_string(),
colorName: "blue".to_string(), colorName: "blue".to_string(),
options: vec![ options: vec![
FilterOption { FilterOption {
id: "post_date".to_string(), id: "post_date".to_string(),
title: "Newest".to_string(), title: "Newest".to_string(),
}, },
FilterOption { FilterOption {
id: "video_viewed".to_string(), id: "video_viewed".to_string(),
title: "Most Viewed".to_string(), title: "Most Viewed".to_string(),
}, },
FilterOption { FilterOption {
id: "rating".to_string(), id: "rating".to_string(),
title: "Top Rated".to_string(), title: "Top Rated".to_string(),
}, },
FilterOption { FilterOption {
id: "duration".to_string(), id: "duration".to_string(),
title: "Longest".to_string(), title: "Longest".to_string(),
}, },
FilterOption { FilterOption {
id: "pseudo_random".to_string(), id: "pseudo_random".to_string(),
title: "Random".to_string(), title: "Random".to_string(),
}, },
], ],
multiSelect: false, multiSelect: false,
}], }],
nsfw: true, nsfw: true,
cacheDuration: Some(1800), cacheDuration: Some(1800),
} }
} }
async fn get( async fn get(
@@ -79,9 +79,15 @@ fn build_channel(&self, clientversion: ClientVersion) -> Channel {
cache: VideoCache, cache: VideoCache,
page: u8, page: u8,
sort: &str, sort: &str,
options: ServerOptions options: ServerOptions,
) -> Result<Vec<VideoItem>> { ) -> Result<Vec<VideoItem>> {
let expected_sorts = vec!["post_date", "video_viewed", "rating", "duration", "pseudo_random"]; let expected_sorts = vec![
"post_date",
"video_viewed",
"rating",
"duration",
"pseudo_random",
];
let sort = if expected_sorts.contains(&sort) { let sort = if expected_sorts.contains(&sort) {
sort sort
} else { } else {
@@ -94,7 +100,7 @@ fn build_channel(&self, clientversion: ClientVersion) -> Channel {
} else { } else {
format!("{}/{}/?sort_by={}", self.url, page, sort) format!("{}/{}/?sort_by={}", self.url, page, sort)
}; };
let mut old_items: Vec<VideoItem> = vec![]; let mut old_items: Vec<VideoItem> = vec![];
if !(sort == "pseudo_random") { if !(sort == "pseudo_random") {
old_items = match cache.get(&index) { old_items = match cache.get(&index) {
@@ -116,12 +122,8 @@ fn build_channel(&self, clientversion: ClientVersion) -> Channel {
let text = match requester.get(&url, None).await { let text = match requester.get(&url, None).await {
Ok(text) => text, Ok(text) => text,
Err(e) => { Err(e) => {
report_provider_error( report_provider_error("rule34gen", "get.request", &format!("url={url}; error={e}"))
"rule34gen", .await;
"get.request",
&format!("url={url}; error={e}"),
)
.await;
return Ok(old_items); return Ok(old_items);
} }
}; };
@@ -140,9 +142,15 @@ fn build_channel(&self, clientversion: ClientVersion) -> Channel {
page: u8, page: u8,
query: &str, query: &str,
sort: &str, sort: &str,
options: ServerOptions options: ServerOptions,
) -> Result<Vec<VideoItem>> { ) -> Result<Vec<VideoItem>> {
let expected_sorts = vec!["post_date", "video_viewed", "rating", "duration", "pseudo_random"]; let expected_sorts = vec![
"post_date",
"video_viewed",
"rating",
"duration",
"pseudo_random",
];
let sort = if expected_sorts.contains(&sort) { let sort = if expected_sorts.contains(&sort) {
sort sort
} else { } else {
@@ -154,7 +162,10 @@ fn build_channel(&self, clientversion: ClientVersion) -> Channel {
let url = if page <= 1 { let url = if page <= 1 {
format!("{}/search/{}/?sort_by={}", self.url, search_slug, sort) format!("{}/search/{}/?sort_by={}", self.url, search_slug, sort)
} else { } else {
format!("{}/search/{}/{}/?sort_by={}", self.url, search_slug, page, sort) format!(
"{}/search/{}/{}/?sort_by={}",
self.url, search_slug, page, sort
)
}; };
// Check our Video Cache. If the result is younger than 1 hour, we return it. // Check our Video Cache. If the result is younger than 1 hour, we return it.
@@ -287,7 +298,18 @@ fn build_channel(&self, clientversion: ClientVersion) -> Channel {
} }
return items; return items;
} }
let video_listing_content = html.split("<div class=\"thumbs clearfix\" id=\"custom_list_videos").collect::<Vec<&str>>().get(1).copied().unwrap_or_default().split("<div class=\"pagination\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default().to_string(); let video_listing_content = html
.split("<div class=\"thumbs clearfix\" id=\"custom_list_videos")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<div class=\"pagination\"")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
let raw_videos = video_listing_content let raw_videos = video_listing_content
.split("<div class=\"item thumb video_") .split("<div class=\"item thumb video_")
.collect::<Vec<&str>>()[1..] .collect::<Vec<&str>>()[1..]
@@ -298,42 +320,104 @@ fn build_channel(&self, clientversion: ClientVersion) -> Channel {
// println!("Line {}: {}", index, line); // println!("Line {}: {}", index, line);
// } // }
if video_segment.contains("https://rule34gen.com/images/advertisements"){ if video_segment.contains("https://rule34gen.com/images/advertisements") {
continue; continue;
} }
let mut title = video_segment.split("<div class=\"thumb_title\">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() let mut title = video_segment
.split("<div class=\"thumb_title\">")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<") .split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(); .to_string();
// html decode // html decode
title = decode(title.as_bytes()).to_string().unwrap_or(title); title = decode(title.as_bytes()).to_string().unwrap_or(title);
let id = video_segment.split("https://rule34gen.com/video/").collect::<Vec<&str>>().get(1).copied().unwrap_or_default().split("/").collect::<Vec<&str>>().get(0).copied().unwrap_or_default().to_string(); let id = video_segment
let raw_duration = video_segment.split("<div class=\"time\">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .split("https://rule34gen.com/video/")
.split("<") .collect::<Vec<&str>>()
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .get(1)
.to_string(); .copied()
let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32; .unwrap_or_default()
let views = parse_abbreviated_number(&video_segment .split("/")
.split("<div class=\"views\">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default().split("</svg>").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .collect::<Vec<&str>>()
.split("<") .get(0)
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()).unwrap_or(0); .copied()
//https://rule34gen.com/get_file/47/5e71602b7642f9b997f90c979a368c99b8aad90d89/3942000/3942353/3942353_preview.mp4/ .unwrap_or_default()
//https://rule34gen.com/get_file/47/5e71602b7642f9b997f90c979a368c99b8aad90d89/3942000/3942353/3942353_preview.mp4/
let thumb = video_segment.split("<img class=\"thumb lazy-load\" src=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default().split("data-original=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string(); .to_string();
let url = video_segment.split("<a class=\"th js-open-popup\" href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() let raw_duration = video_segment
.split("<div class=\"time\">")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
let views = parse_abbreviated_number(
&video_segment
.split("<div class=\"views\">")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("</svg>")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default(),
)
.unwrap_or(0);
//https://rule34gen.com/get_file/47/5e71602b7642f9b997f90c979a368c99b8aad90d89/3942000/3942353/3942353_preview.mp4/
//https://rule34gen.com/get_file/47/5e71602b7642f9b997f90c979a368c99b8aad90d89/3942000/3942353/3942353_preview.mp4/
let thumb = video_segment
.split("<img class=\"thumb lazy-load\" src=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("data-original=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"") .split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
let url = video_segment
.split("<a class=\"th js-open-popup\" href=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(); .to_string();
// let preview = video_segment.split("<div class=\"img wrap_image\" data-preview=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() // let preview = video_segment.split("<div class=\"img wrap_image\" data-preview=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
// .split("\"") // .split("\"")
// .collect::<Vec<&str>>().get(0).copied().unwrap_or_default() // .collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
// .to_string(); // .to_string();
let video_item = VideoItem::new( let video_item = VideoItem::new(
id, id,
title, title,
@@ -346,13 +430,10 @@ fn build_channel(&self, clientversion: ClientVersion) -> Channel {
// .preview(preview) // .preview(preview)
; ;
items.push(video_item); items.push(video_item);
} }
return items; return items;
} }
} }
#[async_trait] #[async_trait]
@@ -388,7 +469,7 @@ impl Provider for Rule34genProvider {
} }
} }
} }
fn get_channel(&self, clientversion: ClientVersion) -> Option<crate::status::Channel> { fn get_channel(&self, clientversion: ClientVersion) -> Option<crate::status::Channel> {
Some(self.build_channel(clientversion)) Some(self.build_channel(clientversion))
} }

22
src/providers/rule34video.rs
View File

@@ -1,5 +1,5 @@
use crate::api::ClientVersion;
use crate::DbPool; use crate::DbPool;
use crate::api::ClientVersion;
use crate::providers::Provider; use crate::providers::Provider;
use crate::status::*; use crate::status::*;
use crate::util::cache::VideoCache; use crate::util::cache::VideoCache;
@@ -133,7 +133,15 @@ impl Rule34videoProvider {
let text = requester.get(&url, None).await.unwrap_or_else(|e| { let text = requester.get(&url, None).await.unwrap_or_else(|e| {
eprintln!("Error fetching rule34video URL {}: {}", url, e); eprintln!("Error fetching rule34video URL {}: {}", url, e);
let _ = send_discord_error_report(e.to_string(), None, Some(&url), None, file!(), line!(), module_path!()); let _ = send_discord_error_report(
e.to_string(),
None,
Some(&url),
None,
file!(),
line!(),
module_path!(),
);
"".to_string() "".to_string()
}); });
let video_items = self.get_video_items_from_html(text); let video_items = self.get_video_items_from_html(text);
@@ -197,7 +205,15 @@ impl Rule34videoProvider {
let text = requester.get(&url, None).await.unwrap_or_else(|e| { let text = requester.get(&url, None).await.unwrap_or_else(|e| {
eprintln!("Error fetching rule34video URL {}: {}", url, e); eprintln!("Error fetching rule34video URL {}: {}", url, e);
let _ = send_discord_error_report(e.to_string(), None, Some(&url), None, file!(), line!(), module_path!()); let _ = send_discord_error_report(
e.to_string(),
None,
Some(&url),
None,
file!(),
line!(),
module_path!(),
);
"".to_string() "".to_string()
}); });
let video_items = self.get_video_items_from_html(text); let video_items = self.get_video_items_from_html(text);

16
src/providers/sxyprn.rs
View File

@@ -1,5 +1,5 @@
use crate::api::ClientVersion;
use crate::DbPool; use crate::DbPool;
use crate::api::ClientVersion;
use crate::providers::Provider; use crate::providers::Provider;
use crate::status::*; use crate::status::*;
use crate::util::cache::VideoCache; use crate::util::cache::VideoCache;
@@ -124,7 +124,8 @@ impl SxyprnProvider {
"all" => "all", "all" => "all",
_ => "top", _ => "top",
}; };
let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let url_str = format!( let url_str = format!(
"{}/blog/all/{}.html?fl={}&sm={}", "{}/blog/all/{}.html?fl={}&sm={}",
@@ -175,7 +176,8 @@ impl SxyprnProvider {
file!(), file!(),
line!(), line!(),
module_path!(), module_path!(),
).await; )
.await;
return Ok(old_items); return Ok(old_items);
} }
}; };
@@ -207,7 +209,8 @@ impl SxyprnProvider {
_ => "latest", _ => "latest",
}; };
// Extract needed fields from options at the start // Extract needed fields from options at the start
let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let search_string = query.replace(" ", "-"); let search_string = query.replace(" ", "-");
let url_str = format!( let url_str = format!(
"{}/{}.html?page={}&sm={}", "{}/{}.html?page={}&sm={}",
@@ -249,7 +252,7 @@ impl SxyprnProvider {
{ {
Ok(items) => items, Ok(items) => items,
Err(e) => { Err(e) => {
println!("Error parsing video items: {}", e);// 1. Convert the error to a string immediately println!("Error parsing video items: {}", e); // 1. Convert the error to a string immediately
send_discord_error_report( send_discord_error_report(
e.to_string(), e.to_string(),
Some(format_error_chain(&e)), Some(format_error_chain(&e)),
@@ -258,7 +261,8 @@ impl SxyprnProvider {
file!(), file!(),
line!(), line!(),
module_path!(), module_path!(),
).await; )
.await;
return Ok(old_items); return Ok(old_items);
} }
}; };

205
src/providers/xxdbx.rs
View File

@@ -8,9 +8,9 @@ use crate::videos::{ServerOptions, VideoItem};
use async_trait::async_trait; use async_trait::async_trait;
use error_chain::error_chain; use error_chain::error_chain;
use htmlentity::entity::{ICodedDataTrait, decode}; use htmlentity::entity::{ICodedDataTrait, decode};
use regex::Regex;
use std::sync::{Arc, RwLock}; use std::sync::{Arc, RwLock};
use std::vec; use std::vec;
use regex::Regex;
error_chain! { error_chain! {
foreign_links { foreign_links {
@@ -56,26 +56,24 @@ impl XxdbxProvider {
favicon: "https://www.google.com/s2/favicons?sz=64&domain=xxdbx.com".to_string(), favicon: "https://www.google.com/s2/favicons?sz=64&domain=xxdbx.com".to_string(),
status: "active".to_string(), status: "active".to_string(),
categories: vec![], categories: vec![],
options: vec![ options: vec![ChannelOption {
ChannelOption { id: "sort".to_string(),
id: "sort".to_string(), title: "Sort".to_string(),
title: "Sort".to_string(), description: "Sort the Videos".to_string(),
description: "Sort the Videos".to_string(), systemImage: "list.number".to_string(),
systemImage: "list.number".to_string(), colorName: "blue".to_string(),
colorName: "blue".to_string(), options: vec![
options: vec![ FilterOption {
FilterOption { id: "new".into(),
id: "new".into(), title: "New".into(),
title: "New".into(), },
}, FilterOption {
FilterOption { id: "popular".into(),
id: "popular".into(), title: "Most Popular".into(),
title: "Most Popular".into(), },
}, ],
], multiSelect: false,
multiSelect: false, }],
},
],
nsfw: true, nsfw: true,
cacheDuration: Some(1800), cacheDuration: Some(1800),
} }
@@ -92,10 +90,7 @@ impl XxdbxProvider {
"popular" => "most-popular".to_string(), "popular" => "most-popular".to_string(),
_ => "".to_string(), _ => "".to_string(),
}; };
let video_url = format!( let video_url = format!("{}/{}?page={}", self.url, sort_string, page);
"{}/{}?page={}",
self.url, sort_string, page
);
let old_items = match cache.get(&video_url) { let old_items = match cache.get(&video_url) {
Some((time, items)) => { Some((time, items)) => {
if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
@@ -109,7 +104,8 @@ impl XxdbxProvider {
} }
}; };
let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let text = match requester.get(&video_url, None).await { let text = match requester.get(&video_url, None).await {
Ok(text) => text, Ok(text) => text,
Err(e) => { Err(e) => {
@@ -163,7 +159,7 @@ impl XxdbxProvider {
.unwrap_or(false) .unwrap_or(false)
{ {
search_type = "stars"; search_type = "stars";
} else if is_valid_date(&search_string){ } else if is_valid_date(&search_string) {
search_type = "dates"; search_type = "dates";
} }
@@ -186,7 +182,8 @@ impl XxdbxProvider {
} }
}; };
let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester"); let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let text = match requester.get(&video_url, None).await { let text = match requester.get(&video_url, None).await {
Ok(text) => text, Ok(text) => text,
@@ -210,9 +207,17 @@ impl XxdbxProvider {
return vec![]; return vec![];
} }
let mut items: Vec<VideoItem> = Vec::new(); let mut items: Vec<VideoItem> = Vec::new();
let raw_videos = html.split("</article>").collect::<Vec<&str>>().get(0).copied().unwrap_or_default() let raw_videos = html
.split("</article>")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.split("<div class=\"vids\">") .split("<div class=\"vids\">")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<div class=\"v\">") .split("<div class=\"v\">")
.collect::<Vec<&str>>()[1..] .collect::<Vec<&str>>()[1..]
.to_vec(); .to_vec();
@@ -221,47 +226,126 @@ impl XxdbxProvider {
// for (index, line) in vid.iter().enumerate() { // for (index, line) in vid.iter().enumerate() {
// println!("Line {}: {}\n\n", index, line); // println!("Line {}: {}\n\n", index, line);
// } // }
let video_url: String = format!("{}{}", self.url, video_segment.split("<a href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() let video_url: String = format!(
.split("\"") "{}{}",
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() self.url,
.to_string()); video_segment
.split("<a href=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string()
);
let mut title = video_segment let mut title = video_segment
.split("<div class=\"v_title\">") .split("<div class=\"v_title\">")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<") .split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.trim() .trim()
.to_string(); .to_string();
// html decode // html decode
title = decode(title.as_bytes()).to_string().unwrap_or(title); title = decode(title.as_bytes()).to_string().unwrap_or(title);
let id = video_url.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string(); let id = video_url
.split("/")
.collect::<Vec<&str>>()
.get(4)
.copied()
.unwrap_or_default()
.to_string();
let thumb = format!("https:{}", video_segment.split("<img ").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() let thumb = format!(
.split("src=\"").collect::<Vec<&str>>().last().copied().unwrap_or_default() "https:{}",
.split("\"") video_segment
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .split("<img ")
.to_string()); .collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("src=\"")
.collect::<Vec<&str>>()
.last()
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string()
);
let raw_duration = video_segment let raw_duration = video_segment
.split("<div class=\"v_dur\">") .split("<div class=\"v_dur\">")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<") .split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(); .to_string();
let duration = parse_time_to_seconds(raw_duration.as_str()).unwrap_or(0) as u32; let duration = parse_time_to_seconds(raw_duration.as_str()).unwrap_or(0) as u32;
let preview = format!("https:{}",video_segment let preview = format!(
.split("data-preview=\"") "https:{}",
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default() video_segment
.split("\"") .split("data-preview=\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.to_string()); .get(1)
let tags = video_segment.split("<div class=\"v_tags\">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .copied()
.split("</div>").collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string()
);
let tags = video_segment
.split("<div class=\"v_tags\">")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("</div>")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.split("<a href=\"") .split("<a href=\"")
.collect::<Vec<&str>>()[1..] .collect::<Vec<&str>>()[1..]
.into_iter().map(|s| s.split("\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default().replace("%20"," ").to_string()).collect::<Vec<String>>(); .into_iter()
.map(|s| {
s.split("\"")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.replace("%20", " ")
.to_string()
})
.collect::<Vec<String>>();
for tag in tags.clone() { for tag in tags.clone() {
let shorted_tag = tag.split("/").collect::<Vec<&str>>().get(2).copied().unwrap_or_default().to_string(); let shorted_tag = tag
.split("/")
.collect::<Vec<&str>>()
.get(2)
.copied()
.unwrap_or_default()
.to_string();
if tag.contains("channels") if tag.contains("channels")
&& self && self
.channels .channels
@@ -293,7 +377,18 @@ impl XxdbxProvider {
thumb, thumb,
duration, duration,
) )
.tags(tags.into_iter().map(|s| s.split("/").collect::<Vec<&str>>().last().copied().unwrap_or_default().to_string()).collect::<Vec<String>>()) .tags(
tags.into_iter()
.map(|s| {
s.split("/")
.collect::<Vec<&str>>()
.last()
.copied()
.unwrap_or_default()
.to_string()
})
.collect::<Vec<String>>(),
)
.preview(preview); .preview(preview);
items.push(video_item); items.push(video_item);
} }

90
src/providers/xxthots.rs
View File

@@ -1,5 +1,5 @@
use crate::api::ClientVersion;
use crate::DbPool; use crate::DbPool;
use crate::api::ClientVersion;
use crate::providers::Provider; use crate::providers::Provider;
use crate::status::*; use crate::status::*;
use crate::util::cache::VideoCache; use crate::util::cache::VideoCache;
@@ -72,7 +72,11 @@ impl XxthotsProvider {
options: ServerOptions, options: ServerOptions,
) -> Result<Vec<VideoItem>> { ) -> Result<Vec<VideoItem>> {
let (sort_path, list_str, sort_by) = match sort { let (sort_path, list_str, sort_by) = match sort {
"popular" => ("/most-popular/", "list_videos_common_videos_list", "video_viewed"), "popular" => (
"/most-popular/",
"list_videos_common_videos_list",
"video_viewed",
),
"top-rated" => ("/top-rated/", "list_videos_common_videos_list", "rating"), "top-rated" => ("/top-rated/", "list_videos_common_videos_list", "rating"),
_ => ( _ => (
"/latest-updates/", "/latest-updates/",
@@ -194,7 +198,10 @@ impl XxthotsProvider {
let mut items: Vec<VideoItem> = Vec::new(); let mut items: Vec<VideoItem> = Vec::new();
let raw_videos: Vec<&str> = html let raw_videos: Vec<&str> = html
.split("<div class=\"pagination\"") .split("<div class=\"pagination\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.split("<div class=\"thumb thumb_rel item \">") .split("<div class=\"thumb thumb_rel item \">")
.skip(1) .skip(1)
.collect(); .collect();
@@ -203,41 +210,87 @@ impl XxthotsProvider {
// for (index, line) in vid.iter().enumerate() { // for (index, line) in vid.iter().enumerate() {
// println!("Line {}: {}", index, line); // println!("Line {}: {}", index, line);
// } // }
let video_url: String = video_segment.split("<a href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() let video_url: String = video_segment
.split("<a href=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"") .split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(); .to_string();
let mut title = video_segment.split("\" title=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() let mut title = video_segment
.split("\" title=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"") .split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(); .to_string();
// html decode // html decode
title = decode(title.as_bytes()).to_string().unwrap_or(title); title = decode(title.as_bytes()).to_string().unwrap_or(title);
let id = video_url.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string(); let id = video_url
.split("/")
.collect::<Vec<&str>>()
.get(4)
.copied()
.unwrap_or_default()
.to_string();
let raw_duration = video_segment let raw_duration = video_segment
.split("<div class=\"time\">") .split("<div class=\"time\">")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<") .split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(); .to_string();
let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32; let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
let thumb = video_segment let thumb = video_segment
.split("<img class=\"lazy-load") .split("<img class=\"lazy-load")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("data-original=\"") .split("data-original=\"")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"") .split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(); .to_string();
let views_part = video_segment let views_part = video_segment
.split("svg-icon icon-eye") .split("svg-icon icon-eye")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("</i>") .split("</i>")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<") .split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(); .to_string();
let views = parse_abbreviated_number(&views_part).unwrap_or(0) as u32; let views = parse_abbreviated_number(&views_part).unwrap_or(0) as u32;
@@ -279,7 +332,10 @@ impl Provider for XxthotsProvider {
self.get(cache, page.parse::<u8>().unwrap_or(1), &sort, options) self.get(cache, page.parse::<u8>().unwrap_or(1), &sort, options)
.await .await
} }
_ => self.get(cache, page.parse::<u8>().unwrap_or(1), &sort, options).await, _ => {
self.get(cache, page.parse::<u8>().unwrap_or(1), &sort, options)
.await
}
}; };
match videos { match videos {
Ok(v) => v, Ok(v) => v,

127
src/providers/youjizz.rs
View File

@@ -1,15 +1,15 @@
use crate::api::ClientVersion;
use crate::util::parse_abbreviated_number;
use crate::DbPool; use crate::DbPool;
use crate::api::ClientVersion;
use crate::providers::Provider; use crate::providers::Provider;
use crate::status::*; use crate::status::*;
use crate::util::cache::VideoCache; use crate::util::cache::VideoCache;
use crate::util::parse_abbreviated_number;
use crate::util::time::parse_time_to_seconds; use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoItem}; use crate::videos::{ServerOptions, VideoItem};
use async_trait::async_trait;
use error_chain::error_chain; use error_chain::error_chain;
use htmlentity::entity::{ICodedDataTrait, decode}; use htmlentity::entity::{ICodedDataTrait, decode};
use std::vec; use std::vec;
use async_trait::async_trait;
error_chain! { error_chain! {
foreign_links { foreign_links {
@@ -141,7 +141,12 @@ impl YoujizzProvider {
query: &str, query: &str,
options: ServerOptions, options: ServerOptions,
) -> Result<Vec<VideoItem>> { ) -> Result<Vec<VideoItem>> {
let video_url = format!("{}/search/{}-{}.html", self.url, query.to_lowercase().trim(), page); let video_url = format!(
"{}/search/{}-{}.html",
self.url,
query.to_lowercase().trim(),
page
);
// Check our Video Cache. If the result is younger than 1 hour, we return it. // Check our Video Cache. If the result is younger than 1 hour, we return it.
let old_items = match cache.get(&video_url) { let old_items = match cache.get(&video_url) {
Some((time, items)) => { Some((time, items)) => {
@@ -187,7 +192,12 @@ impl YoujizzProvider {
return vec![]; return vec![];
} }
let mut items: Vec<VideoItem> = Vec::new(); let mut items: Vec<VideoItem> = Vec::new();
let raw_videos = html.split("class=\"mobile-only\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default() let raw_videos = html
.split("class=\"mobile-only\"")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.split("class=\"default video-item\"") .split("class=\"default video-item\"")
.collect::<Vec<&str>>()[1..] .collect::<Vec<&str>>()[1..]
.to_vec(); .to_vec();
@@ -200,32 +210,98 @@ impl YoujizzProvider {
// println!("Skipping video segment due to placeholder thumbnail"); // println!("Skipping video segment due to placeholder thumbnail");
// continue; // continue;
// } // }
let video_url: String = format!("{}{}",self.url, video_segment.split("href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() let video_url: String = format!(
"{}{}",
self.url,
video_segment
.split("href=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"") .split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default().to_string()); .collect::<Vec<&str>>()
let mut title = video_segment.split("class=\"video-title\">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .get(0)
.split(">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .copied()
.unwrap_or_default()
.to_string()
);
let mut title = video_segment
.split("class=\"video-title\">")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split(">")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<") .split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(); .to_string();
// html decode // html decode
title = decode(title.as_bytes()).to_string().unwrap_or(title); title = decode(title.as_bytes()).to_string().unwrap_or(title);
let id = video_url.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string(); let id = video_url
.split("/")
.collect::<Vec<&str>>()
.get(4)
.copied()
.unwrap_or_default()
.to_string();
let thumb = format!("https:{}",video_segment.split("<img ").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() let thumb = format!(
.split("data-original=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() "https:{}",
.split("\"") video_segment
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .split("<img ")
.to_string()); .collect::<Vec<&str>>()
let raw_duration = video_segment.split("fa fa-clock-o\"></i>&nbsp;").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() .get(1)
.copied()
.unwrap_or_default()
.split("data-original=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string()
);
let raw_duration = video_segment
.split("fa fa-clock-o\"></i>&nbsp;")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<") .split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(); .to_string();
let duration = parse_time_to_seconds(raw_duration.as_str()).unwrap_or(0) as u32; let duration = parse_time_to_seconds(raw_duration.as_str()).unwrap_or(0) as u32;
let views = parse_abbreviated_number(video_segment.split("format-views\">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default() let views = parse_abbreviated_number(
.split("<") video_segment
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default() .split("format-views\">")
.to_string().as_str()).unwrap_or(0) as u32; .collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string()
.as_str(),
)
.unwrap_or(0) as u32;
let video_item = VideoItem::new( let video_item = VideoItem::new(
id, id,
@@ -235,14 +311,11 @@ impl YoujizzProvider {
thumb, thumb,
duration, duration,
) )
.views(views) .views(views);
;
items.push(video_item); items.push(video_item);
} }
return items; return items;
} }
} }
#[async_trait] #[async_trait]
@@ -261,7 +334,7 @@ impl Provider for YoujizzProvider {
let _ = pool; let _ = pool;
let videos: std::result::Result<Vec<VideoItem>, Error> = match query { let videos: std::result::Result<Vec<VideoItem>, Error> = match query {
Some(q) => { Some(q) => {
self.query(cache, page.parse::<u8>().unwrap_or(1), &q,options) self.query(cache, page.parse::<u8>().unwrap_or(1), &q, options)
.await .await
} }
None => { None => {

2
src/proxies/hanimecdn.rs
View File

@@ -1,8 +1,8 @@
use ntex::http::header::{CONTENT_LENGTH, CONTENT_TYPE};
use ntex::{ use ntex::{
http::Response, http::Response,
web::{self, HttpRequest, error}, web::{self, HttpRequest, error},
}; };
use ntex::http::header::{CONTENT_LENGTH, CONTENT_TYPE};
use crate::util::requester::Requester; use crate::util::requester::Requester;

51
src/proxies/hqpornerthumb.rs Normal file
View File

@@ -0,0 +1,51 @@
use ntex::http::header::{CONTENT_LENGTH, CONTENT_TYPE};
use ntex::{
http::Response,
web::{self, HttpRequest, error},
};
use crate::util::requester::Requester;
pub async fn get_image(
req: HttpRequest,
requester: web::types::State<Requester>,
) -> Result<impl web::Responder, web::Error> {
let endpoint = req.match_info().query("endpoint").to_string();
let image_url = if endpoint.starts_with("http://") || endpoint.starts_with("https://") {
endpoint
} else {
format!("https://{}", endpoint.trim_start_matches('/'))
};
let upstream = match requester
.get_ref()
.clone()
.get_raw_with_headers(
image_url.as_str(),
vec![("Referer".to_string(), "https://hqporner.com/".to_string())],
)
.await
{
Ok(response) => response,
Err(_) => return Ok(web::HttpResponse::NotFound().finish()),
};
let status = upstream.status();
let headers = upstream.headers().clone();
let bytes = upstream.bytes().await.map_err(error::ErrorBadGateway)?;
let mut resp = Response::build(status);
if let Some(ct) = headers.get(CONTENT_TYPE) {
if let Ok(ct_str) = ct.to_str() {
resp.set_header(CONTENT_TYPE, ct_str);
}
}
if let Some(cl) = headers.get(CONTENT_LENGTH) {
if let Ok(cl_str) = cl.to_str() {
resp.set_header(CONTENT_LENGTH, cl_str);
}
}
Ok(resp.body(bytes.to_vec()))
}

28
src/proxies/javtiful.rs
View File

@@ -3,15 +3,12 @@ use wreq::Version;
use crate::util::requester::Requester; use crate::util::requester::Requester;
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct JavtifulProxy { pub struct JavtifulProxy {}
}
impl JavtifulProxy { impl JavtifulProxy {
pub fn new() -> Self { pub fn new() -> Self {
JavtifulProxy { JavtifulProxy {}
}
} }
pub async fn get_video_url( pub async fn get_video_url(
@@ -25,18 +22,15 @@ impl JavtifulProxy {
if text.is_empty() { if text.is_empty() {
return "".to_string(); return "".to_string();
} }
let video_id = url let video_id = url.split('/').nth(4).unwrap_or("").to_string();
.split('/')
.nth(4)
.unwrap_or("")
.to_string();
let token = text.split("data-csrf-token=\"") let token = text
.split("data-csrf-token=\"")
.nth(1) .nth(1)
.and_then(|s| s.split('"').next()) .and_then(|s| s.split('"').next())
.unwrap_or("") .unwrap_or("")
.to_string(); .to_string();
let form = wreq::multipart::Form::new() let form = wreq::multipart::Form::new()
.text("video_id", video_id.clone()) .text("video_id", video_id.clone())
.text("pid_c", "".to_string()) .text("pid_c", "".to_string())
@@ -54,11 +48,13 @@ impl JavtifulProxy {
Err(_) => return "".to_string(), Err(_) => return "".to_string(),
}; };
let text = resp.text().await.unwrap_or_default(); let text = resp.text().await.unwrap_or_default();
let json: serde_json::Value = serde_json::from_str(&text).unwrap_or(serde_json::Value::Null); let json: serde_json::Value =
let video_url = json.get("playlists") serde_json::from_str(&text).unwrap_or(serde_json::Value::Null);
let video_url = json
.get("playlists")
.map(|v| v.to_string().replace("\"", "")) .map(|v| v.to_string().replace("\"", ""))
.unwrap_or_default(); .unwrap_or_default();
return video_url; return video_url;
} }
} }

18
src/proxies/mod.rs
View File

@@ -2,9 +2,10 @@ use ntex::web;
use crate::{proxies::sxyprn::SxyprnProxy, util::requester::Requester}; use crate::{proxies::sxyprn::SxyprnProxy, util::requester::Requester};
pub mod sxyprn;
pub mod hanimecdn; pub mod hanimecdn;
pub mod hqpornerthumb;
pub mod javtiful; pub mod javtiful;
pub mod sxyprn;
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub enum AnyProxy { pub enum AnyProxy {
@@ -13,23 +14,14 @@ pub enum AnyProxy {
} }
pub trait Proxy { pub trait Proxy {
async fn get_video_url( async fn get_video_url(&self, url: String, requester: web::types::State<Requester>) -> String;
&self,
url: String,
requester: web::types::State<Requester>,
) -> String;
} }
impl Proxy for AnyProxy { impl Proxy for AnyProxy {
async fn get_video_url( async fn get_video_url(&self, url: String, requester: web::types::State<Requester>) -> String {
&self,
url: String,
requester: web::types::State<Requester>,
) -> String {
match self { match self {
AnyProxy::Sxyprn(p) => p.get_video_url(url, requester).await, AnyProxy::Sxyprn(p) => p.get_video_url(url, requester).await,
AnyProxy::Javtiful(p) => p.get_video_url(url, requester).await, AnyProxy::Javtiful(p) => p.get_video_url(url, requester).await,
} }
} }
} }

41
src/proxies/sxyprn.rs
View File

@@ -1,4 +1,4 @@
use base64::{engine::general_purpose, Engine as _}; use base64::{Engine as _, engine::general_purpose};
use ntex::web; use ntex::web;
use crate::util::requester::Requester; use crate::util::requester::Requester;
@@ -24,13 +24,11 @@ fn boo(sum1: u32, sum2: u32) -> String {
} }
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct SxyprnProxy { pub struct SxyprnProxy {}
}
impl SxyprnProxy { impl SxyprnProxy {
pub fn new() -> Self { pub fn new() -> Self {
SxyprnProxy { SxyprnProxy {}
}
} }
pub async fn get_video_url( pub async fn get_video_url(
@@ -45,16 +43,23 @@ impl SxyprnProxy {
return "".to_string(); return "".to_string();
} }
let data_string = text.split("data-vnfo='").collect::<Vec<&str>>()[1] let data_string = text.split("data-vnfo='").collect::<Vec<&str>>()[1]
.split("\":\"").collect::<Vec<&str>>()[1] .split("\":\"")
.split("\"}").collect::<Vec<&str>>()[0].replace("\\",""); .collect::<Vec<&str>>()[1]
.split("\"}")
.collect::<Vec<&str>>()[0]
.replace("\\", "");
//println!("src: {}",data_string); //println!("src: {}",data_string);
let mut tmp = data_string let mut tmp = data_string
.split("/") .split("/")
.map(|s| s.to_string()) .map(|s| s.to_string())
.collect::<Vec<String>>(); .collect::<Vec<String>>();
//println!("tmp: {:?}",tmp); //println!("tmp: {:?}",tmp);
tmp[1] = format!("{}8/{}", tmp[1], boo(ssut51(tmp[6].as_str()), ssut51(tmp[7].as_str()))); tmp[1] = format!(
"{}8/{}",
tmp[1],
boo(ssut51(tmp[6].as_str()), ssut51(tmp[7].as_str()))
);
//println!("tmp[1]: {:?}",tmp[1]); //println!("tmp[1]: {:?}",tmp[1]);
//preda //preda
tmp[5] = format!( tmp[5] = format!(
@@ -62,17 +67,25 @@ impl SxyprnProxy {
tmp[5].parse::<u32>().unwrap() - ssut51(tmp[6].as_str()) - ssut51(tmp[7].as_str()) tmp[5].parse::<u32>().unwrap() - ssut51(tmp[6].as_str()) - ssut51(tmp[7].as_str())
); );
//println!("tmp: {:?}",tmp); //println!("tmp: {:?}",tmp);
let sxyprn_video_url = format!("https://sxyprn.com{}",tmp.join("/")); let sxyprn_video_url = format!("https://sxyprn.com{}", tmp.join("/"));
let response = requester.get_raw(&sxyprn_video_url).await; let response = requester.get_raw(&sxyprn_video_url).await;
match response { match response {
Ok(resp) => { Ok(resp) => {
return format!("https:{}", resp.headers().get("Location").unwrap().to_str().unwrap_or("").to_string()); return format!(
}, "https:{}",
resp.headers()
.get("Location")
.unwrap()
.to_str()
.unwrap_or("")
.to_string()
);
}
Err(e) => { Err(e) => {
println!("Error fetching video URL: {}", e); println!("Error fetching video URL: {}", e);
} }
} }
return "".to_string(); return "".to_string();
} }
} }

23
src/proxy.rs
View File

@@ -2,8 +2,8 @@ use ntex::web::{self, HttpRequest};
use crate::proxies::javtiful::JavtifulProxy; use crate::proxies::javtiful::JavtifulProxy;
use crate::proxies::sxyprn::SxyprnProxy; use crate::proxies::sxyprn::SxyprnProxy;
use crate::util::requester::Requester;
use crate::proxies::*; use crate::proxies::*;
use crate::util::requester::Requester;
pub fn config(cfg: &mut web::ServiceConfig) { pub fn config(cfg: &mut web::ServiceConfig) {
cfg.service( cfg.service(
@@ -21,21 +21,26 @@ pub fn config(cfg: &mut web::ServiceConfig) {
.route(web::post().to(crate::proxies::hanimecdn::get_image)) .route(web::post().to(crate::proxies::hanimecdn::get_image))
.route(web::get().to(crate::proxies::hanimecdn::get_image)), .route(web::get().to(crate::proxies::hanimecdn::get_image)),
) )
; .service(
web::resource("/hqporner-thumb/{endpoint}*")
.route(web::post().to(crate::proxies::hqpornerthumb::get_image))
.route(web::get().to(crate::proxies::hqpornerthumb::get_image)),
);
} }
async fn proxy2redirect(
async fn proxy2redirect(req: HttpRequest, req: HttpRequest,
requester: web::types::State<Requester>,) -> Result<impl web::Responder, web::Error> { requester: web::types::State<Requester>,
) -> Result<impl web::Responder, web::Error> {
let proxy = get_proxy(req.uri().to_string().split("/").collect::<Vec<&str>>()[2]).unwrap(); let proxy = get_proxy(req.uri().to_string().split("/").collect::<Vec<&str>>()[2]).unwrap();
let endpoint = req.match_info().query("endpoint").to_string(); let endpoint = req.match_info().query("endpoint").to_string();
let video_url = match proxy.get_video_url(endpoint, requester).await{ let video_url = match proxy.get_video_url(endpoint, requester).await {
url if url != "" => url, url if url != "" => url,
_ => "Error".to_string(), _ => "Error".to_string(),
}; };
Ok(web::HttpResponse::Found() Ok(web::HttpResponse::Found()
.header("Location", video_url) .header("Location", video_url)
.finish()) .finish())
} }
fn get_proxy(proxy: &str) -> Option<AnyProxy> { fn get_proxy(proxy: &str) -> Option<AnyProxy> {
@@ -44,4 +49,4 @@ fn get_proxy(proxy: &str) -> Option<AnyProxy> {
"javtiful" => Some(AnyProxy::Javtiful(JavtifulProxy::new())), "javtiful" => Some(AnyProxy::Javtiful(JavtifulProxy::new())),
_ => None, _ => None,
} }
} }

18
src/status.rs
View File

@@ -24,19 +24,19 @@ pub struct Channel {
#[derive(serde::Serialize)] #[derive(serde::Serialize)]
pub struct ChannelOption { pub struct ChannelOption {
pub id: String, //"channels", pub id: String, //"channels",
pub title: String, //"Sites", pub title: String, //"Sites",
pub description: String, //"Websites included in search results.", pub description: String, //"Websites included in search results.",
pub systemImage: String, //"network", pub systemImage: String, //"network",
pub colorName: String, //"purple", pub colorName: String, //"purple",
pub options: Vec<FilterOption>, //[], pub options: Vec<FilterOption>, //[],
pub multiSelect: bool, //true pub multiSelect: bool, //true
} }
#[derive(serde::Serialize, Debug, Clone)] #[derive(serde::Serialize, Debug, Clone)]
pub struct FilterOption{ pub struct FilterOption {
pub id: String, //"sort", pub id: String, //"sort",
pub title: String, //"Sort", pub title: String, //"Sort",
} }
#[derive(serde::Serialize)] #[derive(serde::Serialize)]

2
src/util/cache.rs
View File

@@ -45,7 +45,7 @@ impl VideoCache {
cache.remove(key); cache.remove(key);
} }
} }
pub fn entries(&self) -> Option<Vec<(String, (SystemTime, Vec<VideoItem>))>> { pub fn entries(&self) -> Option<Vec<(String, (SystemTime, Vec<VideoItem>))>> {
if let Ok(cache) = self.cache.lock() { if let Ok(cache) = self.cache.lock() {
// Return a cloned vector of the cache entries // Return a cloned vector of the cache entries

16
src/util/discord.rs
View File

@@ -1,10 +1,10 @@
use std::error::Error; use crate::util::requester;
use std::fmt::Write as _;
use std::time::{SystemTime, UNIX_EPOCH};
use dashmap::DashMap; use dashmap::DashMap;
use once_cell::sync::Lazy; use once_cell::sync::Lazy;
use serde_json::json; use serde_json::json;
use crate::util::requester; use std::error::Error;
use std::fmt::Write as _;
use std::time::{SystemTime, UNIX_EPOCH};
// Global cache: Map<ErrorSignature, LastSentTimestamp> // Global cache: Map<ErrorSignature, LastSentTimestamp>
static ERROR_CACHE: Lazy<DashMap<String, u64>> = Lazy::new(DashMap::new); static ERROR_CACHE: Lazy<DashMap<String, u64>> = Lazy::new(DashMap::new);
@@ -42,11 +42,11 @@ pub async fn send_discord_error_report(
if let Some(_) = ERROR_CACHE.get(&error_signature) { if let Some(_) = ERROR_CACHE.get(&error_signature) {
// if now - *last_sent < COOLDOWN_SECONDS { // if now - *last_sent < COOLDOWN_SECONDS {
// Error is still in cooldown, skip sending // Error is still in cooldown, skip sending
return; return;
// } // }
} }
// Update the cache with the current timestamp // Update the cache with the current timestamp
ERROR_CACHE.insert(error_signature, now); ERROR_CACHE.insert(error_signature, now);
// --------------------------- // ---------------------------
@@ -104,4 +104,4 @@ pub async fn send_discord_error_report(
let mut requester = requester::Requester::new(); let mut requester = requester::Requester::new();
let _ = requester.post_json(&webhook_url, &payload, vec![]).await; let _ = requester.post_json(&webhook_url, &payload, vec![]).await;
} }

9
src/util/flaresolverr.rs
View File

@@ -57,10 +57,7 @@ pub struct Flaresolverr {
impl Flaresolverr { impl Flaresolverr {
pub fn new(url: String) -> Self { pub fn new(url: String) -> Self {
Self { Self { url, proxy: false }
url,
proxy: false,
}
} }
pub fn set_proxy(&mut self, proxy: bool) { pub fn set_proxy(&mut self, proxy: bool) {
@@ -71,9 +68,7 @@ impl Flaresolverr {
&self, &self,
request: FlareSolverrRequest, request: FlareSolverrRequest,
) -> Result<FlareSolverrResponse, Box<dyn std::error::Error>> { ) -> Result<FlareSolverrResponse, Box<dyn std::error::Error>> {
let client = Client::builder() let client = Client::builder().emulation(Emulation::Firefox136).build()?;
.emulation(Emulation::Firefox136)
.build()?;
let mut req = client let mut req = client
.post(&self.url) .post(&self.url)

11
src/util/mod.rs
View File

@@ -1,9 +1,9 @@
pub mod time;
pub mod flaresolverr;
pub mod cache; pub mod cache;
pub mod requester;
pub mod discord; pub mod discord;
pub mod flaresolverr;
pub mod proxy; pub mod proxy;
pub mod requester;
pub mod time;
pub fn parse_abbreviated_number(s: &str) -> Option<u32> { pub fn parse_abbreviated_number(s: &str) -> Option<u32> {
let s = s.trim(); let s = s.trim();
@@ -20,7 +20,10 @@ pub fn parse_abbreviated_number(s: &str) -> Option<u32> {
"" => 1.0, "" => 1.0,
_ => return None, _ => return None,
}; };
num_part.parse::<f64>().ok().map(|n| (n * multiplier) as u32) num_part
.parse::<f64>()
.ok()
.map(|n| (n * multiplier) as u32)
} }
pub fn interleave<T: Clone>(lists: &[Vec<T>]) -> Vec<T> { pub fn interleave<T: Clone>(lists: &[Vec<T>]) -> Vec<T> {

15
src/util/proxy.rs
View File

@@ -1,6 +1,6 @@
use std::fmt; use std::fmt;
use std::sync::Arc;
use std::str::FromStr; use std::str::FromStr;
use std::sync::Arc;
use serde::Serialize; use serde::Serialize;
use tokio::sync::{OnceCell, RwLock}; use tokio::sync::{OnceCell, RwLock};
@@ -71,11 +71,7 @@ impl fmt::Display for Proxy {
"{}://{}@{}:{}", "{}://{}@{}:{}",
self.protocol, username, self.host, self.port self.protocol, username, self.host, self.port
), ),
(None, Some(_)) => write!( (None, Some(_)) => write!(f, "{}://{}:{}", self.protocol, self.host, self.port),
f,
"{}://{}:{}",
self.protocol, self.host, self.port
),
(None, None) => write!(f, "{}://{}:{}", self.protocol, self.host, self.port), (None, None) => write!(f, "{}://{}:{}", self.protocol, self.host, self.port),
} }
} }
@@ -224,8 +220,7 @@ pub fn init_all_proxies_background(requester: Requester) {
for list in PROXY_LIST { for list in PROXY_LIST {
let proxy_cache = proxy_cache.clone(); let proxy_cache = proxy_cache.clone();
let mut requester = requester.clone(); let mut requester = requester.clone();
tasks tasks.spawn(async move { fetch_proxies(&mut requester, list, proxy_cache).await });
.spawn(async move { fetch_proxies(&mut requester, list, proxy_cache).await });
} }
while let Some(result) = tasks.join_next().await { while let Some(result) = tasks.join_next().await {
@@ -278,9 +273,7 @@ impl FromStr for Proxy {
.ok_or_else(|| ProxyParseError::new("proxy url is missing host"))? .ok_or_else(|| ProxyParseError::new("proxy url is missing host"))?
.to_string(); .to_string();
let port = url let port = url.port().unwrap_or(80);
.port()
.unwrap_or(80);
Ok(Proxy { Ok(Proxy {
protocol: url.scheme().to_string(), protocol: url.scheme().to_string(),

24
src/util/requester.rs
View File

@@ -1,17 +1,17 @@
use serde::Serialize; use serde::Serialize;
use wreq::multipart::Form;
use std::env; use std::env;
use wreq::Client; use wreq::Client;
use wreq::Proxy; use wreq::Proxy;
use wreq::Response; use wreq::Response;
use wreq::Version; use wreq::Version;
use wreq::header::HeaderValue; use wreq::header::HeaderValue;
use wreq::multipart::Form;
use wreq::redirect::Policy; use wreq::redirect::Policy;
use wreq_util::Emulation; use wreq_util::Emulation;
use crate::util::proxy;
use crate::util::flaresolverr::FlareSolverrRequest; use crate::util::flaresolverr::FlareSolverrRequest;
use crate::util::flaresolverr::Flaresolverr; use crate::util::flaresolverr::Flaresolverr;
use crate::util::proxy;
// A Send + Sync error type for all async paths // A Send + Sync error type for all async paths
type AnyErr = Box<dyn std::error::Error + Send + Sync + 'static>; type AnyErr = Box<dyn std::error::Error + Send + Sync + 'static>;
@@ -99,7 +99,7 @@ impl Requester {
request.send().await request.send().await
} }
pub async fn post_json<S>( pub async fn post_json<S>(
&mut self, &mut self,
url: &str, url: &str,
data: &S, data: &S,
@@ -131,7 +131,11 @@ pub async fn post_json<S>(
data: &str, data: &str,
headers: Vec<(&str, &str)>, headers: Vec<(&str, &str)>,
) -> Result<Response, wreq::Error> { ) -> Result<Response, wreq::Error> {
let mut request = self.client.post(url).version(Version::HTTP_11).body(data.to_string()); let mut request = self
.client
.post(url)
.version(Version::HTTP_11)
.body(data.to_string());
// Set custom headers // Set custom headers
for (key, value) in headers.iter() { for (key, value) in headers.iter() {
@@ -154,8 +158,7 @@ pub async fn post_json<S>(
form: Form, form: Form,
headers: Vec<(String, String)>, headers: Vec<(String, String)>,
_http_version: Option<Version>, _http_version: Option<Version>,
) -> Result<Response, wreq::Error> ) -> Result<Response, wreq::Error> {
{
let http_version = match _http_version { let http_version = match _http_version {
Some(v) => v, Some(v) => v,
None => Version::HTTP_11, None => Version::HTTP_11,
@@ -178,7 +181,11 @@ pub async fn post_json<S>(
request.send().await request.send().await
} }
pub async fn get(&mut self, url: &str, _http_version: Option<Version>) -> Result<String, AnyErr> { pub async fn get(
&mut self,
url: &str,
_http_version: Option<Version>,
) -> Result<String, AnyErr> {
let http_version = match _http_version { let http_version = match _http_version {
Some(v) => v, Some(v) => v,
None => Version::HTTP_11, None => Version::HTTP_11,
@@ -190,7 +197,7 @@ pub async fn post_json<S>(
let proxy = Proxy::all(&proxy_url).unwrap(); let proxy = Proxy::all(&proxy_url).unwrap();
request = request.proxy(proxy); request = request.proxy(proxy);
} }
} }
let response = request.send().await?; let response = request.send().await?;
if response.status().is_success() || response.status().as_u16() == 404 { if response.status().is_success() || response.status().as_u16() == 404 {
return Ok(response.text().await?); return Ok(response.text().await?);
@@ -208,7 +215,6 @@ pub async fn post_json<S>(
} }
} }
// If direct request failed, try FlareSolverr. Map its error to a Send+Sync error immediately, // If direct request failed, try FlareSolverr. Map its error to a Send+Sync error immediately,
// so no non-Send error value lives across later `.await`s. // so no non-Send error value lives across later `.await`s.
let flare_url = match env::var("FLARE_URL") { let flare_url = match env::var("FLARE_URL") {

34
src/videos.rs
View File

@@ -37,17 +37,18 @@ pub struct VideosRequest {
pub channel: Option<String>, //"youtube", pub channel: Option<String>, //"youtube",
pub sort: Option<String>, //"new", pub sort: Option<String>, //"new",
pub query: Option<String>, //"kittens", pub query: Option<String>, //"kittens",
pub page: Option<FlexibleNumber>, //1, pub page: Option<FlexibleNumber>, //1,
pub perPage: Option<FlexibleNumber>, //10, pub perPage: Option<FlexibleNumber>, //10,
// Your server's global options will be sent in the videos request // Your server's global options will be sent in the videos request
// pub flavor: "mint chocolate chip" // pub flavor: "mint chocolate chip"
pub featured: Option<String>, // "featured", pub featured: Option<String>, // "featured",
pub category: Option<String>, // "pmv" pub category: Option<String>, // "pmv"
pub sites: Option<String>, // pub sites: Option<String>, //
pub filter: Option<String>, // pub all_provider_sites: Option<String>, //
pub language: Option<String>, // pub filter: Option<String>, //
pub networks: Option<String>, // pub language: Option<String>, //
pub stars: Option<String>, // pub networks: Option<String>, //
pub stars: Option<String>, //
pub categories: Option<String>, pub categories: Option<String>,
pub duration: Option<String>, pub duration: Option<String>,
} }
@@ -60,11 +61,11 @@ pub struct ServerOptions {
pub filter: Option<String>, pub filter: Option<String>,
pub language: Option<String>, // "en" pub language: Option<String>, // "en"
pub requester: Option<Requester>, pub requester: Option<Requester>,
pub network: Option<String>, // pub network: Option<String>, //
pub stars: Option<String>, // pub stars: Option<String>, //
pub categories: Option<String>, // pub categories: Option<String>, //
pub duration: Option<String>, // pub duration: Option<String>, //
pub sort: Option<String>, // pub sort: Option<String>, //
} }
#[derive(serde::Serialize, Debug)] #[derive(serde::Serialize, Debug)]
@@ -78,11 +79,6 @@ pub struct VideoEmbed {
pub html: String, pub html: String,
pub source: String, pub source: String,
} }
impl VideoEmbed {
pub fn new(html: String, source: String) -> Self {
VideoEmbed { html, source }
}
}
#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)] #[derive(serde::Serialize, serde::Deserialize, Debug, Clone)]
pub struct VideoItem { pub struct VideoItem {
pub duration: u32, // 110, pub duration: u32, // 110,
@@ -148,7 +144,7 @@ impl VideoItem {
serde_json::from_str::<VideoItem>(&s) serde_json::from_str::<VideoItem>(&s)
} }
pub fn tags(mut self, tags: Vec<String>) -> Self { pub fn tags(mut self, tags: Vec<String>) -> Self {
if tags.is_empty(){ if tags.is_empty() {
return self; return self;
} }
self.tags = Some(tags); self.tags = Some(tags);
@@ -179,7 +175,7 @@ impl VideoItem {
self self
} }
pub fn formats(mut self, formats: Vec<VideoFormat>) -> Self { pub fn formats(mut self, formats: Vec<VideoFormat>) -> Self {
if formats.is_empty(){ if formats.is_empty() {
return self; return self;
} }
self.formats = Some(formats); self.formats = Some(formats);