simon/hottub
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fixes and cleanup

Browse Source
This commit is contained in:
Simon
2026-03-05 18:18:48 +00:00
parent 76fd5a4f4f
commit 2627505ade
49 changed files with 3245 additions and 1376 deletions
Download Patch File Download Diff File Expand all files Collapse all files

64
src/api.rs
View File

@@ -1,15 +1,18 @@
use crate::providers::{
ALL_PROVIDERS, DynProvider, panic_payload_to_string, report_provider_error,
run_provider_guarded,
};
use crate::util::cache::VideoCache;
use crate::util::discord::send_discord_error_report;
use crate::util::proxy::{Proxy, all_proxies_snapshot};
use crate::util::requester::Requester;
use crate::{DbPool, db, status::*, videos::*};
use ntex::http::header;
use ntex::web;
use ntex::web::HttpRequest;
use std::cmp::Ordering;
use std::io;
use tokio::task;
use crate::providers::{ALL_PROVIDERS, DynProvider, panic_payload_to_string, report_provider_error, run_provider_guarded};
use crate::util::cache::VideoCache;
use crate::util::discord::send_discord_error_report;
use crate::util::proxy::{Proxy, all_proxies_snapshot};
use crate::util::requester::Requester;
use crate::{DbPool, db, status::*, videos::*};
#[derive(Debug, Clone)]
pub struct ClientVersion {
@@ -119,10 +122,9 @@ async fn status(req: HttpRequest) -> Result<impl web::Responder, web::Error> {
let mut status = Status::new();
for (provider_name, provider) in ALL_PROVIDERS.iter() {
let channel_result =
std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
provider.get_channel(clientversion.clone())
}));
let channel_result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
provider.get_channel(clientversion.clone())
}));
match channel_result {
Ok(Some(channel)) => status.add_channel(channel),
Ok(None) => {}
@@ -218,7 +220,16 @@ async fn videos_post(
.as_deref()
.unwrap_or("all")
.to_string();
let sites = video_request.sites.as_deref().unwrap_or("").to_string();
let sites = if channel == "all" {
video_request
.all_provider_sites
.as_deref()
.or(video_request.sites.as_deref())
.unwrap_or("")
.to_string()
} else {
video_request.sites.as_deref().unwrap_or("").to_string()
};
let filter = video_request.filter.as_deref().unwrap_or("new").to_string();
let language = video_request
.language
@@ -261,22 +272,25 @@ async fn videos_post(
)
.await;
// There is a bug in Hottub38 that makes the client error for a 403-url even though formats work fine
// There is a bug in Hottub38 that makes the client error for a 403-url even though formats work fine
if clientversion == ClientVersion::new(38, 0, "Hot%20Tub".to_string()) {
// filter out videos without preview for old clients
video_items = video_items.into_iter().filter_map(|video| {
let last_url = video
.formats
.as_ref()
.and_then(|formats| formats.last().map(|f| f.url.clone()));
if let Some(url) = last_url {
let mut v = video;
v.url = url;
return Some(v);
}
Some(video)
}).collect();
}
video_items = video_items
.into_iter()
.filter_map(|video| {
let last_url = video
.formats
.as_ref()
.and_then(|formats| formats.last().map(|f| f.url.clone()));
if let Some(url) = last_url {
let mut v = video;
v.url = url;
return Some(v);
}
Some(video)
})
.collect();
}
videos.items = video_items.clone();
if video_items.len() == 0 {

62
src/main.rs
View File

@@ -1,31 +1,32 @@
#![warn(unused_extern_crates)]
#![allow(non_snake_case)]
use std::{env, thread};
use diesel::{r2d2::{self, ConnectionManager}, SqliteConnection};
use diesel::{
SqliteConnection,
r2d2::{self, ConnectionManager},
};
use dotenvy::dotenv;
use ntex_files as fs;
use ntex::web;
use ntex_files as fs;
mod api;
mod proxy;
mod db;
mod models;
mod providers;
mod proxies;
mod proxy;
mod schema;
mod status;
mod util;
mod videos;
mod proxies;
type DbPool = r2d2::Pool<ConnectionManager<SqliteConnection>>;
// #[macro_use(c)]
// extern crate cute;
#[ntex::main]
async fn main() -> std::io::Result<()> {
// std::env::set_var("RUST_BACKTRACE", "1");
@@ -33,7 +34,7 @@ async fn main() -> std::io::Result<()> {
// Enable request logging
if std::env::var("RUST_LOG").is_err() {
unsafe{
unsafe {
std::env::set_var("RUST_LOG", "warn");
}
}
@@ -49,20 +50,22 @@ async fn main() -> std::io::Result<()> {
let mut requester = util::requester::Requester::new();
requester.set_proxy(env::var("PROXY").unwrap_or("0".to_string()) != "0".to_string());
let cache: util::cache::VideoCache = crate::util::cache::VideoCache::new().max_size(100_000).to_owned();
thread::spawn(move || {
// Create a tiny runtime just for these async tasks
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("build tokio runtime");
let cache: util::cache::VideoCache = crate::util::cache::VideoCache::new()
.max_size(100_000)
.to_owned();
rt.block_on(async move {
providers::init_providers_now();
});
thread::spawn(move || {
// Create a tiny runtime just for these async tasks
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("build tokio runtime");
rt.block_on(async move {
providers::init_providers_now();
});
});
web::HttpServer::new(move || {
web::App::new()
.state(pool.clone())
@@ -72,17 +75,16 @@ async fn main() -> std::io::Result<()> {
.service(web::scope("/api").configure(api::config))
.service(web::scope("/proxy").configure(proxy::config))
.service(
web::resource("/")
.route(web::get().to(|req: web::HttpRequest| async move{
let host = match std::env::var("DOMAIN"){
Ok(d) => d,
Err(_) => req.connection_info().host().to_string()
};
let source_forward_header = format!("hottub://source?url={}", host);
web::HttpResponse::Found()
.header("Location", source_forward_header)
.finish()
}))
web::resource("/").route(web::get().to(|req: web::HttpRequest| async move {
let host = match std::env::var("DOMAIN") {
Ok(d) => d,
Err(_) => req.connection_info().host().to_string(),
};
let source_forward_header = format!("hottub://source?url={}", host);
web::HttpResponse::Found()
.header("Location", source_forward_header)
.finish()
})),
)
.service(fs::Files::new("/", "static").index_file("index.html"))
})

2
src/models.rs
View File

@@ -1,5 +1,5 @@
use diesel::prelude::*;
use serde::{Serialize};
use serde::Serialize;
#[derive(Debug, Clone, Serialize, Queryable, Insertable)]
#[diesel(table_name = crate::schema::videos)]

34
src/providers/all.rs
View File

@@ -1,18 +1,18 @@
use std::fs;
use std::time::Duration;
use crate::DbPool;
use crate::api::{ClientVersion, get_provider};
use crate::providers::{DynProvider, Provider, report_provider_error, run_provider_guarded};
use crate::status::{Channel, ChannelOption, FilterOption};
use crate::util::cache::VideoCache;
use crate::util::interleave;
use crate::videos::{ServerOptions, VideoItem};
use async_trait::async_trait;
use capitalize::Capitalize;
use cute::c;
use error_chain::error_chain;
use futures::StreamExt;
use futures::stream::FuturesUnordered;
use crate::api::{get_provider, ClientVersion};
use crate::providers::{DynProvider, Provider, report_provider_error, run_provider_guarded};
use crate::status::{Channel, ChannelOption, FilterOption};
use crate::util::cache::VideoCache;
use crate::util::interleave;
use crate::videos::{ServerOptions, VideoItem};
use crate::DbPool;
use std::fs;
use std::time::Duration;
error_chain! {
foreign_links {
@@ -21,11 +21,9 @@ error_chain! {
}
}
#[derive(Debug, Clone)]
#[allow(dead_code)]
pub struct AllProvider {
}
pub struct AllProvider {}
impl AllProvider {
pub fn new() -> Self {
@@ -81,7 +79,8 @@ impl Provider for AllProvider {
None => {
// fire-and-forget reporting of missing provider keys
tokio::spawn(async move {
report_provider_error("all", "all.get_videos.unknown_provider", &name).await;
report_provider_error("all", "all.get_videos.unknown_provider", &name)
.await;
});
None
}
@@ -128,7 +127,7 @@ impl Provider for AllProvider {
},
_ = &mut timeout_timer => {
// 55 seconds passed. Stop waiting and return what we have.
// The tasks remaining in 'futures' will continue running in the
// The tasks remaining in 'futures' will continue running in the
// background because they were 'tokio::spawn'ed.
break;
},
@@ -146,10 +145,7 @@ impl Provider for AllProvider {
.filter_map(|entry| entry.ok())
.filter_map(|entry| entry.file_name().into_string().ok())
.filter(|name| name.ends_with(".rs"))
.filter(|name| {
!name.contains("mod.rs")
&& !name.contains("all.rs")
})
.filter(|name| !name.contains("mod.rs") && !name.contains("all.rs"))
.map(|name| name.replace(".rs", ""))
.collect::<Vec<String>>();
let sites = c![FilterOption {
@@ -166,7 +162,7 @@ impl Provider for AllProvider {
status: "active".to_string(),
categories: vec![],
options: vec![ChannelOption {
id: "sites".to_string(),
id: "all_provider_sites".to_string(),
title: "Sites".to_string(),
description: "What Sites to use".to_string(),
systemImage: "list.number".to_string(),

92
src/providers/beeg.rs
View File

@@ -37,9 +37,18 @@ pub struct BeegProvider {
impl BeegProvider {
pub fn new() -> Self {
let provider = BeegProvider {
sites: Arc::new(RwLock::new(vec![FilterOption { id: "all".into(), title: "All".into() }])),
stars: Arc::new(RwLock::new(vec![FilterOption { id: "all".into(), title: "All".into() }])),
categories: Arc::new(RwLock::new(vec![FilterOption { id: "all".into(), title: "All".into() }])),
sites: Arc::new(RwLock::new(vec![FilterOption {
id: "all".into(),
title: "All".into(),
}])),
stars: Arc::new(RwLock::new(vec![FilterOption {
id: "all".into(),
title: "All".into(),
}])),
categories: Arc::new(RwLock::new(vec![FilterOption {
id: "all".into(),
title: "All".into(),
}])),
};
provider.spawn_initial_load();
@@ -52,7 +61,10 @@ impl BeegProvider {
let stars = Arc::clone(&self.stars);
thread::spawn(move || {
let rt = match tokio::runtime::Builder::new_current_thread().enable_all().build() {
let rt = match tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
{
Ok(rt) => rt,
Err(e) => {
eprintln!("beeg runtime init failed: {}", e);
@@ -77,14 +89,18 @@ impl BeegProvider {
async fn fetch_tags() -> Result<Value> {
let mut requester = util::requester::Requester::new();
let text = match requester
.get("https://store.externulls.com/tag/facts/tags?get_original=true&slug=index", None)
.await {
Ok(text) => text,
Err(e) => {
eprintln!("beeg fetch_tags failed: {}", e);
return Err(ErrorKind::Parse("failed to fetch tags".into()).into());
}
};
.get(
"https://store.externulls.com/tag/facts/tags?get_original=true&slug=index",
None,
)
.await
{
Ok(text) => text,
Err(e) => {
eprintln!("beeg fetch_tags failed: {}", e);
return Err(ErrorKind::Parse("failed to fetch tags".into()).into());
}
};
Ok(serde_json::from_str(&text)?)
}
@@ -99,7 +115,13 @@ impl BeegProvider {
s.get("tg_name").and_then(|v| v.as_str()),
s.get("tg_slug").and_then(|v| v.as_str()),
) {
Self::push_unique(&stars, FilterOption { id: id.into(), title: name.into() });
Self::push_unique(
&stars,
FilterOption {
id: id.into(),
title: name.into(),
},
);
}
}
Ok(())
@@ -139,7 +161,13 @@ impl BeegProvider {
s.get("tg_name").and_then(|v| v.as_str()),
s.get("tg_slug").and_then(|v| v.as_str()),
) {
Self::push_unique(&sites, FilterOption { id: id.into(), title: name.into() });
Self::push_unique(
&sites,
FilterOption {
id: id.into(),
title: name.into(),
},
);
}
}
Ok(())
@@ -178,7 +206,11 @@ impl BeegProvider {
description: "Filter for different Networks".into(),
systemImage: "list.dash".into(),
colorName: "purple".into(),
options: self.categories.read().map(|v| v.clone()).unwrap_or_default(),
options: self
.categories
.read()
.map(|v| v.clone())
.unwrap_or_default(),
multiSelect: false,
},
ChannelOption {
@@ -220,7 +252,8 @@ impl BeegProvider {
}
let video_url = format!(
"https://store.externulls.com/facts/tag?limit=100&offset={}{}",
page - 1, match slug {
page - 1,
match slug {
"" => "&id=27173".to_string(),
_ => format!("&slug={}", slug.replace(" ", "")),
}
@@ -237,7 +270,8 @@ impl BeegProvider {
vec![]
}
};
let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let text = match requester.get(&video_url, None).await {
Ok(text) => text,
Err(e) => {
@@ -289,7 +323,8 @@ impl BeegProvider {
}
};
let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let text = match requester.get(&video_url, None).await {
Ok(text) => text,
@@ -323,14 +358,24 @@ impl BeegProvider {
};
for video in array {
let file = match video.get("file") { Some(v) => v, None => continue };
let hls = match file.get("hls_resources") { Some(v) => v, None => continue };
let file = match video.get("file") {
Some(v) => v,
None => continue,
};
let hls = match file.get("hls_resources") {
Some(v) => v,
None => continue,
};
let key = match hls.get("fl_cdn_multi").and_then(|v| v.as_str()) {
Some(v) => v,
None => continue,
};
let id = file.get("id").and_then(|v| v.as_i64()).unwrap_or(0).to_string();
let id = file
.get("id")
.and_then(|v| v.as_i64())
.unwrap_or(0)
.to_string();
let title = file
.get("data")
.and_then(|v| v.get(0))
@@ -352,7 +397,10 @@ impl BeegProvider {
.and_then(|s| parse_abbreviated_number(s))
.unwrap_or(0);
let thumb = format!("https://thumbs.externulls.com/videos/{}/0.webp?size=480x270", id);
let thumb = format!(
"https://thumbs.externulls.com/videos/{}/0.webp?size=480x270",
id
);
let mut item = VideoItem::new(
id,

9
src/providers/chaturbate.rs
View File

@@ -252,7 +252,8 @@ impl ChaturbateProvider {
let mut title = video_segment
.get("room_subject")
.and_then(|v| v.as_str())
.map(String::from).unwrap_or("".to_string());
.map(String::from)
.unwrap_or("".to_string());
// html decode
title = decode(title.as_bytes()).to_string().unwrap_or(title);
let id = username.clone();
@@ -262,7 +263,11 @@ impl ChaturbateProvider {
.unwrap_or(&serde_json::Value::String("".to_string()))
.as_str()
.unwrap_or("")
.split("?").collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.split("?")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
let views = video_segment
.get("viewers")

285
src/providers/freepornvideosxxx.rs
View File

@@ -120,17 +120,40 @@ impl FreepornvideosxxxProvider {
.copied()
.unwrap_or_default()
.split("custom_list_models_models_list_pagination")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default();
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default();
for stars_element in stars_div.split("<a ").collect::<Vec<&str>>()[1..].to_vec() {
let star_url = stars_element.split("href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
let star_url = stars_element
.split("href=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default();
let star_id = star_url.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string();
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default();
let star_id = star_url
.split("/")
.collect::<Vec<&str>>()
.get(4)
.copied()
.unwrap_or_default()
.to_string();
let star_name = stars_element
.split("<strong class=\"title\">")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
Self::push_unique(
&stars,
@@ -168,19 +191,47 @@ impl FreepornvideosxxxProvider {
}
let sites_div = text
.split("id=\"list_content_sources_sponsors_list_items\"")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("class=\"pagination\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default();
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default();
for sites_element in
sites_div.split("class=\"headline\"").collect::<Vec<&str>>()[1..].to_vec()
{
let site_url = sites_element.split("href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
let site_url = sites_element
.split("href=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default();
let site_id = site_url.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string();
let site_name = sites_element.split("<h2>").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default();
let site_id = site_url
.split("/")
.collect::<Vec<&str>>()
.get(4)
.copied()
.unwrap_or_default()
.to_string();
let site_name = sites_element
.split("<h2>")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
Self::push_unique(
&sites,
@@ -207,22 +258,52 @@ impl FreepornvideosxxxProvider {
return Ok(());
}
};
let networks_div = text.split("class=\"sites__list\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
let networks_div = text
.split("class=\"sites__list\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("</div>")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default();
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default();
for network_element in
networks_div.split("sites__item").collect::<Vec<&str>>()[1..].to_vec()
{
if network_element.contains("sites__all") {
continue;
}
let network_url = network_element.split("href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
let network_url = network_element
.split("href=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default();
let network_id = network_url.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string();
let network_name = network_element.split(">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default();
let network_id = network_url
.split("/")
.collect::<Vec<&str>>()
.get(4)
.copied()
.unwrap_or_default()
.to_string();
let network_name = network_element
.split(">")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
Self::push_unique(
&networks,
@@ -271,7 +352,8 @@ impl FreepornvideosxxxProvider {
name: "FreePornVideos XXX".to_string(),
description: "Free Porn Videos".to_string(),
premium: false,
favicon: "https://www.google.com/s2/favicons?sz=64&domain=www.freepornvideos.xxx".to_string(),
favicon: "https://www.google.com/s2/favicons?sz=64&domain=www.freepornvideos.xxx"
.to_string(),
status: "active".to_string(),
categories: vec![],
options: vec![
@@ -522,11 +604,22 @@ impl FreepornvideosxxxProvider {
if !html.contains("class=\"item\"") {
return items;
}
let raw_videos = html.split("videos_list_pagination").collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
let raw_videos = html
.split("videos_list_pagination")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.split(" class=\"pagination\" ")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.split("class=\"list-videos\"")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("class=\"item\"")
.collect::<Vec<&str>>()[1..]
.to_vec();
@@ -535,39 +628,94 @@ impl FreepornvideosxxxProvider {
// for (index, line) in vid.iter().enumerate() {
// println!("Line {}: {}", index, line);
// }
let video_url: String = video_segment.split("<a href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
let video_url: String = video_segment
.split("<a href=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
let mut title = video_segment.split(" title=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
let mut title = video_segment
.split(" title=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
// html decode
title = decode(title.as_bytes()).to_string().unwrap_or(title);
let id = video_url.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string();
let id = video_url
.split("/")
.collect::<Vec<&str>>()
.get(4)
.copied()
.unwrap_or_default()
.to_string();
let thumb = match video_segment.split("<img ").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
let thumb = match video_segment
.split("<img ")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.contains("data-src=\"")
{
true => video_segment.split("<img ").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
true => video_segment
.split("<img ")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("data-src=\"")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(),
false => video_segment.split("<img ").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
false => video_segment
.split("<img ")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("src=\"")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(),
};
let raw_duration = video_segment
.split("<span class=\"duration\">")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.split(" ")
.collect::<Vec<&str>>()
.last()
@@ -577,9 +725,15 @@ impl FreepornvideosxxxProvider {
let views = parse_abbreviated_number(
video_segment
.split("<div class=\"views\">")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string()
.as_str(),
)
@@ -587,9 +741,15 @@ impl FreepornvideosxxxProvider {
let preview = video_segment
.split("data-preview=\"")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
let site_name = title
.split("]")
@@ -603,9 +763,15 @@ impl FreepornvideosxxxProvider {
let mut tags = match video_segment.contains("class=\"models\">") {
true => video_segment
.split("class=\"models\">")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("</div>")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.split("href=\"")
.collect::<Vec<&str>>()[1..]
.into_iter()
@@ -613,17 +779,38 @@ impl FreepornvideosxxxProvider {
Self::push_unique(
&self.stars,
FilterOption {
id: s.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string(),
title: s.split(">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
id: s
.split("/")
.collect::<Vec<&str>>()
.get(4)
.copied()
.unwrap_or_default()
.to_string(),
title: s
.split(">")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.trim()
.to_string(),
},
);
s.split(">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
s.split(">")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.trim()
.to_string()
})

33
src/providers/hanime.rs
View File

@@ -4,8 +4,8 @@ use futures::future::join_all;
use serde_json::json;
use std::vec;
use crate::api::ClientVersion;
use crate::DbPool;
use crate::api::ClientVersion;
use crate::db;
use crate::providers::{Provider, report_provider_error, report_provider_error_background};
use crate::status::*;
@@ -272,7 +272,8 @@ impl HanimeProvider {
id
);
let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let payload = json!({
"width": 571, "height": 703, "ab": "kh" }
);
@@ -289,7 +290,7 @@ impl HanimeProvider {
],
)
.await; // Initial request to set cookies
ntex::time::sleep(ntex::time::Seconds(1)).await;
ntex::time::sleep(ntex::time::Seconds(1)).await;
let text = requester
.get_raw_with_headers(
&url,
@@ -328,7 +329,12 @@ impl HanimeProvider {
let mut url_vec = vec![];
for el in urls.split("\"url\":\"").collect::<Vec<&str>>() {
let url = el.split("\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default();
let url = el
.split("\"")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default();
if !url.is_empty() && url.contains("m3u8") {
url_vec.push(url.to_string());
}
@@ -381,11 +387,23 @@ impl HanimeProvider {
) -> Result<Vec<VideoItem>> {
let index = format!("hanime:{}:{}:{}", query, page, sort);
let order_by = match sort.contains(".") {
true => sort.split(".").collect::<Vec<&str>>().get(0).copied().unwrap_or_default().to_string(),
true => sort
.split(".")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(),
false => "created_at_unix".to_string(),
};
let ordering = match sort.contains(".") {
true => sort.split(".").collect::<Vec<&str>>().get(1).copied().unwrap_or_default().to_string(),
true => sort
.split(".")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.to_string(),
false => "desc".to_string(),
};
let old_items = match cache.get(&index) {
@@ -408,7 +426,8 @@ impl HanimeProvider {
.order_by(order_by)
.ordering(ordering);
let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let response = match requester
.post_json("https://search.htv-services.com/search", &search, vec![])
.await

33
src/providers/hentaihaven.rs
View File

@@ -137,11 +137,7 @@ impl HentaihavenProvider {
options: ServerOptions,
pool: DbPool,
) -> Result<Vec<VideoItem>> {
let video_url = format!(
"{}/?s={}",
self.url,
query.replace(" ", "+"),
);
let video_url = format!("{}/?s={}", self.url, query.replace(" ", "+"),);
// Check our Video Cache. If the result is younger than 1 hour, we return it.
let old_items = match cache.get(&video_url) {
Some((time, items)) => {
@@ -171,8 +167,7 @@ impl HentaihavenProvider {
return Ok(old_items);
}
};
if page > 1
{
if page > 1 {
return Ok(vec![]);
}
let video_items: Vec<VideoItem> = self
@@ -438,8 +433,7 @@ impl HentaihavenProvider {
.last()
.and_then(|s| s.split("summary-content\">").nth(1))
.and_then(|s| s.split(" Total").nth(0))
.map(|s|
s.trim().parse::<u32>().unwrap_or(0))
.map(|s| s.trim().parse::<u32>().unwrap_or(0))
.unwrap_or(0);
let mut formats = vec![];
let episode_block = html
@@ -470,14 +464,23 @@ impl HentaihavenProvider {
let format = VideoFormat::new(episode_url, "1080p".to_string(), "m3u8".to_string())
.format_id(episode_title.clone())
.http_header("Connection".to_string(), "keep-alive".to_string())
.http_header("User-Agent".to_string(), "Mozilla/5.0 (X11; Linux x86_64; rv:140.0) Gecko/20100101 Firefox/140.0".to_string())
.http_header("Accept".to_string(), "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8".to_string())
.http_header(
"User-Agent".to_string(),
"Mozilla/5.0 (X11; Linux x86_64; rv:140.0) Gecko/20100101 Firefox/140.0"
.to_string(),
)
.http_header(
"Accept".to_string(),
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8".to_string(),
)
.http_header("Accept-Language".to_string(), "en-US,en;q=0.5".to_string())
.http_header("Accept-Encoding".to_string(), "gzip, deflate, br".to_string())
.http_header(
"Accept-Encoding".to_string(),
"gzip, deflate, br".to_string(),
)
.http_header("Sec-Fetch-Mode".to_string(), "navigate".to_string())
.http_header("Origin".to_string(), self.url.clone())
.format_note(episode_title.clone())
;
.http_header("Origin".to_string(), self.url.clone())
.format_note(episode_title.clone());
formats.push(format);
}
if formats.is_empty() {

173
src/providers/homoxxx.rs
View File

@@ -1,5 +1,5 @@
use crate::api::ClientVersion;
use crate::DbPool;
use crate::api::ClientVersion;
use crate::providers::{Provider, report_provider_error};
use crate::status::*;
use crate::util::cache::VideoCache;
@@ -11,7 +11,7 @@ use error_chain::error_chain;
use htmlentity::entity::{ICodedDataTrait, decode};
use std::env;
use std::vec;
use wreq::{Client};
use wreq::Client;
use wreq_util::Emulation;
error_chain! {
@@ -67,12 +67,7 @@ impl HomoxxxProvider {
cacheDuration: Some(1800),
}
}
async fn get(
&self,
cache: VideoCache,
page: u8,
sort: &str,
) -> Result<Vec<VideoItem>> {
async fn get(&self, cache: VideoCache, page: u8, sort: &str) -> Result<Vec<VideoItem>> {
let sort_string = match sort {
"trending" => "/trending",
"popular" => "/popular",
@@ -93,13 +88,22 @@ impl HomoxxxProvider {
};
// let proxy = Proxy::all("http://192.168.0.103:8081").unwrap();
let client = Client::builder().cert_verification(false).emulation(Emulation::Firefox136).build()?;
let client = Client::builder()
.cert_verification(false)
.emulation(Emulation::Firefox136)
.build()?;
let mut response = client.get(video_url.clone())
// .proxy(proxy.clone())
.send().await?;
let mut response = client
.get(video_url.clone())
// .proxy(proxy.clone())
.send()
.await?;
if response.status().is_redirection() {
let location = match response.headers().get("Location").and_then(|h| h.to_str().ok()) {
let location = match response
.headers()
.get("Location")
.and_then(|h| h.to_str().ok())
{
Some(location) => location,
None => {
report_provider_error(
@@ -132,12 +136,7 @@ impl HomoxxxProvider {
let flare_url = match env::var("FLARE_URL") {
Ok(url) => url,
Err(e) => {
report_provider_error(
"homoxxx",
"get.flare_url",
&e.to_string(),
)
.await;
report_provider_error("homoxxx", "get.flare_url", &e.to_string()).await;
return Ok(old_items);
}
};
@@ -168,17 +167,18 @@ impl HomoxxxProvider {
Ok(video_items)
}
}
async fn query(
&self,
cache: VideoCache,
page: u8,
query: &str,
) -> Result<Vec<VideoItem>> {
async fn query(&self, cache: VideoCache, page: u8, query: &str) -> Result<Vec<VideoItem>> {
let search_string = query.to_lowercase().trim().replace(" ", "-");
let mut video_url = format!("{}/search/{}/{}/", self.url, search_string, page);
if search_string.starts_with("@"){
let url_part = search_string.split("@").collect::<Vec<&str>>().get(1).copied().unwrap_or_default().replace(":", "/");
if search_string.starts_with("@") {
let url_part = search_string
.split("@")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.replace(":", "/");
video_url = format!("{}/{}/", self.url, url_part);
}
// Check our Video Cache. If the result is younger than 1 hour, we return it.
@@ -197,14 +197,23 @@ impl HomoxxxProvider {
};
// let proxy = Proxy::all("http://192.168.0.103:8081").unwrap();
let client = Client::builder().cert_verification(false).emulation(Emulation::Firefox136).build()?;
let client = Client::builder()
.cert_verification(false)
.emulation(Emulation::Firefox136)
.build()?;
let mut response = client.get(video_url.clone())
// .proxy(proxy.clone())
.send().await?;
let mut response = client
.get(video_url.clone())
// .proxy(proxy.clone())
.send()
.await?;
if response.status().is_redirection() {
let location = match response.headers().get("Location").and_then(|h| h.to_str().ok()) {
let location = match response
.headers()
.get("Location")
.and_then(|h| h.to_str().ok())
{
Some(location) => location,
None => {
report_provider_error(
@@ -237,12 +246,7 @@ impl HomoxxxProvider {
let flare_url = match env::var("FLARE_URL") {
Ok(url) => url,
Err(e) => {
report_provider_error(
"homoxxx",
"query.flare_url",
&e.to_string(),
)
.await;
report_provider_error("homoxxx", "query.flare_url", &e.to_string()).await;
return Ok(old_items);
}
};
@@ -277,7 +281,12 @@ impl HomoxxxProvider {
return vec![];
}
let mut items: Vec<VideoItem> = Vec::new();
let raw_videos = html.split("pagination").collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
let raw_videos = html
.split("pagination")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.split("<div class=\"item \">")
.collect::<Vec<&str>>()[1..]
.to_vec();
@@ -286,31 +295,81 @@ impl HomoxxxProvider {
// for (index, line) in vid.iter().enumerate() {
// println!("Line {}: {}", index, line);
// }
let video_url: String = video_segment.split("<a href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default().to_string();
let preview_url = video_segment.split("data-preview-custom=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
let video_url: String = video_segment
.split("<a href=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
let mut title = video_segment.split("\" title=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
let preview_url = video_segment
.split("data-preview-custom=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
let mut title = video_segment
.split("\" title=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
// html decode
title = decode(title.as_bytes()).to_string().unwrap_or(title);
let id = video_url.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string();
let id = video_url
.split("/")
.collect::<Vec<&str>>()
.get(4)
.copied()
.unwrap_or_default()
.to_string();
let raw_duration = video_segment
.split("<p class=\"duration_item\">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<p class=\"duration_item\">")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
let thumb = video_segment.split("thumb lazyload").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("data-src=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
let thumb = video_segment
.split("thumb lazyload")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("data-src=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
let video_item = VideoItem::new(
id,
@@ -320,14 +379,11 @@ impl HomoxxxProvider {
thumb,
duration,
)
.preview(preview_url)
;
.preview(preview_url);
items.push(video_item);
}
return items;
}
}
#[async_trait]
@@ -346,10 +402,7 @@ impl Provider for HomoxxxProvider {
let _ = per_page;
let _ = pool;
let videos: std::result::Result<Vec<VideoItem>, Error> = match query {
Some(q) => {
self.query(cache, page.parse::<u8>().unwrap_or(1), &q,)
.await
}
Some(q) => self.query(cache, page.parse::<u8>().unwrap_or(1), &q).await,
None => {
self.get(cache, page.parse::<u8>().unwrap_or(1), &sort)
.await

181
src/providers/hqporner.rs
View File

@@ -9,10 +9,9 @@ use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoFormat, VideoItem};
use async_trait::async_trait;
use error_chain::error_chain;
use futures::future::join_all;
use futures::stream::{FuturesUnordered, StreamExt};
use htmlentity::entity::{ICodedDataTrait, decode};
use std::sync::{Arc, RwLock};
use std::thread::sleep;
use std::{thread, vec};
use titlecase::Titlecase;
@@ -263,17 +262,37 @@ impl HqpornerProvider {
})
.unwrap_or_default();
let futures = raw_videos
.into_iter()
.map(|el| self.get_video_item(el, requester.clone()));
// Limit concurrent detail-page requests to reduce transient connect errors.
let mut in_flight = FuturesUnordered::new();
let mut iter = raw_videos.into_iter();
let mut items = Vec::new();
const MAX_IN_FLIGHT: usize = 6;
join_all(futures)
.await
.into_iter()
.inspect(|r| {
if let Err(e) = r {
loop {
while in_flight.len() < MAX_IN_FLIGHT {
let Some(seg) = iter.next() else {
break;
};
in_flight.push(self.get_video_item(seg, requester.clone()));
}
let Some(result) = in_flight.next().await else {
break;
};
match result {
Ok(item)
if item
.formats
.as_ref()
.map(|formats| !formats.is_empty())
.unwrap_or(false) =>
{
items.push(item);
}
Ok(_) => {}
Err(e) => {
let msg = e.to_string();
let chain = format_error_chain(e);
let chain = format_error_chain(&e);
tokio::spawn(async move {
let _ = send_discord_error_report(
msg,
@@ -287,10 +306,10 @@ impl HqpornerProvider {
.await;
});
}
})
.filter_map(Result::ok)
.filter(|item| item.formats.as_ref().map(|formats| !formats.is_empty()).unwrap_or(false))
.collect()
}
}
items
}
async fn get_video_item(&self, seg: String, mut requester: Requester) -> Result<VideoItem> {
@@ -319,13 +338,22 @@ impl HqpornerProvider {
.and_then(|s| s.split('.').next())
.ok_or_else(|| ErrorKind::Parse(format!("id \n{seg}").into()))?
.to_string();
let thumb = format!(
"https:{}",
seg.split("onmouseleave='defaultImage(\"")
.nth(1)
.and_then(|s| s.split('"').next())
.ok_or_else(|| ErrorKind::Parse(format!("thumb \n{seg}").into()))?
);
let thumb_raw = seg
.split("onmouseleave='defaultImage(\"")
.nth(1)
.and_then(|s| s.split('"').next())
.ok_or_else(|| ErrorKind::Parse(format!("thumb \n{seg}").into()))?;
let thumb_abs = if thumb_raw.starts_with("//") {
format!("https:{}", thumb_raw)
} else if thumb_raw.starts_with("http://") || thumb_raw.starts_with("https://") {
thumb_raw.to_string()
} else {
format!("https://{}", thumb_raw.trim_start_matches('/'))
};
let thumb = match thumb_abs.strip_prefix("https://") {
Some(path) => format!("https://hottub.spacemoehre.de/proxy/hqporner-thumb/{path}"),
None => thumb_abs,
};
let raw_duration = seg
.split("<span class=\"icon fa-clock-o meta-data\">")
.nth(1)
@@ -335,7 +363,7 @@ impl HqpornerProvider {
let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
let (tags, formats) = self.extract_media(&video_url, &mut requester).await?;
Ok(
VideoItem::new(id, title, video_url, "hqporner".into(), thumb, duration)
.formats(formats)
@@ -350,17 +378,35 @@ impl HqpornerProvider {
) -> Result<(Vec<String>, Vec<VideoFormat>)> {
let mut formats = vec![];
let mut tags = vec![];
let resp = requester
.get_raw_with_headers(
url,
vec![("Referer".to_string(), "https://hqporner.com/".into())],
)
let headers = vec![("Referer".to_string(), "https://hqporner.com/".into())];
let mut text = match self
.fetch_text_with_retries(requester, url, &headers, 3)
.await
.map_err(|e| Error::from(format!("Request failed: {}", e)))?;
let text = resp
.text()
.await
.map_err(|e| Error::from(format!("Text conversion failed: {}", e)))?;
{
Ok(text) => text,
Err(primary_err) => {
if url.contains("://hqporner.com/") {
let fallback_url = url.replace("://hqporner.com/", "://www.hqporner.com/");
self.fetch_text_with_retries(requester, &fallback_url, &headers, 3)
.await
.map_err(|fallback_err| {
Error::from(format!(
"Request failed: primary={primary_err}; fallback={fallback_err}"
))
})?
} else {
return Err(Error::from(format!("Request failed: {}", primary_err)));
}
}
};
if text.is_empty() && url.contains("://hqporner.com/") {
let fallback_url = url.replace("://hqporner.com/", "://www.hqporner.com/");
text = self
.fetch_text_with_retries(requester, &fallback_url, &headers, 3)
.await
.unwrap_or_default();
}
if text.contains("Why do I see it?") {
return Ok((tags, formats));
@@ -395,22 +441,11 @@ impl HqpornerProvider {
.and_then(|s| s.split('\'').next())
.ok_or("No player link")?
);
let mut r = requester
.get_raw_with_headers(
&player_url,
vec![("Referer".to_string(), "https://hqporner.com/".into())],
).await;
if let Err(_e) = &r {
sleep(std::time::Duration::from_secs(1));
r = requester
.get_raw_with_headers(
&player_url,
vec![("Referer".to_string(), "https://hqporner.com/".into())],
).await;
}
let response = match r {
Ok(response) => response,
let response_text = match self
.fetch_text_with_retries(requester, &player_url, &headers, 2)
.await
{
Ok(text) => text,
Err(e) => {
let err = format!("altplayer request failed: {e}");
send_discord_error_report(
@@ -426,13 +461,10 @@ impl HqpornerProvider {
return Ok((tags, formats));
}
};
let text2 = response
.text()
.await
.map_err(|e| Error::from(format!("Text conversion failed: {}", e)))?;
let text2 = response_text;
// Check for error response
if text2.starts_with("ERR:"){
if text2.starts_with("ERR:") {
return Ok((tags, formats));
}
@@ -467,6 +499,37 @@ impl HqpornerProvider {
Ok((tags, formats))
}
async fn fetch_text_with_retries(
&self,
requester: &mut Requester,
url: &str,
headers: &[(String, String)],
max_attempts: u8,
) -> std::result::Result<String, String> {
let mut last_err = String::new();
for attempt in 1..=max_attempts {
match requester.get_raw_with_headers(url, headers.to_vec()).await {
Ok(resp) => match resp.text().await {
Ok(text) => return Ok(text),
Err(e) => {
last_err =
format!("text read failed (attempt {attempt}/{max_attempts}): {e}");
}
},
Err(e) => {
last_err = format!("request failed (attempt {attempt}/{max_attempts}): {e}");
}
}
if attempt < max_attempts {
tokio::time::sleep(std::time::Duration::from_millis(250 * attempt as u64)).await;
}
}
Err(last_err)
}
}
#[async_trait]
@@ -488,7 +551,15 @@ impl Provider for HqpornerProvider {
};
res.unwrap_or_else(|e| {
eprintln!("Hqporner error: {e}");
let _ = send_discord_error_report(e.to_string(), Some(format_error_chain(&e)), None, None, file!(), line!(), module_path!());
let _ = send_discord_error_report(
e.to_string(),
Some(format_error_chain(&e)),
None,
None,
file!(),
line!(),
module_path!(),
);
vec![]
})
}

6
src/providers/hypnotube.rs
View File

@@ -212,7 +212,8 @@ impl HypnotubeProvider {
vec![]
}
};
let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let text = match requester.get(&video_url, Some(Version::HTTP_11)).await {
Ok(text) => text,
Err(e) => {
@@ -273,7 +274,8 @@ impl HypnotubeProvider {
}
};
let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let post_response = match requester
.post(
format!("{}/searchgate.php", self.url).as_str(),

111
src/providers/javtiful.rs
View File

@@ -11,9 +11,9 @@ use crate::videos::{ServerOptions, VideoFormat, VideoItem};
use async_trait::async_trait;
use error_chain::error_chain;
use futures::future::join_all;
use htmlentity::entity::{decode, ICodedDataTrait};
use htmlentity::entity::{ICodedDataTrait, decode};
use std::sync::{Arc, RwLock};
use std::{vec};
use std::vec;
use titlecase::Titlecase;
use wreq::Version;
@@ -118,10 +118,7 @@ impl JavtifulProvider {
"most viewed" => "/sort=most_viewed",
_ => "",
};
let video_url = format!(
"{}/videos{}?page={}",
self.url, sort_string, page
);
let video_url = format!("{}/videos{}?page={}", self.url, sort_string, page);
let old_items = match cache.get(&video_url) {
Some((time, items)) => {
if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
@@ -149,7 +146,12 @@ impl JavtifulProvider {
return Ok(old_items);
}
};
if page > 1 && !text.contains(&format!("<li class=\"page-item active\"><span class=\"page-link\">{}</span>", page)) {
if page > 1
&& !text.contains(&format!(
"<li class=\"page-item active\"><span class=\"page-link\">{}</span>",
page
))
{
return Ok(vec![]);
}
let video_items: Vec<VideoItem> = self
@@ -178,7 +180,10 @@ impl JavtifulProvider {
};
let video_url = format!(
"{}/search/videos{}?search_query={}&page={}",
self.url, sort_string, query.replace(" ","+"), page
self.url,
sort_string,
query.replace(" ", "+"),
page
);
// Check our Video Cache. If the result is younger than 1 hour, we return it.
let old_items = match cache.get(&video_url) {
@@ -209,7 +214,12 @@ impl JavtifulProvider {
return Ok(old_items);
}
};
if page > 1 && !text.contains(&format!("<li class=\"page-item active\"><span class=\"page-link\">{}</span>", page)) {
if page > 1
&& !text.contains(&format!(
"<li class=\"page-item active\"><span class=\"page-link\">{}</span>",
page
))
{
return Ok(vec![]);
}
let video_items: Vec<VideoItem> = self
@@ -233,11 +243,10 @@ impl JavtifulProvider {
return vec![];
}
let block = match html
.split("pagination ")
.next()
.and_then(|s| s.split("row row-cols-1 row-cols-sm-2 row-cols-lg-3 row-cols-xl-4").nth(1))
{
let block = match html.split("pagination ").next().and_then(|s| {
s.split("row row-cols-1 row-cols-sm-2 row-cols-lg-3 row-cols-xl-4")
.nth(1)
}) {
Some(b) => b,
None => {
eprint!("Javtiful Provider: Failed to get block from html");
@@ -250,9 +259,10 @@ impl JavtifulProvider {
file!(),
line!(),
module_path!(),
).await;
return vec![]
},
)
.await;
return vec![];
}
};
let futures = block
@@ -281,7 +291,8 @@ impl JavtifulProvider {
file!(), // Note: these might report the utility line
line!(), // better to hardcode or pass from outside
module_path!(),
).await;
)
.await;
});
}
})
@@ -289,11 +300,7 @@ impl JavtifulProvider {
.collect()
}
async fn get_video_item(
&self,
seg: String,
mut requester: Requester,
) -> Result<VideoItem> {
async fn get_video_item(&self, seg: String, mut requester: Requester) -> Result<VideoItem> {
let video_url = seg
.split(" href=\"")
.nth(1)
@@ -309,7 +316,10 @@ impl JavtifulProvider {
.trim()
.to_string();
title = decode(title.as_bytes()).to_string().unwrap_or(title).titlecase();
title = decode(title.as_bytes())
.to_string()
.unwrap_or(title)
.titlecase();
let id = video_url
.split('/')
.nth(5)
@@ -340,26 +350,17 @@ impl JavtifulProvider {
.unwrap_or("")
.to_string();
let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
let (tags, formats, views) =
self.extract_media(&video_url, &mut requester).await?;
let (tags, formats, views) = self.extract_media(&video_url, &mut requester).await?;
if preview.len() == 0 {
preview = format!("https://trailers.jav.si/preview/{id}.mp4");
}
let video_item = VideoItem::new(
id,
title,
video_url,
"javtiful".into(),
thumb,
duration,
)
.formats(formats)
.tags(tags)
.preview(preview)
.views(views);
let video_item = VideoItem::new(id, title, video_url, "javtiful".into(), thumb, duration)
.formats(formats)
.tags(tags)
.preview(preview)
.views(views);
Ok(video_item)
}
async fn extract_media(
@@ -371,7 +372,9 @@ impl JavtifulProvider {
.get(url, Some(Version::HTTP_2))
.await
.map_err(|e| Error::from(format!("{}", e)))?;
let tags = text.split("related-actress").next()
let tags = text
.split("related-actress")
.next()
.and_then(|s| s.split("video-comments").next())
.and_then(|s| s.split(">Tags<").nth(1))
.map(|tag_block| {
@@ -383,26 +386,34 @@ impl JavtifulProvider {
.split('>')
.nth(1)
.and_then(|s| s.split('<').next())
.map(|s| decode(s.as_bytes()).to_string().unwrap_or(s.to_string()).titlecase())
.map(|s| {
decode(s.as_bytes())
.to_string()
.unwrap_or(s.to_string())
.titlecase()
})
})
.collect()
})
.unwrap_or_else(|| vec![]);
for tag in &tags {
Self::push_unique(&self.categories, FilterOption {
id: tag.to_ascii_lowercase().replace(" ","+"),
title: tag.to_string(),
});
Self::push_unique(
&self.categories,
FilterOption {
id: tag.to_ascii_lowercase().replace(" ", "+"),
title: tag.to_string(),
},
);
}
let views = text.split(" Views ")
let views = text
.split(" Views ")
.next()
.and_then(|s| s.split(" ").last())
.and_then(|s| s.replace(".","")
.parse::<u32>().ok())
.and_then(|s| s.replace(".", "").parse::<u32>().ok())
.unwrap_or(0);
let quality="1080p".to_string();
let video_url = url.replace("javtiful.com","hottub.spacemoehre.de/proxy/javtiful");
let quality = "1080p".to_string();
let video_url = url.replace("javtiful.com", "hottub.spacemoehre.de/proxy/javtiful");
Ok((
tags,
vec![VideoFormat::new(video_url, quality, "video/mp4".into())],

216
src/providers/missav.rs
View File

@@ -1,20 +1,20 @@
use std::vec;
use async_trait::async_trait;
use diesel::r2d2;
use error_chain::error_chain;
use htmlentity::entity::{decode, ICodedDataTrait};
use futures::future::join_all;
use wreq::Version;
use crate::DbPool;
use crate::api::ClientVersion;
use crate::db;
use crate::providers::Provider;
use crate::status::*;
use crate::util::cache::VideoCache;
use crate::util::discord::{format_error_chain, send_discord_error_report};
use crate::videos::ServerOptions;
use crate::videos::{VideoItem};
use crate::DbPool;
use crate::util::requester::Requester;
use crate::videos::ServerOptions;
use crate::videos::VideoItem;
use async_trait::async_trait;
use diesel::r2d2;
use error_chain::error_chain;
use futures::future::join_all;
use htmlentity::entity::{ICodedDataTrait, decode};
use std::vec;
use wreq::Version;
error_chain! {
foreign_links {
@@ -39,7 +39,7 @@ pub struct MissavProvider {
impl MissavProvider {
pub fn new() -> Self {
MissavProvider {
url: "https://missav.ws".to_string()
url: "https://missav.ws".to_string(),
}
}
@@ -177,7 +177,14 @@ impl MissavProvider {
}
}
async fn get(&self, cache: VideoCache, pool: DbPool, page: u8, mut sort: String, options: ServerOptions) -> Result<Vec<VideoItem>> {
async fn get(
&self,
cache: VideoCache,
pool: DbPool,
page: u8,
mut sort: String,
options: ServerOptions,
) -> Result<Vec<VideoItem>> {
// Use ok_or to avoid unwrapping options
let language = options.language.as_ref().ok_or("Missing language")?;
let filter = options.filter.as_ref().ok_or("Missing filter")?;
@@ -187,35 +194,57 @@ impl MissavProvider {
sort = format!("&sort={}", sort);
}
let url_str = format!("{}/{}/{}?page={}{}", self.url, language, filter, page, sort);
if let Some((time, items)) = cache.get(&url_str) {
if time.elapsed().unwrap_or_default().as_secs() < 3600 {
return Ok(items.clone());
}
}
let text = requester.get(&url_str, Some(Version::HTTP_2)).await.unwrap_or_else(|e| {
eprintln!("Error fetching Missav URL {}: {}", url_str, e);
let _ = send_discord_error_report(e.to_string(), None, Some(&url_str), None, file!(), line!(), module_path!());
"".to_string()
});
let text = requester
.get(&url_str, Some(Version::HTTP_2))
.await
.unwrap_or_else(|e| {
eprintln!("Error fetching Missav URL {}: {}", url_str, e);
let _ = send_discord_error_report(
e.to_string(),
None,
Some(&url_str),
None,
file!(),
line!(),
module_path!(),
);
"".to_string()
});
let video_items = self.get_video_items_from_html(text, pool, requester).await;
if !video_items.is_empty() {
cache.insert(url_str, video_items.clone());
}
Ok(video_items)
}
async fn query(&self, cache: VideoCache, pool: DbPool, page: u8, query: &str, mut sort: String, options: ServerOptions) -> Result<Vec<VideoItem>> {
async fn query(
&self,
cache: VideoCache,
pool: DbPool,
page: u8,
query: &str,
mut sort: String,
options: ServerOptions,
) -> Result<Vec<VideoItem>> {
let language = options.language.as_ref().ok_or("Missing language")?;
let mut requester = options.requester.clone().ok_or("Missing requester")?;
let search_string = query.replace(" ", "%20");
if !sort.is_empty() {
sort = format!("&sort={}", sort);
}
let url_str = format!("{}/{}/search/{}?page={}{}", self.url, language, search_string, page, sort);
let url_str = format!(
"{}/{}/search/{}?page={}{}",
self.url, language, search_string, page, sort
);
if let Some((time, items)) = cache.get(&url_str) {
if time.elapsed().unwrap_or_default().as_secs() < 3600 {
@@ -223,24 +252,44 @@ impl MissavProvider {
}
}
let text = requester.get(&url_str, Some(Version::HTTP_2)).await.unwrap_or_else(|e| {
eprintln!("Error fetching Missav URL {}: {}", url_str, e);
let _ = send_discord_error_report(e.to_string(), None, Some(&url_str), None, file!(), line!(), module_path!());
"".to_string()
});
let text = requester
.get(&url_str, Some(Version::HTTP_2))
.await
.unwrap_or_else(|e| {
eprintln!("Error fetching Missav URL {}: {}", url_str, e);
let _ = send_discord_error_report(
e.to_string(),
None,
Some(&url_str),
None,
file!(),
line!(),
module_path!(),
);
"".to_string()
});
let video_items = self.get_video_items_from_html(text, pool, requester).await;
if !video_items.is_empty() {
cache.insert(url_str, video_items.clone());
}
Ok(video_items)
}
async fn get_video_items_from_html(&self, html: String, pool: DbPool, requester: Requester) -> Vec<VideoItem> {
if html.is_empty() { return vec![]; }
async fn get_video_items_from_html(
&self,
html: String,
pool: DbPool,
requester: Requester,
) -> Vec<VideoItem> {
if html.is_empty() {
return vec![];
}
let segments: Vec<&str> = html.split("@mouseenter=\"setPreview(\'").collect();
if segments.len() < 2 { return vec![]; }
if segments.len() < 2 {
return vec![];
}
let mut urls = vec![];
for video_segment in &segments[1..] {
@@ -253,14 +302,27 @@ impl MissavProvider {
}
}
let futures = urls.into_iter().map(|url| self.get_video_item(url, pool.clone(), requester.clone()));
join_all(futures).await.into_iter().filter_map(Result::ok).collect()
let futures = urls
.into_iter()
.map(|url| self.get_video_item(url, pool.clone(), requester.clone()));
join_all(futures)
.await
.into_iter()
.filter_map(Result::ok)
.collect()
}
async fn get_video_item(&self, url_str: String, pool: DbPool, mut requester: Requester) -> Result<VideoItem> {
async fn get_video_item(
&self,
url_str: String,
pool: DbPool,
mut requester: Requester,
) -> Result<VideoItem> {
// 1. Database Check
{
let mut conn = pool.get().map_err(|e| Error::from(format!("Pool error: {}", e)))?;
let mut conn = pool
.get()
.map_err(|e| Error::from(format!("Pool error: {}", e)))?;
if let Ok(Some(entry)) = db::get_video(&mut conn, url_str.clone()) {
if let Ok(video_item) = serde_json::from_str::<VideoItem>(entry.as_str()) {
return Ok(video_item);
@@ -269,11 +331,22 @@ impl MissavProvider {
}
// 2. Fetch Page
let vid = requester.get(&url_str, Some(Version::HTTP_2)).await.unwrap_or_else(|e| {
eprintln!("Error fetching Missav URL {}: {}", url_str, e);
let _ = send_discord_error_report(e.to_string(), None, Some(&url_str), None, file!(), line!(), module_path!());
"".to_string()
});
let vid = requester
.get(&url_str, Some(Version::HTTP_2))
.await
.unwrap_or_else(|e| {
eprintln!("Error fetching Missav URL {}: {}", url_str, e);
let _ = send_discord_error_report(
e.to_string(),
None,
Some(&url_str),
None,
file!(),
line!(),
module_path!(),
);
"".to_string()
});
// Helper closure to extract content between two strings
let extract = |html: &str, start_tag: &str, end_tag: &str| -> Option<String> {
@@ -285,24 +358,33 @@ impl MissavProvider {
let mut title = extract(&vid, "<meta property=\"og:title\" content=\"", "\"")
.ok_or_else(|| ErrorKind::ParsingError(format!("title\n{:?}", vid)))?;
title = decode(title.as_bytes()).to_string().unwrap_or(title);
if url_str.contains("uncensored") {
title = format!("[Uncensored] {}", title);
}
let thumb = extract(&vid, "<meta property=\"og:image\" content=\"", "\"")
.unwrap_or_default();
let thumb =
extract(&vid, "<meta property=\"og:image\" content=\"", "\"").unwrap_or_default();
let duration = extract(&vid, "<meta property=\"og:video:duration\" content=\"", "\"")
.and_then(|d| d.parse::<u32>().ok())
.unwrap_or(0);
let duration = extract(
&vid,
"<meta property=\"og:video:duration\" content=\"",
"\"",
)
.and_then(|d| d.parse::<u32>().ok())
.unwrap_or(0);
let id = url_str.split('/').last().ok_or("No ID found")?.to_string();
// 3. Extract Tags (Generic approach to avoid repetitive code)
let mut tags = vec![];
for (label, prefix) in [("Actress:", "@actress"), ("Actor:", "@actor"), ("Maker:", "@maker"), ("Genre:", "@genre")] {
for (label, prefix) in [
("Actress:", "@actress"),
("Actor:", "@actor"),
("Maker:", "@maker"),
("Genre:", "@genre"),
] {
let marker = format!("<span>{}</span>", label);
if let Some(section) = extract(&vid, &marker, "</div>") {
for part in section.split("class=\"text-nord13 font-medium\">").skip(1) {
@@ -331,15 +413,24 @@ impl MissavProvider {
parts.get(6)?,
parts.get(7)?
))
})().ok_or_else(|| ErrorKind::ParsingError(format!("video_url\n{:?}", vid).to_string()))?;
})()
.ok_or_else(|| ErrorKind::ParsingError(format!("video_url\n{:?}", vid).to_string()))?;
let video_item = VideoItem::new(id, title, video_url, "missav".to_string(), thumb, duration)
.tags(tags)
.preview(format!("https://fourhoi.com/{}/preview.mp4", url_str.split('/').last().unwrap_or_default()));
let video_item =
VideoItem::new(id, title, video_url, "missav".to_string(), thumb, duration)
.tags(tags)
.preview(format!(
"https://fourhoi.com/{}/preview.mp4",
url_str.split('/').last().unwrap_or_default()
));
// 5. Cache to DB
if let Ok(mut conn) = pool.get() {
let _ = db::insert_video(&mut conn, &url_str, &serde_json::to_string(&video_item).unwrap_or_default());
let _ = db::insert_video(
&mut conn,
&url_str,
&serde_json::to_string(&video_item).unwrap_or_default(),
);
}
Ok(video_item)
@@ -348,7 +439,16 @@ impl MissavProvider {
#[async_trait]
impl Provider for MissavProvider {
async fn get_videos(&self, cache: VideoCache, pool: DbPool, sort: String, query: Option<String>, page: String, _per_page: String, options: ServerOptions) -> Vec<VideoItem> {
async fn get_videos(
&self,
cache: VideoCache,
pool: DbPool,
sort: String,
query: Option<String>,
page: String,
_per_page: String,
options: ServerOptions,
) -> Vec<VideoItem> {
let page_num = page.parse::<u8>().unwrap_or(1);
let result = match query {
Some(q) => self.query(cache, pool, page_num, &q, sort, options).await,
@@ -357,7 +457,15 @@ impl Provider for MissavProvider {
result.unwrap_or_else(|e| {
eprintln!("Error fetching videos: {}", e);
let _ = send_discord_error_report(e.to_string(), Some(format_error_chain(&e)), None, None, file!(), line!(), module_path!());
let _ = send_discord_error_report(
e.to_string(),
Some(format_error_chain(&e)),
None,
None,
file!(),
line!(),
module_path!(),
);
vec![]
})
}

174
src/providers/mod.rs
View File

@@ -7,7 +7,11 @@ use std::panic::AssertUnwindSafe;
use std::sync::Arc;
use crate::{
DbPool, api::ClientVersion, status::Channel, util::{cache::VideoCache, discord::send_discord_error_report, requester::Requester}, videos::{ServerOptions, VideoItem}
DbPool,
api::ClientVersion,
status::Channel,
util::{cache::VideoCache, discord::send_discord_error_report, requester::Requester},
videos::{ServerOptions, VideoItem},
};
pub mod all;
@@ -24,27 +28,27 @@ pub mod pornhat;
pub mod redtube;
pub mod rule34video;
// pub mod hentaimoon;
pub mod beeg;
pub mod missav;
pub mod porn00;
pub mod sxyprn;
pub mod xxthots;
pub mod omgxxx;
pub mod paradisehill;
pub mod porn00;
pub mod pornzog;
pub mod youjizz;
pub mod beeg;
pub mod sxyprn;
pub mod tnaflix;
pub mod xxthots;
pub mod youjizz;
// pub mod pornxp;
pub mod rule34gen;
pub mod xxdbx;
pub mod hqporner;
pub mod noodlemagazine;
pub mod pimpbunny;
pub mod javtiful;
pub mod hypnotube;
pub mod chaturbate;
pub mod freepornvideosxxx;
pub mod hentaihaven;
pub mod chaturbate;
pub mod hqporner;
pub mod hypnotube;
pub mod javtiful;
pub mod noodlemagazine;
pub mod pimpbunny;
pub mod rule34gen;
pub mod xxdbx;
// pub mod tube8;
// convenient alias
@@ -53,47 +57,128 @@ pub type DynProvider = Arc<dyn Provider>;
pub static ALL_PROVIDERS: Lazy<HashMap<&'static str, DynProvider>> = Lazy::new(|| {
let mut m = HashMap::default();
m.insert("all", Arc::new(all::AllProvider::new()) as DynProvider);
m.insert("perverzija", Arc::new(perverzija::PerverzijaProvider::new()) as DynProvider);
m.insert("hanime", Arc::new(hanime::HanimeProvider::new()) as DynProvider);
m.insert("pornhub", Arc::new(pornhub::PornhubProvider::new()) as DynProvider);
m.insert(
"perverzija",
Arc::new(perverzija::PerverzijaProvider::new()) as DynProvider,
);
m.insert(
"hanime",
Arc::new(hanime::HanimeProvider::new()) as DynProvider,
);
m.insert(
"pornhub",
Arc::new(pornhub::PornhubProvider::new()) as DynProvider,
);
m.insert(
"rule34video",
Arc::new(rule34video::Rule34videoProvider::new()) as DynProvider,
);
m.insert("redtube", Arc::new(redtube::RedtubeProvider::new()) as DynProvider);
m.insert("okporn", Arc::new(okporn::OkpornProvider::new()) as DynProvider);
m.insert("pornhat", Arc::new(pornhat::PornhatProvider::new()) as DynProvider);
m.insert(
"redtube",
Arc::new(redtube::RedtubeProvider::new()) as DynProvider,
);
m.insert(
"okporn",
Arc::new(okporn::OkpornProvider::new()) as DynProvider,
);
m.insert(
"pornhat",
Arc::new(pornhat::PornhatProvider::new()) as DynProvider,
);
m.insert(
"perfectgirls",
Arc::new(perfectgirls::PerfectgirlsProvider::new()) as DynProvider,
);
m.insert("okxxx", Arc::new(okxxx::OkxxxProvider::new()) as DynProvider);
m.insert("homoxxx", Arc::new(homoxxx::HomoxxxProvider::new()) as DynProvider);
m.insert("missav", Arc::new(missav::MissavProvider::new()) as DynProvider);
m.insert("xxthots", Arc::new(xxthots::XxthotsProvider::new()) as DynProvider);
m.insert("sxyprn", Arc::new(sxyprn::SxyprnProvider::new()) as DynProvider);
m.insert("porn00", Arc::new(porn00::Porn00Provider::new()) as DynProvider);
m.insert("youjizz", Arc::new(youjizz::YoujizzProvider::new()) as DynProvider);
m.insert(
"okxxx",
Arc::new(okxxx::OkxxxProvider::new()) as DynProvider,
);
m.insert(
"homoxxx",
Arc::new(homoxxx::HomoxxxProvider::new()) as DynProvider,
);
m.insert(
"missav",
Arc::new(missav::MissavProvider::new()) as DynProvider,
);
m.insert(
"xxthots",
Arc::new(xxthots::XxthotsProvider::new()) as DynProvider,
);
m.insert(
"sxyprn",
Arc::new(sxyprn::SxyprnProvider::new()) as DynProvider,
);
m.insert(
"porn00",
Arc::new(porn00::Porn00Provider::new()) as DynProvider,
);
m.insert(
"youjizz",
Arc::new(youjizz::YoujizzProvider::new()) as DynProvider,
);
m.insert(
"paradisehill",
Arc::new(paradisehill::ParadisehillProvider::new()) as DynProvider,
);
m.insert("pornzog", Arc::new(pornzog::PornzogProvider::new()) as DynProvider);
m.insert("omgxxx", Arc::new(omgxxx::OmgxxxProvider::new()) as DynProvider);
m.insert(
"pornzog",
Arc::new(pornzog::PornzogProvider::new()) as DynProvider,
);
m.insert(
"omgxxx",
Arc::new(omgxxx::OmgxxxProvider::new()) as DynProvider,
);
m.insert("beeg", Arc::new(beeg::BeegProvider::new()) as DynProvider);
m.insert("tnaflix", Arc::new(tnaflix::TnaflixProvider::new()) as DynProvider);
m.insert(
"tnaflix",
Arc::new(tnaflix::TnaflixProvider::new()) as DynProvider,
);
// m.insert("pornxp", Arc::new(pornxp::PornxpProvider::new()) as DynProvider);
m.insert("rule34gen", Arc::new(rule34gen::Rule34genProvider::new()) as DynProvider);
m.insert("xxdbx", Arc::new(xxdbx::XxdbxProvider::new()) as DynProvider);
m.insert("hqporner", Arc::new(hqporner::HqpornerProvider::new()) as DynProvider);
m.insert("pmvhaven", Arc::new(pmvhaven::PmvhavenProvider::new()) as DynProvider);
m.insert("noodlemagazine", Arc::new(noodlemagazine::NoodlemagazineProvider::new()) as DynProvider);
m.insert("pimpbunny", Arc::new(pimpbunny::PimpbunnyProvider::new()) as DynProvider);
m.insert("javtiful", Arc::new(javtiful::JavtifulProvider::new()) as DynProvider);
m.insert("hypnotube", Arc::new(hypnotube::HypnotubeProvider::new()) as DynProvider);
m.insert("freepornvideosxxx", Arc::new(freepornvideosxxx::FreepornvideosxxxProvider::new()) as DynProvider);
m.insert("hentaihaven", Arc::new(hentaihaven::HentaihavenProvider::new()) as DynProvider);
m.insert("chaturbate", Arc::new(chaturbate::ChaturbateProvider::new()) as DynProvider);
m.insert(
"rule34gen",
Arc::new(rule34gen::Rule34genProvider::new()) as DynProvider,
);
m.insert(
"xxdbx",
Arc::new(xxdbx::XxdbxProvider::new()) as DynProvider,
);
m.insert(
"hqporner",
Arc::new(hqporner::HqpornerProvider::new()) as DynProvider,
);
m.insert(
"pmvhaven",
Arc::new(pmvhaven::PmvhavenProvider::new()) as DynProvider,
);
m.insert(
"noodlemagazine",
Arc::new(noodlemagazine::NoodlemagazineProvider::new()) as DynProvider,
);
m.insert(
"pimpbunny",
Arc::new(pimpbunny::PimpbunnyProvider::new()) as DynProvider,
);
m.insert(
"javtiful",
Arc::new(javtiful::JavtifulProvider::new()) as DynProvider,
);
m.insert(
"hypnotube",
Arc::new(hypnotube::HypnotubeProvider::new()) as DynProvider,
);
m.insert(
"freepornvideosxxx",
Arc::new(freepornvideosxxx::FreepornvideosxxxProvider::new()) as DynProvider,
);
m.insert(
"hentaihaven",
Arc::new(hentaihaven::HentaihavenProvider::new()) as DynProvider,
);
m.insert(
"chaturbate",
Arc::new(chaturbate::ChaturbateProvider::new()) as DynProvider,
);
// m.insert("tube8", Arc::new(tube8::Tube8Provider::new()) as DynProvider);
// add more here as you migrate them
m
@@ -159,7 +244,11 @@ pub fn report_provider_error_background(provider_name: &str, context: &str, msg:
});
}
pub fn requester_or_default(options: &ServerOptions, provider_name: &str, context: &str) -> Requester {
pub fn requester_or_default(
options: &ServerOptions,
provider_name: &str,
context: &str,
) -> Requester {
match options.requester.clone() {
Some(requester) => requester,
None => {
@@ -173,7 +262,6 @@ pub fn requester_or_default(options: &ServerOptions, provider_name: &str, contex
}
}
#[async_trait]
pub trait Provider: Send + Sync {
async fn get_videos(

34
src/providers/noodlemagazine.rs
View File

@@ -3,18 +3,18 @@ use crate::api::ClientVersion;
use crate::providers::Provider;
use crate::status::*;
use crate::util::cache::VideoCache;
use crate::util::requester::Requester;
use crate::util::parse_abbreviated_number;
use crate::util::requester::Requester;
use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoFormat, VideoItem};
use async_trait::async_trait;
use error_chain::error_chain;
use futures::future::join_all;
use htmlentity::entity::{decode, ICodedDataTrait};
use wreq::Version;
use titlecase::Titlecase;
use htmlentity::entity::{ICodedDataTrait, decode};
use std::vec;
use titlecase::Titlecase;
use wreq::Version;
error_chain! {
foreign_links {
@@ -223,18 +223,16 @@ impl NoodlemagazineProvider {
.await
.ok_or_else(|| Error::from("media extraction failed"))?;
Ok(
VideoItem::new(
id,
title,
video_url,
"noodlemagazine".into(),
thumb,
duration,
)
.views(views)
.formats(formats),
Ok(VideoItem::new(
id,
title,
video_url,
"noodlemagazine".into(),
thumb,
duration,
)
.views(views)
.formats(formats))
}
async fn extract_media(
@@ -247,11 +245,7 @@ impl NoodlemagazineProvider {
.await
.unwrap_or_default();
let json_str = text
.split("window.playlist = ")
.nth(1)?
.split(';')
.next()?;
let json_str = text.split("window.playlist = ").nth(1)?.split(';').next()?;
let json: serde_json::Value = serde_json::from_str(json_str).ok()?;
let sources = json["sources"].as_array()?;

156
src/providers/okporn.rs
View File

@@ -1,18 +1,18 @@
use crate::api::ClientVersion;
use crate::DbPool;
use crate::api::ClientVersion;
use crate::providers::{Provider, report_provider_error};
use crate::status::*;
use crate::util::cache::VideoCache;
use crate::util::flaresolverr::{FlareSolverrRequest, Flaresolverr};
use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoItem};
use async_trait::async_trait;
use error_chain::error_chain;
use htmlentity::entity::{ICodedDataTrait, decode};
use std::env;
use std::vec;
use wreq::{Client};
use wreq::Client;
use wreq_util::Emulation;
use async_trait::async_trait;
error_chain! {
foreign_links {
@@ -67,12 +67,7 @@ impl OkpornProvider {
cacheDuration: Some(1800),
}
}
async fn get(
&self,
cache: VideoCache,
page: u8,
sort: &str,
) -> Result<Vec<VideoItem>> {
async fn get(&self, cache: VideoCache, page: u8, sort: &str) -> Result<Vec<VideoItem>> {
let sort_string = match sort {
"trending" => "/trending",
"popular" => "/popular",
@@ -93,13 +88,22 @@ impl OkpornProvider {
};
// let proxy = Proxy::all("http://192.168.0.103:8081").unwrap();
let client = Client::builder().cert_verification(false).emulation(Emulation::Firefox136).build()?;
let client = Client::builder()
.cert_verification(false)
.emulation(Emulation::Firefox136)
.build()?;
let mut response = client.get(video_url.clone())
// .proxy(proxy.clone())
.send().await?;
let mut response = client
.get(video_url.clone())
// .proxy(proxy.clone())
.send()
.await?;
if response.status().is_redirection() {
let location = match response.headers().get("Location").and_then(|h| h.to_str().ok()) {
let location = match response
.headers()
.get("Location")
.and_then(|h| h.to_str().ok())
{
Some(location) => location,
None => {
report_provider_error(
@@ -131,12 +135,7 @@ impl OkpornProvider {
let flare_url = match env::var("FLARE_URL") {
Ok(url) => url,
Err(e) => {
report_provider_error(
"okporn",
"get.flare_url",
&e.to_string(),
)
.await;
report_provider_error("okporn", "get.flare_url", &e.to_string()).await;
return Ok(old_items);
}
};
@@ -167,12 +166,7 @@ impl OkpornProvider {
Ok(video_items)
}
}
async fn query(
&self,
cache: VideoCache,
page: u8,
query: &str,
) -> Result<Vec<VideoItem>> {
async fn query(&self, cache: VideoCache, page: u8, query: &str) -> Result<Vec<VideoItem>> {
let search_string = query.to_lowercase().trim().replace(" ", "-");
let video_url = format!("{}/search/{}/{}/", self.url, search_string, page);
@@ -192,14 +186,23 @@ impl OkpornProvider {
};
// let proxy = Proxy::all("http://192.168.0.103:8081").unwrap();
let client = Client::builder().cert_verification(false).emulation(Emulation::Firefox136).build()?;
let client = Client::builder()
.cert_verification(false)
.emulation(Emulation::Firefox136)
.build()?;
let mut response = client.get(video_url.clone())
// .proxy(proxy.clone())
.send().await?;
let mut response = client
.get(video_url.clone())
// .proxy(proxy.clone())
.send()
.await?;
if response.status().is_redirection() {
let location = match response.headers().get("Location").and_then(|h| h.to_str().ok()) {
let location = match response
.headers()
.get("Location")
.and_then(|h| h.to_str().ok())
{
Some(location) => location,
None => {
report_provider_error(
@@ -232,12 +235,7 @@ impl OkpornProvider {
let flare_url = match env::var("FLARE_URL") {
Ok(url) => url,
Err(e) => {
report_provider_error(
"okporn",
"query.flare_url",
&e.to_string(),
)
.await;
report_provider_error("okporn", "query.flare_url", &e.to_string()).await;
return Ok(old_items);
}
};
@@ -272,34 +270,78 @@ impl OkpornProvider {
return vec![];
}
let mut items: Vec<VideoItem> = Vec::new();
let raw_videos = html
.split("<div class=\"item ")
.collect::<Vec<&str>>()[1..]
.to_vec();
let raw_videos = html.split("<div class=\"item ").collect::<Vec<&str>>()[1..].to_vec();
for video_segment in &raw_videos {
// let vid = video_segment.split("\n").collect::<Vec<&str>>();
// for (index, line) in vid.iter().enumerate() {
// println!("Line {}: {}", index, line);
// }
let video_url: String = format!("{}{}", self.url, video_segment.split("<a href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
let video_url: String = format!(
"{}{}",
self.url,
video_segment
.split("<a href=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default());
let mut title = video_segment.split("\" title=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
);
let mut title = video_segment
.split("\" title=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
// html decode
title = decode(title.as_bytes()).to_string().unwrap_or(title);
let id = video_url.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string();
let raw_duration = video_segment.split("<span class=\"duration_item\">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
let id = video_url
.split("/")
.collect::<Vec<&str>>()
.get(4)
.copied()
.unwrap_or_default()
.to_string();
let raw_duration = video_segment
.split("<span class=\"duration_item\">")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
let thumb = video_segment.split("<img class=\"thumb lazy-load\" src=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default().split("data-original=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
let thumb = video_segment
.split("<img class=\"thumb lazy-load\" src=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("data-original=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
let video_item = VideoItem::new(
@@ -309,14 +351,11 @@ impl OkpornProvider {
"okporn".to_string(),
thumb,
duration,
)
;
);
items.push(video_item);
}
return items;
}
}
#[async_trait]
@@ -335,10 +374,7 @@ impl Provider for OkpornProvider {
let _ = per_page;
let _ = pool;
let videos: std::result::Result<Vec<VideoItem>, Error> = match query {
Some(q) => {
self.query(cache, page.parse::<u8>().unwrap_or(1), &q,)
.await
}
Some(q) => self.query(cache, page.parse::<u8>().unwrap_or(1), &q).await,
None => {
self.get(cache, page.parse::<u8>().unwrap_or(1), &sort)
.await

242
src/providers/okxxx.rs
View File

@@ -1,19 +1,19 @@
use crate::api::ClientVersion;
use crate::util::parse_abbreviated_number;
use crate::DbPool;
use crate::api::ClientVersion;
use crate::providers::{Provider, report_provider_error};
use crate::status::*;
use crate::util::cache::VideoCache;
use crate::util::flaresolverr::{FlareSolverrRequest, Flaresolverr};
use crate::util::parse_abbreviated_number;
use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoItem};
use async_trait::async_trait;
use error_chain::error_chain;
use htmlentity::entity::{ICodedDataTrait, decode};
use std::env;
use std::vec;
use wreq::{Client};
use wreq::Client;
use wreq_util::Emulation;
use async_trait::async_trait;
error_chain! {
foreign_links {
@@ -68,12 +68,7 @@ impl OkxxxProvider {
cacheDuration: Some(1800),
}
}
async fn get(
&self,
cache: VideoCache,
page: u8,
sort: &str,
) -> Result<Vec<VideoItem>> {
async fn get(&self, cache: VideoCache, page: u8, sort: &str) -> Result<Vec<VideoItem>> {
let sort_string = match sort {
"trending" => "/trending",
"popular" => "/popular",
@@ -94,13 +89,22 @@ impl OkxxxProvider {
};
// let proxy = Proxy::all("http://192.168.0.103:8081").unwrap();
let client = Client::builder().cert_verification(false).emulation(Emulation::Firefox136).build()?;
let client = Client::builder()
.cert_verification(false)
.emulation(Emulation::Firefox136)
.build()?;
let mut response = client.get(video_url.clone())
// .proxy(proxy.clone())
.send().await?;
let mut response = client
.get(video_url.clone())
// .proxy(proxy.clone())
.send()
.await?;
if response.status().is_redirection() {
let location = match response.headers().get("Location").and_then(|h| h.to_str().ok()) {
let location = match response
.headers()
.get("Location")
.and_then(|h| h.to_str().ok())
{
Some(location) => location,
None => {
report_provider_error(
@@ -133,12 +137,7 @@ impl OkxxxProvider {
let flare_url = match env::var("FLARE_URL") {
Ok(url) => url,
Err(e) => {
report_provider_error(
"okxxx",
"get.flare_url",
&e.to_string(),
)
.await;
report_provider_error("okxxx", "get.flare_url", &e.to_string()).await;
return Ok(old_items);
}
};
@@ -169,17 +168,18 @@ impl OkxxxProvider {
Ok(video_items)
}
}
async fn query(
&self,
cache: VideoCache,
page: u8,
query: &str,
) -> Result<Vec<VideoItem>> {
async fn query(&self, cache: VideoCache, page: u8, query: &str) -> Result<Vec<VideoItem>> {
let search_string = query.to_lowercase().trim().replace(" ", "-");
let mut video_url = format!("{}/search/{}/{}/", self.url, search_string, page);
if search_string.starts_with("@"){
let url_part = search_string.split("@").collect::<Vec<&str>>().get(1).copied().unwrap_or_default().replace(":", "/");
if search_string.starts_with("@") {
let url_part = search_string
.split("@")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.replace(":", "/");
video_url = format!("{}/{}/", self.url, url_part);
}
// Check our Video Cache. If the result is younger than 1 hour, we return it.
@@ -198,14 +198,23 @@ impl OkxxxProvider {
};
// let proxy = Proxy::all("http://192.168.0.103:8081").unwrap();
let client = Client::builder().cert_verification(false).emulation(Emulation::Firefox136).build()?;
let client = Client::builder()
.cert_verification(false)
.emulation(Emulation::Firefox136)
.build()?;
let mut response = client.get(video_url.clone())
// .proxy(proxy.clone())
.send().await?;
let mut response = client
.get(video_url.clone())
// .proxy(proxy.clone())
.send()
.await?;
if response.status().is_redirection() {
let location = match response.headers().get("Location").and_then(|h| h.to_str().ok()) {
let location = match response
.headers()
.get("Location")
.and_then(|h| h.to_str().ok())
{
Some(location) => location,
None => {
report_provider_error(
@@ -238,12 +247,7 @@ impl OkxxxProvider {
let flare_url = match env::var("FLARE_URL") {
Ok(url) => url,
Err(e) => {
report_provider_error(
"okxxx",
"query.flare_url",
&e.to_string(),
)
.await;
report_provider_error("okxxx", "query.flare_url", &e.to_string()).await;
return Ok(old_items);
}
};
@@ -278,7 +282,12 @@ impl OkxxxProvider {
return vec![];
}
let mut items: Vec<VideoItem> = Vec::new();
let raw_videos = html.split("<div class=\"pagination\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
let raw_videos = html
.split("<div class=\"pagination\"")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.split("item thumb-bl thumb-bl-video video_")
.collect::<Vec<&str>>()[1..]
.to_vec();
@@ -287,61 +296,150 @@ impl OkxxxProvider {
// for (index, line) in vid.iter().enumerate() {
// println!("Line {}: {}", index, line);
// }
let video_url: String = format!("{}{}", self.url, video_segment.split("<a href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
let video_url: String = format!(
"{}{}",
self.url,
video_segment
.split("<a href=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default());
let preview_url = video_segment.split("data-preview-custom=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
);
let preview_url = video_segment
.split("data-preview-custom=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
let mut title = video_segment.split("\" title=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
let mut title = video_segment
.split("\" title=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
// html decode
title = decode(title.as_bytes()).to_string().unwrap_or(title);
let id = video_url.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string();
let raw_duration = video_segment.split("fa fa-clock-o").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<span>").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
let id = video_url
.split("/")
.collect::<Vec<&str>>()
.get(4)
.copied()
.unwrap_or_default()
.to_string();
let raw_duration = video_segment
.split("fa fa-clock-o")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<span>")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
let thumb = format!("https:{}", video_segment.split(" class=\"thumb lazy-load\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("data-original=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string());
let thumb = format!(
"https:{}",
video_segment
.split(" class=\"thumb lazy-load\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("data-original=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string()
);
let mut tags = vec![];
if video_segment.contains("href=\"/sites/"){
if video_segment.contains("href=\"/sites/") {
let raw_tags = video_segment.split("href=\"/sites/").collect::<Vec<&str>>()[1..]
.iter()
.map(|s| s.split("/\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default().to_string())
.map(|s| {
s.split("/\"")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string()
})
.collect::<Vec<String>>();
for tag in raw_tags {
if !tag.is_empty() {
tags.push(format!("@sites:{}",tag));
tags.push(format!("@sites:{}", tag));
}
}
}
if video_segment.contains("href=\"/models/"){
let raw_tags = video_segment.split("href=\"/models/").collect::<Vec<&str>>()[1..]
if video_segment.contains("href=\"/models/") {
let raw_tags = video_segment
.split("href=\"/models/")
.collect::<Vec<&str>>()[1..]
.iter()
.map(|s| s.split("/\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default().to_string())
.map(|s| {
s.split("/\"")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string()
})
.collect::<Vec<String>>();
for tag in raw_tags {
if !tag.is_empty() {
tags.push(format!("@models:{}",tag));
tags.push(format!("@models:{}", tag));
}
}
}
let views_part = video_segment.split("fa fa-eye").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<span>").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
let views_part = video_segment
.split("fa fa-eye")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<span>")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
let views = parse_abbreviated_number(&views_part).unwrap_or(0) as u32;
@@ -355,14 +453,11 @@ impl OkxxxProvider {
)
.preview(preview_url)
.views(views)
.tags(tags)
;
.tags(tags);
items.push(video_item);
}
return items;
}
}
#[async_trait]
@@ -381,10 +476,7 @@ impl Provider for OkxxxProvider {
let _ = per_page;
let _ = pool;
let videos: std::result::Result<Vec<VideoItem>, Error> = match query {
Some(q) => {
self.query(cache, page.parse::<u8>().unwrap_or(1), &q,)
.await
}
Some(q) => self.query(cache, page.parse::<u8>().unwrap_or(1), &q).await,
None => {
self.get(cache, page.parse::<u8>().unwrap_or(1), &sort)
.await

294
src/providers/omgxxx.rs
View File

@@ -120,17 +120,40 @@ impl OmgxxxProvider {
.copied()
.unwrap_or_default()
.split("custom_list_models_models_list_pagination")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default();
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default();
for stars_element in stars_div.split("<a ").collect::<Vec<&str>>()[1..].to_vec() {
let star_url = stars_element.split("href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
let star_url = stars_element
.split("href=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default();
let star_id = star_url.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string();
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default();
let star_id = star_url
.split("/")
.collect::<Vec<&str>>()
.get(4)
.copied()
.unwrap_or_default()
.to_string();
let star_name = stars_element
.split("<strong class=\"title\">")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
Self::push_unique(
&stars,
@@ -168,19 +191,47 @@ impl OmgxxxProvider {
}
let sites_div = text
.split("id=\"list_content_sources_sponsors_list_items\"")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("class=\"pagination\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default();
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default();
for sites_element in
sites_div.split("class=\"headline\"").collect::<Vec<&str>>()[1..].to_vec()
{
let site_url = sites_element.split("href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
let site_url = sites_element
.split("href=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default();
let site_id = site_url.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string();
let site_name = sites_element.split("<h2>").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default();
let site_id = site_url
.split("/")
.collect::<Vec<&str>>()
.get(4)
.copied()
.unwrap_or_default()
.to_string();
let site_name = sites_element
.split("<h2>")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
Self::push_unique(
&sites,
@@ -207,22 +258,52 @@ impl OmgxxxProvider {
return Ok(());
}
};
let networks_div = text.split("class=\"sites__list\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
let networks_div = text
.split("class=\"sites__list\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("</div>")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default();
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default();
for network_element in
networks_div.split("sites__item").collect::<Vec<&str>>()[1..].to_vec()
{
if network_element.contains("sites__all") {
continue;
}
let network_url = network_element.split("href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
let network_url = network_element
.split("href=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default();
let network_id = network_url.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string();
let network_name = network_element.split(">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default();
let network_id = network_url
.split("/")
.collect::<Vec<&str>>()
.get(4)
.copied()
.unwrap_or_default()
.to_string();
let network_name = network_element
.split(">")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
Self::push_unique(
&networks,
@@ -420,11 +501,7 @@ impl OmgxxxProvider {
}
}
Err(e) => {
report_provider_error_background(
"omgxxx",
"query.stars_read",
&e.to_string(),
);
report_provider_error_background("omgxxx", "query.stars_read", &e.to_string());
}
}
match self.sites.read() {
@@ -438,11 +515,7 @@ impl OmgxxxProvider {
}
}
Err(e) => {
report_provider_error_background(
"omgxxx",
"query.sites_read",
&e.to_string(),
);
report_provider_error_background("omgxxx", "query.sites_read", &e.to_string());
}
}
let mut video_url = format!("{}/{}/{}/{}/", self.url, search_type, search_string, page);
@@ -522,11 +595,22 @@ impl OmgxxxProvider {
if !html.contains("class=\"item\"") {
return items;
}
let raw_videos = html.split("videos_list_pagination").collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
let raw_videos = html
.split("videos_list_pagination")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.split(" class=\"pagination\" ")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.split("class=\"list-videos\"")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("class=\"item\"")
.collect::<Vec<&str>>()[1..]
.to_vec();
@@ -535,39 +619,94 @@ impl OmgxxxProvider {
// for (index, line) in vid.iter().enumerate() {
// println!("Line {}: {}", index, line);
// }
let video_url: String = video_segment.split("<a href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
let video_url: String = video_segment
.split("<a href=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
let mut title = video_segment.split(" title=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
let mut title = video_segment
.split(" title=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
// html decode
title = decode(title.as_bytes()).to_string().unwrap_or(title);
let id = video_url.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string();
let id = video_url
.split("/")
.collect::<Vec<&str>>()
.get(4)
.copied()
.unwrap_or_default()
.to_string();
let thumb = match video_segment.split("img loading").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
let thumb = match video_segment
.split("img loading")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.contains("data-src=\"")
{
true => video_segment.split("img loading").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
true => video_segment
.split("img loading")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("data-src=\"")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(),
false => video_segment.split("img loading").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
false => video_segment
.split("img loading")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("data-original=\"")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string(),
};
let raw_duration = video_segment
.split("<span class=\"duration\">")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.split(" ")
.collect::<Vec<&str>>()
.last()
@@ -577,9 +716,15 @@ impl OmgxxxProvider {
let views = parse_abbreviated_number(
video_segment
.split("<div class=\"views\">")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string()
.as_str(),
)
@@ -587,9 +732,15 @@ impl OmgxxxProvider {
let preview = video_segment
.split("data-preview=\"")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
let site_name = title
.split("]")
@@ -603,9 +754,15 @@ impl OmgxxxProvider {
let mut tags = match video_segment.contains("class=\"models\">") {
true => video_segment
.split("class=\"models\">")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("</div>")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.split("href=\"")
.collect::<Vec<&str>>()[1..]
.into_iter()
@@ -613,17 +770,38 @@ impl OmgxxxProvider {
Self::push_unique(
&self.stars,
FilterOption {
id: s.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string(),
title: s.split(">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
id: s
.split("/")
.collect::<Vec<&str>>()
.get(4)
.copied()
.unwrap_or_default()
.to_string(),
title: s
.split(">")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.trim()
.to_string(),
},
);
s.split(">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
s.split(">")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.trim()
.to_string()
})

22
src/providers/paradisehill.rs
View File

@@ -1,14 +1,14 @@
use crate::api::ClientVersion;
use crate::DbPool;
use crate::api::ClientVersion;
use crate::providers::Provider;
use crate::status::*;
use crate::util::cache::VideoCache;
use crate::util::requester::Requester;
use crate::videos::VideoItem;
use crate::videos::ServerOptions;
use crate::videos::VideoItem;
use async_trait::async_trait;
use error_chain::error_chain;
use htmlentity::entity::{ICodedDataTrait, decode};
use async_trait::async_trait;
error_chain! {
foreign_links {
@@ -35,7 +35,8 @@ impl ParadisehillProvider {
name: "Paradisehill".to_string(),
description: "Porn Movies on Paradise Hill".to_string(),
premium: false,
favicon: "https://www.google.com/s2/favicons?sz=64&domain=en.paradisehill.cc".to_string(),
favicon: "https://www.google.com/s2/favicons?sz=64&domain=en.paradisehill.cc"
.to_string(),
status: "active".to_string(),
categories: vec![],
options: vec![],
@@ -49,7 +50,8 @@ impl ParadisehillProvider {
page: u8,
options: ServerOptions,
) -> Result<Vec<VideoItem>> {
let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let url_str = format!("{}/all/?sort=created_at&page={}", self.url, page);
@@ -99,7 +101,8 @@ impl ParadisehillProvider {
options: ServerOptions,
) -> Result<Vec<VideoItem>> {
// Extract needed fields from options at the start
let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let search_string = query.replace(" ", "+");
let url_str = format!(
"{}/search/?pattern={}&page={}",
@@ -200,7 +203,11 @@ impl ParadisehillProvider {
.unwrap_or_default()
.trim()
.to_string();
let tags = if genre.is_empty() { vec![] } else { vec![genre] };
let tags = if genre.is_empty() {
vec![]
} else {
vec![genre]
};
items.push(
VideoItem::new(id, title, video_url, "paradisehill".to_string(), thumb, 0)
@@ -211,7 +218,6 @@ impl ParadisehillProvider {
items
}
}
#[async_trait]

243
src/providers/perfectgirls.rs
View File

@@ -1,19 +1,19 @@
use crate::api::ClientVersion;
use crate::util::parse_abbreviated_number;
use crate::DbPool;
use crate::api::ClientVersion;
use crate::providers::{Provider, report_provider_error};
use crate::status::*;
use crate::util::cache::VideoCache;
use crate::util::flaresolverr::{FlareSolverrRequest, Flaresolverr};
use crate::util::parse_abbreviated_number;
use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoItem};
use async_trait::async_trait;
use error_chain::error_chain;
use htmlentity::entity::{ICodedDataTrait, decode};
use std::env;
use std::vec;
use wreq::{Client};
use wreq::Client;
use wreq_util::Emulation;
use async_trait::async_trait;
error_chain! {
foreign_links {
@@ -68,12 +68,7 @@ impl PerfectgirlsProvider {
cacheDuration: Some(1800),
}
}
async fn get(
&self,
cache: VideoCache,
page: u8,
sort: &str,
) -> Result<Vec<VideoItem>> {
async fn get(&self, cache: VideoCache, page: u8, sort: &str) -> Result<Vec<VideoItem>> {
let sort_string = match sort {
"trending" => "/trending",
"popular" => "/popular",
@@ -94,13 +89,22 @@ impl PerfectgirlsProvider {
};
// let proxy = Proxy::all("http://192.168.0.103:8081").unwrap();
let client = Client::builder().cert_verification(false).emulation(Emulation::Firefox136).build()?;
let client = Client::builder()
.cert_verification(false)
.emulation(Emulation::Firefox136)
.build()?;
let mut response = client.get(video_url.clone())
// .proxy(proxy.clone())
.send().await?;
let mut response = client
.get(video_url.clone())
// .proxy(proxy.clone())
.send()
.await?;
if response.status().is_redirection() {
let location = match response.headers().get("Location").and_then(|h| h.to_str().ok()) {
let location = match response
.headers()
.get("Location")
.and_then(|h| h.to_str().ok())
{
Some(location) => location,
None => {
report_provider_error(
@@ -133,12 +137,7 @@ impl PerfectgirlsProvider {
let flare_url = match env::var("FLARE_URL") {
Ok(url) => url,
Err(e) => {
report_provider_error(
"perfectgirls",
"get.flare_url",
&e.to_string(),
)
.await;
report_provider_error("perfectgirls", "get.flare_url", &e.to_string()).await;
return Ok(old_items);
}
};
@@ -169,17 +168,18 @@ impl PerfectgirlsProvider {
Ok(video_items)
}
}
async fn query(
&self,
cache: VideoCache,
page: u8,
query: &str,
) -> Result<Vec<VideoItem>> {
async fn query(&self, cache: VideoCache, page: u8, query: &str) -> Result<Vec<VideoItem>> {
let search_string = query.to_lowercase().trim().replace(" ", "-");
let mut video_url = format!("{}/search/{}/{}/", self.url, search_string, page);
if search_string.starts_with("@"){
let url_part = search_string.split("@").collect::<Vec<&str>>().get(1).copied().unwrap_or_default().replace(":", "/");
if search_string.starts_with("@") {
let url_part = search_string
.split("@")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.replace(":", "/");
video_url = format!("{}/{}/", self.url, url_part);
}
// Check our Video Cache. If the result is younger than 1 hour, we return it.
@@ -198,14 +198,23 @@ impl PerfectgirlsProvider {
};
// let proxy = Proxy::all("http://192.168.0.103:8081").unwrap();
let client = Client::builder().cert_verification(false).emulation(Emulation::Firefox136).build()?;
let client = Client::builder()
.cert_verification(false)
.emulation(Emulation::Firefox136)
.build()?;
let mut response = client.get(video_url.clone())
// .proxy(proxy.clone())
.send().await?;
let mut response = client
.get(video_url.clone())
// .proxy(proxy.clone())
.send()
.await?;
if response.status().is_redirection() {
let location = match response.headers().get("Location").and_then(|h| h.to_str().ok()) {
let location = match response
.headers()
.get("Location")
.and_then(|h| h.to_str().ok())
{
Some(location) => location,
None => {
report_provider_error(
@@ -238,12 +247,7 @@ impl PerfectgirlsProvider {
let flare_url = match env::var("FLARE_URL") {
Ok(url) => url,
Err(e) => {
report_provider_error(
"perfectgirls",
"query.flare_url",
&e.to_string(),
)
.await;
report_provider_error("perfectgirls", "query.flare_url", &e.to_string()).await;
return Ok(old_items);
}
};
@@ -278,7 +282,12 @@ impl PerfectgirlsProvider {
return vec![];
}
let mut items: Vec<VideoItem> = Vec::new();
let raw_videos = html.split("<div class=\"pagination\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
let raw_videos = html
.split("<div class=\"pagination\"")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.split("item thumb-bl thumb-bl-video video_")
.collect::<Vec<&str>>()[1..]
.to_vec();
@@ -287,64 +296,152 @@ impl PerfectgirlsProvider {
// for (index, line) in vid.iter().enumerate() {
// println!("Line {}: {}", index, line);
// }
let video_url: String = format!("{}{}", self.url, video_segment.split("<a href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
let video_url: String = format!(
"{}{}",
self.url,
video_segment
.split("<a href=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default());
let preview_url = video_segment.split("data-preview-custom=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
);
let preview_url = video_segment
.split("data-preview-custom=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
let mut title = video_segment.split("\" title=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
let mut title = video_segment
.split("\" title=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
// html decode
title = decode(title.as_bytes()).to_string().unwrap_or(title);
let id = video_url.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string();
let raw_duration = video_segment.split("fa fa-clock-o").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<span>").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
let id = video_url
.split("/")
.collect::<Vec<&str>>()
.get(4)
.copied()
.unwrap_or_default()
.to_string();
let raw_duration = video_segment
.split("fa fa-clock-o")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<span>")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
let mut thumb = video_segment.split(" class=\"thumb lazy-load\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("data-original=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
let mut thumb = video_segment
.split(" class=\"thumb lazy-load\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("data-original=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
if thumb.starts_with("//"){
thumb = format!("https:{}",thumb);
if thumb.starts_with("//") {
thumb = format!("https:{}", thumb);
}
let mut tags = vec![];
if video_segment.contains("href=\"/channels/"){
let raw_tags = video_segment.split("href=\"/channels/").collect::<Vec<&str>>()[1..]
if video_segment.contains("href=\"/channels/") {
let raw_tags = video_segment
.split("href=\"/channels/")
.collect::<Vec<&str>>()[1..]
.iter()
.map(|s| s.split("/\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default().to_string())
.map(|s| {
s.split("/\"")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string()
})
.collect::<Vec<String>>();
for tag in raw_tags {
if !tag.is_empty() {
tags.push(format!("@channels:{}",tag));
tags.push(format!("@channels:{}", tag));
}
}
}
if video_segment.contains("href=\"/pornstars/"){
let raw_tags = video_segment.split("href=\"/pornstars/").collect::<Vec<&str>>()[1..]
if video_segment.contains("href=\"/pornstars/") {
let raw_tags = video_segment
.split("href=\"/pornstars/")
.collect::<Vec<&str>>()[1..]
.iter()
.map(|s| s.split("/\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default().to_string())
.map(|s| {
s.split("/\"")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string()
})
.collect::<Vec<String>>();
for tag in raw_tags {
if !tag.is_empty() {
tags.push(format!("@pornstars:{}",tag));
tags.push(format!("@pornstars:{}", tag));
}
}
}
let views_part = video_segment.split("fa fa-eye").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<span>").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
let views_part = video_segment
.split("fa fa-eye")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<span>")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
let views = parse_abbreviated_number(&views_part).unwrap_or(0) as u32;
@@ -358,14 +455,11 @@ impl PerfectgirlsProvider {
)
.preview(preview_url)
.views(views)
.tags(tags)
;
.tags(tags);
items.push(video_item);
}
return items;
}
}
#[async_trait]
@@ -384,10 +478,7 @@ impl Provider for PerfectgirlsProvider {
let _ = per_page;
let _ = pool;
let videos: std::result::Result<Vec<VideoItem>, Error> = match query {
Some(q) => {
self.query(cache, page.parse::<u8>().unwrap_or(1), &q,)
.await
}
Some(q) => self.query(cache, page.parse::<u8>().unwrap_or(1), &q).await,
None => {
self.get(cache, page.parse::<u8>().unwrap_or(1), &sort)
.await

398
src/providers/perverzija.rs
View File

@@ -6,16 +6,15 @@ use crate::status::*;
use crate::util::cache::VideoCache;
use crate::util::time::parse_time_to_seconds;
use crate::videos::ServerOptions;
use crate::videos::{self, VideoEmbed, VideoItem};
use crate::videos::{self, VideoItem};
use async_trait::async_trait;
use error_chain::error_chain;
use futures::future::join_all;
use htmlentity::entity::{ICodedDataTrait, decode};
use serde::Deserialize;
use serde::Serialize;
use wreq::Version;
use std::vec;
use wreq::Client;
use wreq::Version;
use wreq_util::Emulation;
error_chain! {
@@ -78,6 +77,66 @@ impl PerverzijaProvider {
}
}
fn extract_between<'a>(haystack: &'a str, start: &str, end: &str) -> Option<&'a str> {
let rest = haystack.split(start).nth(1)?;
Some(rest.split(end).next().unwrap_or_default())
}
fn extract_iframe_src(haystack: &str) -> String {
Self::extract_between(haystack, "iframe src=\"", "\"")
.or_else(|| Self::extract_between(haystack, "iframe src=&quot;", "&quot;"))
.unwrap_or_default()
.to_string()
}
fn extract_thumb(haystack: &str) -> String {
let img_segment = haystack.split("<img").nth(1).unwrap_or_default();
let mut thumb = Self::extract_between(img_segment, "data-original=\"", "\"")
.or_else(|| Self::extract_between(img_segment, "data-src=\"", "\""))
.or_else(|| Self::extract_between(img_segment, "src=\"", "\""))
.unwrap_or_default()
.to_string();
if thumb.starts_with("data:image") {
thumb.clear();
} else if thumb.starts_with("//") {
thumb = format!("https:{thumb}");
}
thumb
}
fn extract_title(haystack: &str) -> String {
let mut title = Self::extract_between(haystack, "<h4 class='gv-title'>", "</h4>")
.or_else(|| Self::extract_between(haystack, "<h4 class=\"gv-title\">", "</h4>"))
.or_else(|| Self::extract_between(haystack, " title='", "'"))
.or_else(|| Self::extract_between(haystack, " title=\"", "\""))
.unwrap_or_default()
.to_string();
title = decode(title.as_bytes()).to_string().unwrap_or(title);
if title.contains('<') && title.contains('>') {
let mut plain = String::new();
let mut in_tag = false;
for c in title.chars() {
match c {
'<' => in_tag = true,
'>' => in_tag = false,
_ if !in_tag => plain.push(c),
_ => {}
}
}
let normalized = plain.split_whitespace().collect::<Vec<&str>>().join(" ");
if !normalized.is_empty() {
title = normalized;
}
} else {
title = title.split_whitespace().collect::<Vec<&str>>().join(" ");
}
title.trim().to_string()
}
async fn get(
&self,
cache: VideoCache,
@@ -204,93 +263,91 @@ impl PerverzijaProvider {
fn get_video_items_from_html(&self, html: String, pool: DbPool) -> Vec<VideoItem> {
if html.is_empty() {
println!("HTML is empty");
report_provider_error_background(
"perverzija",
"get_video_items_from_html.empty_html",
"empty html response",
);
return vec![];
}
let mut items: Vec<VideoItem> = Vec::new();
let video_listing_content = html.split("video-listing-content").collect::<Vec<&str>>().get(1).copied().unwrap_or_default();
let raw_videos = video_listing_content
let video_listing_content = html.split("video-listing-content").nth(1).unwrap_or(&html);
let raw_videos: Vec<&str> = video_listing_content
.split("video-item post")
.collect::<Vec<&str>>()[1..]
.to_vec();
for video_segment in &raw_videos {
let vid = video_segment.split("\n").collect::<Vec<&str>>();
if vid.len() > 20 || vid.len() < 8 {
.skip(1)
.collect();
if raw_videos.is_empty() {
report_provider_error_background(
"perverzija",
"get_video_items_from_html.no_segments",
&format!("html_len={}", html.len()),
);
return vec![];
}
for video_segment in raw_videos {
let title = Self::extract_title(video_segment);
let embed_html_raw = Self::extract_between(video_segment, "data-embed='", "'")
.or_else(|| Self::extract_between(video_segment, "data-embed=\"", "\""))
.unwrap_or_default()
.to_string();
let embed_html = decode(embed_html_raw.as_bytes())
.to_string()
.unwrap_or(embed_html_raw.clone());
let mut url_str = Self::extract_iframe_src(&embed_html);
if url_str.is_empty() {
url_str = Self::extract_iframe_src(video_segment);
}
if url_str.is_empty() {
report_provider_error_background(
"perverzija",
"get_video_items_from_html.snippet_shape",
&format!("unexpected snippet length={}", vid.len()),
"get_video_items_from_html.url_missing",
"missing iframe src in segment",
);
continue;
}
let line0 = vid.get(0).copied().unwrap_or_default();
let line1 = vid.get(1).copied().unwrap_or_default();
let line4 = vid.get(4).copied().unwrap_or_default();
let line6 = vid.get(6).copied().unwrap_or_default();
let line7 = vid.get(7).copied().unwrap_or_default();
// for (index, line) in vid.iter().enumerate() {
// println!("Line {}: {}", index, line.to_string().trim());
// }
let mut title = line1.split(">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
// html decode
title = decode(title.as_bytes()).to_string().unwrap_or(title);
if !line1.contains("iframe src=&quot;") {
continue;
}
let url_str = line1.split("iframe src=&quot;").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("&quot;")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string()
.replace("index.php", "xs1.php");
url_str = url_str.replace("index.php", "xs1.php");
if url_str.starts_with("https://streamtape.com/") {
continue; // Skip Streamtape links
}
let id = url_str.split("data=").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("&")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
let id_url = Self::extract_between(video_segment, "data-url='", "'")
.or_else(|| Self::extract_between(video_segment, "data-url=\"", "\""))
.unwrap_or_default()
.to_string();
let mut id = url_str
.split("data=")
.nth(1)
.unwrap_or_default()
.split('&')
.next()
.unwrap_or_default()
.to_string();
if id.is_empty() {
id = id_url
.trim_end_matches('/')
.rsplit('/')
.next()
.unwrap_or_default()
.to_string();
}
let raw_duration = Self::extract_between(video_segment, "time_dur\">", "<")
.or_else(|| Self::extract_between(video_segment, "class=\"time\">", "<"))
.unwrap_or("00:00")
.to_string();
let raw_duration = match vid.len() {
10 => line6.split("time_dur\">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string(),
_ => "00:00".to_string(),
};
let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
if !line4.contains("srcset=")
&& line4.split("src=\"").collect::<Vec<&str>>().len() == 1
{
for (index, line) in vid.iter().enumerate() {
println!("Line {}: {}\n\n", index, line);
}
}
let mut thumb = "".to_string();
for v in vid.clone() {
let line = v.trim();
if line.starts_with("<img ") {
thumb = line.split(" src=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
}
}
let embed_html = line1.split("data-embed='").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("'")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
let id_url = line1.split("data-url='").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("'")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
let thumb = Self::extract_thumb(video_segment);
match pool.get() {
Ok(mut conn) => {
let _ = db::insert_video(&mut conn, &id_url, &url_str);
if !id_url.is_empty() {
let _ = db::insert_video(&mut conn, &id_url, &url_str);
}
}
Err(e) => {
report_provider_error_background(
@@ -301,26 +358,36 @@ impl PerverzijaProvider {
}
}
let referer_url = "https://xtremestream.xyz/".to_string();
let embed = VideoEmbed::new(embed_html, url_str.clone());
let mut tags: Vec<String> = Vec::new(); // Placeholder for tags, adjust as needed
let mut tags: Vec<String> = Vec::new();
let studios_parts = line7.split("a href=\"").collect::<Vec<&str>>();
let studios_parts = video_segment.split("a href=\"").collect::<Vec<&str>>();
for studio in studios_parts.iter().skip(1) {
if studio.starts_with("https://tube.perverzija.com/studio/") {
tags.push(
studio.split("/\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
studio
.split("/\"")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.replace("https://tube.perverzija.com/studio/", "@studio:")
.to_string(),
);
}
}
for tag in line0.split(" ").collect::<Vec<&str>>() {
if tag.starts_with("stars-") {
let tag_name = tag.split("stars-").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
for tag in video_segment.split_whitespace() {
let token =
tag.trim_matches(|c: char| c == '"' || c == '\'' || c == '>' || c == '<');
if token.starts_with("stars-") {
let tag_name = token
.split("stars-")
.nth(1)
.unwrap_or_default()
.split('"')
.next()
.unwrap_or_default()
.to_string();
if !tag_name.is_empty() {
tags.push(format!("@stars:{}", tag_name));
@@ -328,9 +395,11 @@ impl PerverzijaProvider {
}
}
for tag in line0.split(" ").collect::<Vec<&str>>() {
if tag.starts_with("tag-") {
let tag_name = tag.split("tag-").collect::<Vec<&str>>().get(1).copied().unwrap_or_default().to_string();
for tag in video_segment.split_whitespace() {
let token =
tag.trim_matches(|c: char| c == '"' || c == '\'' || c == '>' || c == '<');
if token.starts_with("tag-") {
let tag_name = token.split("tag-").nth(1).unwrap_or_default().to_string();
if !tag_name.is_empty() {
tags.push(tag_name.replace("-", " ").to_string());
}
@@ -339,7 +408,7 @@ impl PerverzijaProvider {
let mut video_item = VideoItem::new(
id,
title,
embed.source.clone(),
url_str.clone(),
"perverzija".to_string(),
thumb,
duration,
@@ -361,7 +430,15 @@ impl PerverzijaProvider {
}
async fn get_video_items_from_html_query(&self, html: String, pool: DbPool) -> Vec<VideoItem> {
let raw_videos = html.split("video-item post").collect::<Vec<&str>>()[1..].to_vec();
let raw_videos: Vec<&str> = html.split("video-item post").skip(1).collect();
if raw_videos.is_empty() {
report_provider_error_background(
"perverzija",
"get_video_items_from_html_query.no_segments",
&format!("html_len={}", html.len()),
);
return vec![];
}
let futures = raw_videos
.into_iter()
.map(|el| self.get_video_item(el, pool.clone()));
@@ -372,53 +449,39 @@ impl PerverzijaProvider {
}
async fn get_video_item(&self, snippet: &str, pool: DbPool) -> Result<VideoItem> {
let vid = snippet.split("\n").collect::<Vec<&str>>();
if vid.len() > 30 || vid.len() < 7 {
if snippet.trim().is_empty() {
report_provider_error_background(
"perverzija",
"get_video_item.snippet_shape",
&format!("unexpected snippet length={}", vid.len()),
"get_video_item.empty_snippet",
"snippet is empty",
);
return Err("Unexpected video snippet length".into());
return Err("empty snippet".into());
}
let line5 = vid.get(5).copied().unwrap_or_default();
let line6 = vid.get(6).copied().unwrap_or_default();
let mut title = line5.split(" title=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
title = decode(title.as_bytes()).to_string().unwrap_or(title);
let title = Self::extract_title(snippet);
let thumb = match line6.split(" src=\"").collect::<Vec<&str>>().len() {
1 => {
for (index, line) in vid.iter().enumerate() {
println!("Line {}: {}", index, line.to_string().trim());
}
return Err("Failed to parse thumbnail URL".into());
}
_ => line6.split(" src=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string(),
};
let thumb = Self::extract_thumb(snippet);
let duration = 0;
let lookup_url = line5.split(" href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
let lookup_url = Self::extract_between(snippet, " href=\"", "\"")
.or_else(|| Self::extract_between(snippet, "data-url='", "'"))
.unwrap_or_default()
.to_string();
if lookup_url.is_empty() {
report_provider_error_background(
"perverzija",
"get_video_item.lookup_url_missing",
"missing lookup url in snippet",
);
return Err("Failed to parse lookup url".into());
}
let referer_url = "https://xtremestream.xyz/".to_string();
let mut conn = match pool.get() {
Ok(conn) => conn,
Err(e) => {
report_provider_error(
"perverzija",
"get_video_item.pool_get",
&e.to_string(),
)
.await;
report_provider_error("perverzija", "get_video_item.pool_get", &e.to_string())
.await;
return Err("couldn't get db connection from pool".into());
}
};
@@ -433,9 +496,21 @@ impl PerverzijaProvider {
if url_str.starts_with("!") {
return Err("Video was removed".into());
}
let mut id = url_str.split("data=").collect::<Vec<&str>>().get(1).copied().unwrap_or_default().to_string();
let mut id = url_str
.split("data=")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.to_string();
if id.contains("&") {
id = id.split("&").collect::<Vec<&str>>().get(0).copied().unwrap_or_default().to_string()
id = id
.split("&")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string()
}
let mut video_item = VideoItem::new(
id,
@@ -481,9 +556,17 @@ impl PerverzijaProvider {
}
};
let mut url_str = text.split("<iframe src=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
let mut url_str = text
.split("<iframe src=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string()
.replace("index.php", "xs1.php");
if !url_str.contains("xtremestream.xyz") {
@@ -494,15 +577,26 @@ impl PerverzijaProvider {
let studios_parts = text
.split("<strong>Studio: </strong>")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("</div>")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.split("<a href=\"")
.collect::<Vec<&str>>();
for studio in studios_parts.iter().skip(1) {
if studio.starts_with("https://tube.perverzija.com/studio/") {
tags.push(
studio.split("/\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
studio
.split("/\"")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.replace("https://tube.perverzija.com/studio/", "@studio:")
.to_string(),
);
@@ -511,15 +605,25 @@ impl PerverzijaProvider {
if text.contains("<strong>Stars: </strong>") {
let stars_parts: Vec<&str> = text
.split("<strong>Stars: </strong>")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("</div>")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.split("<a href=\"")
.collect::<Vec<&str>>();
for star in stars_parts.iter().skip(1) {
if star.starts_with("https://tube.perverzija.com/stars/") {
tags.push(
star.split("/\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
star.split("/\"")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.replace("https://tube.perverzija.com/stars/", "@stars:")
.to_string(),
);
@@ -527,15 +631,27 @@ impl PerverzijaProvider {
}
}
let tags_parts: Vec<&str> = text.split("<strong>Tags: </strong>").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
let tags_parts: Vec<&str> = text
.split("<strong>Tags: </strong>")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("</div>")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.split("<a href=\"")
.collect::<Vec<&str>>();
for star in tags_parts.iter().skip(1) {
if star.starts_with("https://tube.perverzija.com/stars/") {
tags.push(
star.split("/\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
star.split("/\"")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.replace("https://tube.perverzija.com/stars/", "@stars:")
.to_string(),
);
@@ -574,9 +690,21 @@ impl PerverzijaProvider {
if !url_str.contains("xtremestream.xyz") {
return Err("Video URL does not contain xtremestream.xyz".into());
}
let mut id = url_str.split("data=").collect::<Vec<&str>>().get(1).copied().unwrap_or_default().to_string();
let mut id = url_str
.split("data=")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.to_string();
if id.contains("&") {
id = id.split("&").collect::<Vec<&str>>().get(0).copied().unwrap_or_default().to_string()
id = id
.split("&")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string()
}
// if !vid[6].contains(" src=\""){
// for (index,line) in vid.iter().enumerate() {

49
src/providers/pimpbunny.rs
View File

@@ -11,7 +11,7 @@ use crate::videos::{ServerOptions, VideoFormat, VideoItem};
use async_trait::async_trait;
use error_chain::error_chain;
use futures::future::join_all;
use htmlentity::entity::{decode, ICodedDataTrait};
use htmlentity::entity::{ICodedDataTrait, decode};
use std::sync::{Arc, RwLock};
use std::{thread, vec};
use titlecase::Titlecase;
@@ -122,7 +122,8 @@ impl PimpbunnyProvider {
file!(),
line!(),
module_path!(),
).await;
)
.await;
return;
}
};
@@ -138,7 +139,8 @@ impl PimpbunnyProvider {
file!(),
line!(),
module_path!(),
).await;
)
.await;
}
if let Err(e) = Self::load_categories(&url, Arc::clone(&categories)).await {
eprintln!("load_categories failed: {e}");
@@ -150,7 +152,8 @@ impl PimpbunnyProvider {
file!(),
line!(),
module_path!(),
).await;
)
.await;
}
});
});
@@ -183,7 +186,9 @@ impl PimpbunnyProvider {
.unwrap_or("");
for el in block.split("<div class=\"col\">").skip(1) {
if el.contains("pb-promoted-link") || !el.contains("href=\"https://pimpbunny.com/onlyfans-models/") {
if el.contains("pb-promoted-link")
|| !el.contains("href=\"https://pimpbunny.com/onlyfans-models/")
{
continue;
}
@@ -309,7 +314,9 @@ impl PimpbunnyProvider {
let mut video_url = format!(
"{}/search/{}/?mode=async&function=get_block&block_id=list_videos_videos_list_search_result&videos_per_page=32&from_videos={}",
self.url, search_string.replace(" ","-"), page
self.url,
search_string.replace(" ", "-"),
page
);
let sort_string = match options.sort.as_deref().unwrap_or("") {
@@ -423,11 +430,7 @@ impl PimpbunnyProvider {
.collect()
}
async fn get_video_item(
&self,
seg: String,
mut requester: Requester,
) -> Result<VideoItem> {
async fn get_video_item(&self, seg: String, mut requester: Requester) -> Result<VideoItem> {
let video_url = seg
.split(" href=\"")
.nth(1)
@@ -443,7 +446,10 @@ impl PimpbunnyProvider {
.ok_or_else(|| ErrorKind::Parse("video title".into()))?
.trim()
.to_string();
title = decode(title.as_bytes()).to_string().unwrap_or(title).titlecase();
title = decode(title.as_bytes())
.to_string()
.unwrap_or(title)
.titlecase();
let id = video_url
.split('/')
@@ -483,18 +489,13 @@ impl PimpbunnyProvider {
let (tags, formats, views, duration) =
self.extract_media(&video_url, &mut requester).await?;
Ok(VideoItem::new(
id,
title,
video_url,
"pimpbunny".into(),
thumb,
duration,
Ok(
VideoItem::new(id, title, video_url, "pimpbunny".into(), thumb, duration)
.formats(formats)
.tags(tags)
.preview(preview)
.views(views),
)
.formats(formats)
.tags(tags)
.preview(preview)
.views(views))
}
async fn extract_media(
@@ -532,7 +533,7 @@ impl PimpbunnyProvider {
let duration = json["duration"]
.as_str()
.map(|d| parse_time_to_seconds(&d.replace(['P','T','H','M','S'], "")).unwrap_or(0))
.map(|d| parse_time_to_seconds(&d.replace(['P', 'T', 'H', 'M', 'S'], "")).unwrap_or(0))
.unwrap_or(0) as u32;
Ok((

257
src/providers/pmvhaven.rs
View File

@@ -1,6 +1,6 @@
use crate::DbPool;
use crate::api::ClientVersion;
use crate::providers::Provider;
use crate::providers::{Provider, report_provider_error_background, requester_or_default};
use crate::status::*;
use crate::util::cache::VideoCache;
use crate::util::discord::send_discord_error_report;
@@ -8,10 +8,11 @@ use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoFormat, VideoItem};
use async_trait::async_trait;
use error_chain::error_chain;
use htmlentity::entity::{decode, ICodedDataTrait};
use htmlentity::entity::{ICodedDataTrait, decode};
use std::fmt::Write;
use std::sync::{Arc, RwLock};
use std::vec;
use std::fmt::Write;
use url::form_urlencoded::byte_serialize;
error_chain! {
foreign_links {
@@ -135,9 +136,7 @@ impl PmvhavenProvider {
fn is_direct_media_url(url: &str) -> bool {
let lower = url.to_ascii_lowercase();
(lower.starts_with("http://") || lower.starts_with("https://"))
&& (lower.contains("/videos/")
|| lower.contains(".mp4")
|| lower.contains(".m3u8"))
&& (lower.contains("/videos/") || lower.contains(".mp4") || lower.contains(".m3u8"))
}
fn pick_downloadable_media_url(&self, video: &serde_json::Value) -> Option<String> {
@@ -192,67 +191,119 @@ impl PmvhavenProvider {
_ => "",
};
let endpoint = if search.is_empty() {
"api/videos"
} else {
"api/videos/search"
};
let mut url = format!(
"{}/{endpoint}?limit=100&page={page}{duration}{sort}",
self.url
);
let encoded_search: String = byte_serialize(search.as_bytes()).collect();
let mut extra_filters = String::new();
if let Ok(stars) = self.stars.read() {
if let Some(star) = stars.iter().find(|s| s.eq_ignore_ascii_case(&search)) {
url.push_str(&format!("&stars={star}"));
let encoded_star: String = byte_serialize(star.as_bytes()).collect();
extra_filters.push_str(&format!("&stars={encoded_star}"));
}
}
if let Ok(cats) = self.categories.read() {
if let Some(cat) = cats.iter().find(|c| c.eq_ignore_ascii_case(&search)) {
url.push_str(&format!("&tagMode=OR&tags={cat}&expandTags=false"));
let encoded_cat: String = byte_serialize(cat.as_bytes()).collect();
extra_filters.push_str(&format!("&tagMode=OR&tags={encoded_cat}&expandTags=false"));
}
}
if !search.is_empty() {
url.push_str(&format!("&q={search}"));
let mut urls = vec![];
if search.is_empty() {
urls.push(format!(
"{}/api/videos?limit=100&page={page}{duration}{sort}{extra_filters}",
self.url
));
} else {
urls.push(format!(
"{}/api/videos/search?limit=100&page={page}{duration}{sort}{extra_filters}&q={encoded_search}",
self.url
));
urls.push(format!(
"{}/api/videos/search?limit=100&page={page}{duration}{sort}{extra_filters}&query={encoded_search}",
self.url
));
urls.push(format!(
"{}/api/videos?limit=100&page={page}{duration}{sort}{extra_filters}&q={encoded_search}",
self.url
));
urls.push(format!(
"{}/api/videos?limit=100&page={page}{duration}{sort}{extra_filters}&search={encoded_search}",
self.url
));
}
if let Some((time, items)) = cache.get(&url) {
if time.elapsed().unwrap_or_default().as_secs() < 300 {
return Ok(items.clone());
let mut requester = requester_or_default(&options, "pmvhaven", "query");
for url in urls {
if let Some((time, items)) = cache.get(&url) {
if time.elapsed().unwrap_or_default().as_secs() < 300 {
return Ok(items.clone());
}
}
let text = match requester.get(&url, None).await {
Ok(text) => text,
Err(err) => {
report_provider_error_background(
"pmvhaven",
"get.request",
&format!("url={url}; error={err}"),
);
continue;
}
};
let json: serde_json::Value = match serde_json::from_str(&text) {
Ok(json) => json,
Err(err) => {
report_provider_error_background(
"pmvhaven",
"parse.json",
&format!("url={url}; error={err}"),
);
continue;
}
};
let items = self.get_video_items_from_json(json).await;
if !items.is_empty() {
cache.remove(&url);
cache.insert(url, items.clone());
return Ok(items);
}
}
let mut requester = match options.requester {
Some(r) => r,
None => return Ok(vec![]),
};
let text = requester.get(&url, None).await.unwrap_or_default();
let json = serde_json::from_str(&text).unwrap_or(serde_json::Value::Null);
let items = self.get_video_items_from_json(json).await;
if !items.is_empty() {
cache.remove(&url);
cache.insert(url, items.clone());
}
Ok(items)
Ok(vec![])
}
async fn get_video_items_from_json(&self, json: serde_json::Value) -> Vec<VideoItem> {
let mut items = vec![];
if !json.get("success").and_then(|v| v.as_bool()).unwrap_or(false) {
if !json
.get("success")
.and_then(|v| v.as_bool())
.unwrap_or(false)
{
return items;
}
let videos = json.get("data").and_then(|v| v.as_array()).cloned().unwrap_or_default();
let videos = json
.get("data")
.and_then(|v| v.as_array())
.or_else(|| json.get("videos").and_then(|v| v.as_array()))
.cloned()
.unwrap_or_default();
for video in videos {
let title = decode(video.get("title").and_then(|v| v.as_str()).unwrap_or("").as_bytes())
.to_string()
.unwrap_or_default();
let title = decode(
video
.get("title")
.and_then(|v| v.as_str())
.unwrap_or("")
.as_bytes(),
)
.to_string()
.unwrap_or_default();
let id = video
.get("_id")
@@ -266,14 +317,36 @@ impl PmvhavenProvider {
continue;
}
};
let thumb = video.get("thumbnailUrl").and_then(|v| v.as_str()).unwrap_or("").to_string();
let preview = video.get("previewUrl").and_then(|v| v.as_str()).unwrap_or("").to_string();
let thumb = video
.get("thumbnailUrl")
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string();
let preview = video
.get("previewUrl")
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string();
let views = video.get("views").and_then(|v| v.as_u64()).unwrap_or(0);
let duration = parse_time_to_seconds(video.get("duration").and_then(|v| v.as_str()).unwrap_or("0")).unwrap_or(0);
let duration = parse_time_to_seconds(
video
.get("duration")
.and_then(|v| v.as_str())
.unwrap_or("0"),
)
.unwrap_or(0);
let tags = video.get("tags").and_then(|v| v.as_array()).cloned().unwrap_or_default();
let stars = video.get("starsTags").and_then(|v| v.as_array()).cloned().unwrap_or_default();
let tags = video
.get("tags")
.and_then(|v| v.as_array())
.cloned()
.unwrap_or_default();
let stars = video
.get("starsTags")
.and_then(|v| v.as_array())
.cloned()
.unwrap_or_default();
for t in tags.iter() {
if let Some(s) = t.as_str() {
let decoded = decode(s.as_bytes()).to_string().unwrap_or_default();
@@ -293,14 +366,21 @@ impl PmvhavenProvider {
"mp4".to_string()
};
items.push(
VideoItem::new(id, title, video_url.clone(), "pmvhaven".into(), thumb, duration as u32)
.views(views as u32)
.formats(vec![VideoFormat::new(
video_url,
"1080".to_string(),
format_type,
)])
.preview(preview)
VideoItem::new(
id,
title,
video_url.clone(),
"pmvhaven".into(),
thumb,
duration as u32,
)
.views(views as u32)
.formats(vec![VideoFormat::new(
video_url,
"1080".to_string(),
format_type,
)])
.preview(preview),
);
}
@@ -308,6 +388,56 @@ impl PmvhavenProvider {
}
}
#[cfg(test)]
mod tests {
use super::PmvhavenProvider;
use serde_json::json;
#[tokio::test]
async fn parses_videos_from_videos_key() {
let provider = PmvhavenProvider::new();
let payload = json!({
"success": true,
"videos": [{
"_id": "abc123",
"title": "Sample Title",
"videoUrl": "https://video.pmvhaven.com/videos/sample.mp4",
"thumbnailUrl": "https://video.pmvhaven.com/thumbnails/sample.webp",
"previewUrl": "https://video.pmvhaven.com/previews/sample.mp4",
"views": 42,
"duration": "2:11",
"tags": [],
"starsTags": []
}]
});
let items = provider.get_video_items_from_json(payload).await;
assert_eq!(items.len(), 1);
}
#[tokio::test]
async fn parses_videos_from_data_key() {
let provider = PmvhavenProvider::new();
let payload = json!({
"success": true,
"data": [{
"_id": "abc123",
"title": "Sample Title",
"videoUrl": "https://video.pmvhaven.com/videos/sample.mp4",
"thumbnailUrl": "https://video.pmvhaven.com/thumbnails/sample.webp",
"previewUrl": "https://video.pmvhaven.com/previews/sample.mp4",
"views": 42,
"duration": "2:11",
"tags": [],
"starsTags": []
}]
});
let items = provider.get_video_items_from_json(payload).await;
assert_eq!(items.len(), 1);
}
}
#[async_trait]
impl Provider for PmvhavenProvider {
async fn get_videos(
@@ -332,14 +462,15 @@ impl Provider for PmvhavenProvider {
let _ = writeln!(chain_str, "{}. {}", i + 1, cause);
}
send_discord_error_report(
e.to_string(),
Some(chain_str),
Some("PMVHaven Provider"),
Some("Failed to load videos from PMVHaven"),
file!(),
line!(),
module_path!(),
).await;
e.to_string(),
Some(chain_str),
Some("PMVHaven Provider"),
Some("Failed to load videos from PMVHaven"),
file!(),
line!(),
module_path!(),
)
.await;
vec![]
}
}

102
src/providers/porn00.rs
View File

@@ -1,15 +1,15 @@
use crate::api::ClientVersion;
use crate::util::parse_abbreviated_number;
use crate::DbPool;
use crate::api::ClientVersion;
use crate::providers::Provider;
use crate::status::*;
use crate::util::cache::VideoCache;
use crate::util::parse_abbreviated_number;
use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoItem};
use async_trait::async_trait;
use error_chain::error_chain;
use htmlentity::entity::{ICodedDataTrait, decode};
use std::vec;
use async_trait::async_trait;
error_chain! {
foreign_links {
@@ -132,7 +132,10 @@ impl Porn00Provider {
options: ServerOptions,
) -> Result<Vec<VideoItem>> {
let search_string = query.to_lowercase().trim().replace(" ", "-");
let video_url = format!("{}/q/{}/?mode=async&function=get_block&block_id=list_videos_videos_list_search_result&q={}&category_ids=&sort_by=post_date&from_videos={}&from_albums={}&", self.url, search_string, search_string, page, page);
let video_url = format!(
"{}/q/{}/?mode=async&function=get_block&block_id=list_videos_videos_list_search_result&q={}&category_ids=&sort_by=post_date&from_videos={}&from_albums={}&",
self.url, search_string, search_string, page, page
);
// Check our Video Cache. If the result is younger than 1 hour, we return it.
let old_items = match cache.get(&video_url) {
Some((time, items)) => {
@@ -178,7 +181,12 @@ impl Porn00Provider {
return vec![];
}
let mut items: Vec<VideoItem> = Vec::new();
let raw_videos = html.split("<div class=\"pagination\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
let raw_videos = html
.split("<div class=\"pagination\"")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.split("<div class=\"item \">")
.collect::<Vec<&str>>()[1..]
.to_vec();
@@ -187,31 +195,82 @@ impl Porn00Provider {
// for (index, line) in vid.iter().enumerate() {
// println!("Line {}: {}", index, line);
// }
let video_url: String = video_segment.split("<a href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default().to_string();
let mut title = video_segment.split("\" title=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
let video_url: String = video_segment
.split("<a href=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
let mut title = video_segment
.split("\" title=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
// html decode
title = decode(title.as_bytes()).to_string().unwrap_or(title);
let id = video_url.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string();
let raw_duration = video_segment.split("<div class=\"duration\">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
let id = video_url
.split("/")
.collect::<Vec<&str>>()
.get(4)
.copied()
.unwrap_or_default()
.to_string();
let raw_duration = video_segment
.split("<div class=\"duration\">")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
let thumb = video_segment.split("<img class=\"thumb ").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("data-original=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
let thumb = video_segment
.split("<img class=\"thumb ")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("data-original=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
let views_part = video_segment.split("<div class=\"views\">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
let views_part = video_segment
.split("<div class=\"views\">")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
let views = parse_abbreviated_number(&views_part).unwrap_or(0) as u32;
@@ -223,14 +282,11 @@ impl Porn00Provider {
thumb,
duration,
)
.views(views)
;
.views(views);
items.push(video_item);
}
return items;
}
}
#[async_trait]
@@ -249,7 +305,7 @@ impl Provider for Porn00Provider {
let _ = pool;
let videos: std::result::Result<Vec<VideoItem>, Error> = match query {
Some(q) => {
self.query(cache, page.parse::<u8>().unwrap_or(1), &q,options)
self.query(cache, page.parse::<u8>().unwrap_or(1), &q, options)
.await
}
None => {

166
src/providers/pornhat.rs
View File

@@ -1,15 +1,15 @@
use crate::api::ClientVersion;
use crate::util::parse_abbreviated_number;
use crate::DbPool;
use crate::api::ClientVersion;
use crate::providers::Provider;
use crate::status::*;
use crate::util::cache::VideoCache;
use crate::util::parse_abbreviated_number;
use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoItem};
use async_trait::async_trait;
use error_chain::error_chain;
use htmlentity::entity::{ICodedDataTrait, decode};
use std::vec;
use async_trait::async_trait;
error_chain! {
foreign_links {
@@ -69,7 +69,7 @@ impl PornhatProvider {
cache: VideoCache,
page: u8,
sort: &str,
options:ServerOptions
options: ServerOptions,
) -> Result<Vec<VideoItem>> {
let sort_string = match sort {
"trending" => "/trending",
@@ -117,13 +117,19 @@ impl PornhatProvider {
cache: VideoCache,
page: u8,
query: &str,
options:ServerOptions
options: ServerOptions,
) -> Result<Vec<VideoItem>> {
let search_string = query.to_lowercase().trim().replace(" ", "-");
let mut video_url = format!("{}/search/{}/{}/", self.url, search_string, page);
if search_string.starts_with("@"){
let url_part = search_string.split("@").collect::<Vec<&str>>().get(1).copied().unwrap_or_default().replace(":", "/");
if search_string.starts_with("@") {
let url_part = search_string
.split("@")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.replace(":", "/");
video_url = format!("{}/{}/", self.url, url_part);
}
// Check our Video Cache. If the result is younger than 1 hour, we return it.
@@ -170,7 +176,12 @@ impl PornhatProvider {
return vec![];
}
let mut items: Vec<VideoItem> = Vec::new();
let raw_videos = html.split("<div class=\"pagination\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
let raw_videos = html
.split("<div class=\"pagination\"")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.split("item thumb-bl thumb-bl-video video_")
.collect::<Vec<&str>>()[1..]
.to_vec();
@@ -179,61 +190,147 @@ impl PornhatProvider {
// for (index, line) in vid.iter().enumerate() {
// println!("Line {}: {}", index, line);
// }
let video_url: String = format!("{}{}", self.url, video_segment.split("<a href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
let video_url: String = format!(
"{}{}",
self.url,
video_segment
.split("<a href=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default());
let preview_url = video_segment.split("data-preview-custom=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
);
let preview_url = video_segment
.split("data-preview-custom=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
let mut title = video_segment.split("\" title=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
let mut title = video_segment
.split("\" title=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
// html decode
title = decode(title.as_bytes()).to_string().unwrap_or(title);
let id = video_url.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string();
let raw_duration = video_segment.split("fa fa-clock-o").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<span>").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
let id = video_url
.split("/")
.collect::<Vec<&str>>()
.get(4)
.copied()
.unwrap_or_default()
.to_string();
let raw_duration = video_segment
.split("fa fa-clock-o")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<span>")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
let thumb = video_segment.split("<img class=\"thumb lazy-load\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("data-original=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
let thumb = video_segment
.split("<img class=\"thumb lazy-load\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("data-original=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
let mut tags = vec![];
if video_segment.contains("href=\"/sites/"){
if video_segment.contains("href=\"/sites/") {
let raw_tags = video_segment.split("href=\"/sites/").collect::<Vec<&str>>()[1..]
.iter()
.map(|s| s.split("/\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default().to_string())
.map(|s| {
s.split("/\"")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string()
})
.collect::<Vec<String>>();
for tag in raw_tags {
if !tag.is_empty() {
tags.push(format!("@sites:{}",tag));
tags.push(format!("@sites:{}", tag));
}
}
}
if video_segment.contains("href=\"/models/"){
let raw_tags = video_segment.split("href=\"/models/").collect::<Vec<&str>>()[1..]
if video_segment.contains("href=\"/models/") {
let raw_tags = video_segment
.split("href=\"/models/")
.collect::<Vec<&str>>()[1..]
.iter()
.map(|s| s.split("/\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default().to_string())
.map(|s| {
s.split("/\"")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string()
})
.collect::<Vec<String>>();
for tag in raw_tags {
if !tag.is_empty() {
tags.push(format!("@models:{}",tag));
tags.push(format!("@models:{}", tag));
}
}
}
let views_part = video_segment.split("fa fa-eye").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<span>").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
let views_part = video_segment
.split("fa fa-eye")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<span>")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
let views = parse_abbreviated_number(&views_part).unwrap_or(0) as u32;
@@ -247,14 +344,11 @@ impl PornhatProvider {
)
.preview(preview_url)
.views(views)
.tags(tags)
;
.tags(tags);
items.push(video_item);
}
return items;
}
}
#[async_trait]

35
src/providers/pornhub.rs
View File

@@ -1,15 +1,15 @@
use crate::api::ClientVersion;
use crate::util::parse_abbreviated_number;
use crate::DbPool;
use crate::api::ClientVersion;
use crate::providers::Provider;
use crate::status::*;
use crate::util::cache::VideoCache;
use crate::util::parse_abbreviated_number;
use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoItem};
use error_chain::error_chain;
use htmlentity::entity::{decode, ICodedDataTrait};
use async_trait::async_trait;
use error_chain::error_chain;
use htmlentity::entity::{ICodedDataTrait, decode};
use std::vec;
error_chain! {
@@ -130,14 +130,22 @@ impl PornhubProvider {
let mut split_string = "<ul id=\"video";
let search_string = query.to_lowercase().trim().replace(' ', "+");
let mut video_url =
format!("{}/video/search?search={}&page={}", self.url, search_string, page);
let mut video_url = format!(
"{}/video/search?search={}&page={}",
self.url, search_string, page
);
if query.starts_with('@') {
let mut parts = query[1..].split(':');
let a = parts.next().unwrap_or("");
let b = parts.next().unwrap_or("");
video_url = format!("{}/{}/{}/videos?page={}", self.url, a, b.replace(' ', "-"), page);
video_url = format!(
"{}/{}/{}/videos?page={}",
self.url,
a,
b.replace(' ', "-"),
page
);
if query.contains("@model") || query.contains("@pornstar") {
split_string = "mostRecentVideosSection";
@@ -207,7 +215,9 @@ impl PornhubProvider {
.and_then(|s| s.split('"').next());
let video_url = match url_part {
Some(u) if !u.is_empty() && u != "javascript:void(0)" => format!("{}{}", self.url, u),
Some(u) if !u.is_empty() && u != "javascript:void(0)" => {
format!("{}{}", self.url, u)
}
_ => continue,
};
@@ -274,14 +284,7 @@ impl PornhubProvider {
(None, None)
};
let mut item = VideoItem::new(
id,
title,
video_url,
"pornhub".into(),
thumb,
duration,
);
let mut item = VideoItem::new(id, title, video_url, "pornhub".into(), thumb, duration);
if views > 0 {
item = item.views(views);

42
src/providers/pornzog.rs
View File

@@ -1,15 +1,15 @@
use crate::api::ClientVersion;
use crate::DbPool;
use crate::api::ClientVersion;
use crate::providers::Provider;
use crate::status::*;
use crate::util::cache::VideoCache;
use crate::util::discord::{format_error_chain, send_discord_error_report};
use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoItem};
use async_trait::async_trait;
use error_chain::error_chain;
use htmlentity::entity::{ICodedDataTrait, decode};
use std::vec;
use async_trait::async_trait;
error_chain! {
foreign_links {
@@ -112,12 +112,15 @@ impl PornzogProvider {
// SAFE: Check if requester exists instead of unwrap()
let mut requester = match options.requester.clone() {
Some(r) => r,
None => return Ok(old_items),
None => return Ok(old_items),
};
let text = requester.get(&video_url, None).await.map_err(|e| format!("{}", e))?;
let text = requester
.get(&video_url, None)
.await
.map_err(|e| format!("{}", e))?;
let video_items: Vec<VideoItem> = self.get_video_items_from_html(text.clone());
if !video_items.is_empty() {
cache.remove(&video_url);
cache.insert(video_url.clone(), video_items.clone());
@@ -131,7 +134,7 @@ impl PornzogProvider {
if html.is_empty() {
return vec![];
}
let mut items: Vec<VideoItem> = Vec::new();
// Helper for safe splitting: returns Option<&str>
@@ -149,7 +152,7 @@ impl PornzogProvider {
let raw_videos: Vec<&str> = body.split("class=\"thumb-video ").skip(1).collect();
for (idx, video_segment) in raw_videos.iter().enumerate() {
// Attempt to parse each item. If one fails, we log it and continue to the next
// Attempt to parse each item. If one fails, we log it and continue to the next
// instead of crashing the whole request.
let result: Option<VideoItem> = (|| {
let mut video_url = get_part(video_segment, "href=\"", 1)?
@@ -162,7 +165,9 @@ impl PornzogProvider {
}
let title_raw = get_part(video_segment, "alt=\"", 1)?.split("\"").next()?;
let title = decode(title_raw.as_bytes()).to_string().unwrap_or(title_raw.to_string());
let title = decode(title_raw.as_bytes())
.to_string()
.unwrap_or(title_raw.to_string());
// The ID is the 5th element in a "/" split: e.g., "", "video", "123", "title"
let id = video_url.split("/").nth(4)?.to_string();
@@ -179,8 +184,9 @@ impl PornzogProvider {
let tags_section = get_part(video_segment, "class=\"tags\"", 1)?
.split("</p>")
.next()?;
let tags = tags_section.split("<a href=\"")
let tags = tags_section
.split("<a href=\"")
.skip(1)
.filter_map(|el| {
let name = el.split(">").nth(1)?.split("<").next()?;
@@ -188,7 +194,10 @@ impl PornzogProvider {
})
.collect::<Vec<String>>();
Some(VideoItem::new(id, title, video_url, "pornzog".to_string(), thumb, duration).tags(tags))
Some(
VideoItem::new(id, title, video_url, "pornzog".to_string(), thumb, duration)
.tags(tags),
)
})();
match result {
@@ -214,7 +223,7 @@ impl Provider for PornzogProvider {
) -> Vec<VideoItem> {
let _ = per_page;
let _ = pool;
let page_num = page.parse::<u8>().unwrap_or(1);
let query_str = query.unwrap_or_default();
@@ -225,13 +234,13 @@ impl Provider for PornzogProvider {
// 1. Create a collection of owned data so we don't hold references to `e`
let mut error_reports = Vec::new();
// Iterating through the error chain to collect data into owned Strings
for cause in e.iter().skip(1) {
error_reports.push((
cause.to_string(), // Title
format_error_chain(cause), // Description/Chain
format!("caused by: {}", cause) // Message
format!("caused by: {}", cause), // Message
));
}
@@ -245,9 +254,10 @@ impl Provider for PornzogProvider {
file!(),
line!(),
module_path!(),
).await;
)
.await;
}
vec![]
}
}

107
src/providers/redtube.rs
View File

@@ -1,5 +1,5 @@
use crate::api::ClientVersion;
use crate::DbPool;
use crate::api::ClientVersion;
use crate::providers::{Provider, report_provider_error};
use crate::status::*;
use crate::util::cache::VideoCache;
@@ -94,7 +94,7 @@ impl RedtubeProvider {
page: u8,
query: &str,
sort: &str,
options: ServerOptions
options: ServerOptions,
) -> Result<Vec<VideoItem>> {
let _ = sort; //TODO
let search_string = query.to_lowercase().trim().replace(" ", "+");
@@ -146,9 +146,15 @@ impl RedtubeProvider {
let mut items: Vec<VideoItem> = Vec::new();
let video_listing_content = html
.split("<script type=\"application/ld+json\">")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("</script>")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default();
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default();
let mut videos: Value = match serde_json::from_str(video_listing_content) {
Ok(videos) => videos,
Err(e) => {
@@ -173,7 +179,13 @@ impl RedtubeProvider {
let mut title: String = vid["name"].as_str().unwrap_or("").to_string();
// html decode
title = decode(title.as_bytes()).to_string().unwrap_or(title);
let id = video_url.split("=").collect::<Vec<&str>>().get(1).copied().unwrap_or_default().to_string();
let id = video_url
.split("=")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.to_string();
let raw_duration = vid["duration"].as_str().unwrap_or("0");
let duration = raw_duration
.replace("PT", "")
@@ -203,7 +215,12 @@ impl RedtubeProvider {
return vec![];
}
let mut items: Vec<VideoItem> = Vec::new();
let video_listing_content = html.split("videos_grid").collect::<Vec<&str>>().get(1).copied().unwrap_or_default();
let video_listing_content = html
.split("videos_grid")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default();
let videos = video_listing_content
.split("<li id=\"tags_videos_")
.collect::<Vec<&str>>()[1..]
@@ -212,43 +229,93 @@ impl RedtubeProvider {
// for (i, c) in vid.split("\n").enumerate() {
// println!("{}: {}", i, c);
// }
let id = vid.split("data-video-id=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
let id = vid
.split("data-video-id=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
let video_url = format!("{}/{}", self.url, id);
let title = vid.split(" <a title=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
let title = vid
.split(" <a title=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.trim()
.to_string();
let thumb = vid.split("<img").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
let thumb = vid
.split("<img")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split(" data-src=\"")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
let raw_duration = vid
.split("<span class=\"video-properties tm_video_duration\">")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("</span>")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.trim()
.to_string();
let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
let views_str = vid
.split("<span class='info-views'>")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("</span>")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.trim()
.to_string();
let views = parse_abbreviated_number(&views_str).unwrap_or(0) as u32;
let preview = vid.split("<img").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
let preview = vid
.split("<img")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split(" data-mediabook=\"")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
let video_item =

247
src/providers/rule34gen.rs
View File

@@ -1,15 +1,15 @@
use crate::api::*;
use crate::status::*;
use crate::util::parse_abbreviated_number;
use crate::DbPool;
use crate::api::*;
use crate::providers::{Provider, report_provider_error};
use crate::status::*;
use crate::util::cache::VideoCache;
use crate::util::parse_abbreviated_number;
use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoItem};
use async_trait::async_trait;
use error_chain::error_chain;
use htmlentity::entity::{ICodedDataTrait, decode};
use std::vec;
use async_trait::async_trait;
error_chain! {
foreign_links {
@@ -29,49 +29,49 @@ impl Rule34genProvider {
}
}
fn build_channel(&self, clientversion: ClientVersion) -> Channel {
fn build_channel(&self, clientversion: ClientVersion) -> Channel {
let _ = clientversion;
Channel {
id: "rule34gen".to_string(),
name: "Rule34Gen".to_string(),
description: "If it exists, here might be an AI generated video of it".to_string(),
premium: false,
favicon: "https://www.google.com/s2/favicons?sz=64&domain=rule34gen.com".to_string(),
status: "active".to_string(),
categories: vec![],
options: vec![ChannelOption {
id: "sort".to_string(),
title: "Sort".to_string(),
description: "Sort the Videos".to_string(), //"Sort the videos by Date or Name.".to_string(),
systemImage: "list.number".to_string(),
colorName: "blue".to_string(),
options: vec![
FilterOption {
id: "post_date".to_string(),
title: "Newest".to_string(),
},
FilterOption {
id: "video_viewed".to_string(),
title: "Most Viewed".to_string(),
},
FilterOption {
id: "rating".to_string(),
title: "Top Rated".to_string(),
},
FilterOption {
id: "duration".to_string(),
title: "Longest".to_string(),
},
FilterOption {
id: "pseudo_random".to_string(),
title: "Random".to_string(),
},
],
multiSelect: false,
}],
nsfw: true,
cacheDuration: Some(1800),
}
id: "rule34gen".to_string(),
name: "Rule34Gen".to_string(),
description: "If it exists, here might be an AI generated video of it".to_string(),
premium: false,
favicon: "https://www.google.com/s2/favicons?sz=64&domain=rule34gen.com".to_string(),
status: "active".to_string(),
categories: vec![],
options: vec![ChannelOption {
id: "sort".to_string(),
title: "Sort".to_string(),
description: "Sort the Videos".to_string(), //"Sort the videos by Date or Name.".to_string(),
systemImage: "list.number".to_string(),
colorName: "blue".to_string(),
options: vec![
FilterOption {
id: "post_date".to_string(),
title: "Newest".to_string(),
},
FilterOption {
id: "video_viewed".to_string(),
title: "Most Viewed".to_string(),
},
FilterOption {
id: "rating".to_string(),
title: "Top Rated".to_string(),
},
FilterOption {
id: "duration".to_string(),
title: "Longest".to_string(),
},
FilterOption {
id: "pseudo_random".to_string(),
title: "Random".to_string(),
},
],
multiSelect: false,
}],
nsfw: true,
cacheDuration: Some(1800),
}
}
async fn get(
@@ -79,9 +79,15 @@ fn build_channel(&self, clientversion: ClientVersion) -> Channel {
cache: VideoCache,
page: u8,
sort: &str,
options: ServerOptions
options: ServerOptions,
) -> Result<Vec<VideoItem>> {
let expected_sorts = vec!["post_date", "video_viewed", "rating", "duration", "pseudo_random"];
let expected_sorts = vec![
"post_date",
"video_viewed",
"rating",
"duration",
"pseudo_random",
];
let sort = if expected_sorts.contains(&sort) {
sort
} else {
@@ -94,7 +100,7 @@ fn build_channel(&self, clientversion: ClientVersion) -> Channel {
} else {
format!("{}/{}/?sort_by={}", self.url, page, sort)
};
let mut old_items: Vec<VideoItem> = vec![];
if !(sort == "pseudo_random") {
old_items = match cache.get(&index) {
@@ -116,12 +122,8 @@ fn build_channel(&self, clientversion: ClientVersion) -> Channel {
let text = match requester.get(&url, None).await {
Ok(text) => text,
Err(e) => {
report_provider_error(
"rule34gen",
"get.request",
&format!("url={url}; error={e}"),
)
.await;
report_provider_error("rule34gen", "get.request", &format!("url={url}; error={e}"))
.await;
return Ok(old_items);
}
};
@@ -140,9 +142,15 @@ fn build_channel(&self, clientversion: ClientVersion) -> Channel {
page: u8,
query: &str,
sort: &str,
options: ServerOptions
options: ServerOptions,
) -> Result<Vec<VideoItem>> {
let expected_sorts = vec!["post_date", "video_viewed", "rating", "duration", "pseudo_random"];
let expected_sorts = vec![
"post_date",
"video_viewed",
"rating",
"duration",
"pseudo_random",
];
let sort = if expected_sorts.contains(&sort) {
sort
} else {
@@ -154,7 +162,10 @@ fn build_channel(&self, clientversion: ClientVersion) -> Channel {
let url = if page <= 1 {
format!("{}/search/{}/?sort_by={}", self.url, search_slug, sort)
} else {
format!("{}/search/{}/{}/?sort_by={}", self.url, search_slug, page, sort)
format!(
"{}/search/{}/{}/?sort_by={}",
self.url, search_slug, page, sort
)
};
// Check our Video Cache. If the result is younger than 1 hour, we return it.
@@ -287,7 +298,18 @@ fn build_channel(&self, clientversion: ClientVersion) -> Channel {
}
return items;
}
let video_listing_content = html.split("<div class=\"thumbs clearfix\" id=\"custom_list_videos").collect::<Vec<&str>>().get(1).copied().unwrap_or_default().split("<div class=\"pagination\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default().to_string();
let video_listing_content = html
.split("<div class=\"thumbs clearfix\" id=\"custom_list_videos")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<div class=\"pagination\"")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
let raw_videos = video_listing_content
.split("<div class=\"item thumb video_")
.collect::<Vec<&str>>()[1..]
@@ -298,42 +320,104 @@ fn build_channel(&self, clientversion: ClientVersion) -> Channel {
// println!("Line {}: {}", index, line);
// }
if video_segment.contains("https://rule34gen.com/images/advertisements"){
if video_segment.contains("https://rule34gen.com/images/advertisements") {
continue;
}
let mut title = video_segment.split("<div class=\"thumb_title\">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
let mut title = video_segment
.split("<div class=\"thumb_title\">")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
// html decode
title = decode(title.as_bytes()).to_string().unwrap_or(title);
let id = video_segment.split("https://rule34gen.com/video/").collect::<Vec<&str>>().get(1).copied().unwrap_or_default().split("/").collect::<Vec<&str>>().get(0).copied().unwrap_or_default().to_string();
let raw_duration = video_segment.split("<div class=\"time\">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string();
let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
let views = parse_abbreviated_number(&video_segment
.split("<div class=\"views\">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default().split("</svg>").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()).unwrap_or(0);
//https://rule34gen.com/get_file/47/5e71602b7642f9b997f90c979a368c99b8aad90d89/3942000/3942353/3942353_preview.mp4/
//https://rule34gen.com/get_file/47/5e71602b7642f9b997f90c979a368c99b8aad90d89/3942000/3942353/3942353_preview.mp4/
let thumb = video_segment.split("<img class=\"thumb lazy-load\" src=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default().split("data-original=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
let id = video_segment
.split("https://rule34gen.com/video/")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("/")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
let url = video_segment.split("<a class=\"th js-open-popup\" href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
let raw_duration = video_segment
.split("<div class=\"time\">")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
let views = parse_abbreviated_number(
&video_segment
.split("<div class=\"views\">")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("</svg>")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default(),
)
.unwrap_or(0);
//https://rule34gen.com/get_file/47/5e71602b7642f9b997f90c979a368c99b8aad90d89/3942000/3942353/3942353_preview.mp4/
//https://rule34gen.com/get_file/47/5e71602b7642f9b997f90c979a368c99b8aad90d89/3942000/3942353/3942353_preview.mp4/
let thumb = video_segment
.split("<img class=\"thumb lazy-load\" src=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("data-original=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
let url = video_segment
.split("<a class=\"th js-open-popup\" href=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
// let preview = video_segment.split("<div class=\"img wrap_image\" data-preview=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
// .split("\"")
// .collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
// .to_string();
let video_item = VideoItem::new(
id,
title,
@@ -346,13 +430,10 @@ fn build_channel(&self, clientversion: ClientVersion) -> Channel {
// .preview(preview)
;
items.push(video_item);
}
return items;
}
}
#[async_trait]
@@ -388,7 +469,7 @@ impl Provider for Rule34genProvider {
}
}
}
fn get_channel(&self, clientversion: ClientVersion) -> Option<crate::status::Channel> {
Some(self.build_channel(clientversion))
}

22
src/providers/rule34video.rs
View File

@@ -1,5 +1,5 @@
use crate::api::ClientVersion;
use crate::DbPool;
use crate::api::ClientVersion;
use crate::providers::Provider;
use crate::status::*;
use crate::util::cache::VideoCache;
@@ -133,7 +133,15 @@ impl Rule34videoProvider {
let text = requester.get(&url, None).await.unwrap_or_else(|e| {
eprintln!("Error fetching rule34video URL {}: {}", url, e);
let _ = send_discord_error_report(e.to_string(), None, Some(&url), None, file!(), line!(), module_path!());
let _ = send_discord_error_report(
e.to_string(),
None,
Some(&url),
None,
file!(),
line!(),
module_path!(),
);
"".to_string()
});
let video_items = self.get_video_items_from_html(text);
@@ -197,7 +205,15 @@ impl Rule34videoProvider {
let text = requester.get(&url, None).await.unwrap_or_else(|e| {
eprintln!("Error fetching rule34video URL {}: {}", url, e);
let _ = send_discord_error_report(e.to_string(), None, Some(&url), None, file!(), line!(), module_path!());
let _ = send_discord_error_report(
e.to_string(),
None,
Some(&url),
None,
file!(),
line!(),
module_path!(),
);
"".to_string()
});
let video_items = self.get_video_items_from_html(text);

16
src/providers/sxyprn.rs
View File

@@ -1,5 +1,5 @@
use crate::api::ClientVersion;
use crate::DbPool;
use crate::api::ClientVersion;
use crate::providers::Provider;
use crate::status::*;
use crate::util::cache::VideoCache;
@@ -124,7 +124,8 @@ impl SxyprnProvider {
"all" => "all",
_ => "top",
};
let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let url_str = format!(
"{}/blog/all/{}.html?fl={}&sm={}",
@@ -175,7 +176,8 @@ impl SxyprnProvider {
file!(),
line!(),
module_path!(),
).await;
)
.await;
return Ok(old_items);
}
};
@@ -207,7 +209,8 @@ impl SxyprnProvider {
_ => "latest",
};
// Extract needed fields from options at the start
let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let search_string = query.replace(" ", "-");
let url_str = format!(
"{}/{}.html?page={}&sm={}",
@@ -249,7 +252,7 @@ impl SxyprnProvider {
{
Ok(items) => items,
Err(e) => {
println!("Error parsing video items: {}", e);// 1. Convert the error to a string immediately
println!("Error parsing video items: {}", e); // 1. Convert the error to a string immediately
send_discord_error_report(
e.to_string(),
Some(format_error_chain(&e)),
@@ -258,7 +261,8 @@ impl SxyprnProvider {
file!(),
line!(),
module_path!(),
).await;
)
.await;
return Ok(old_items);
}
};

205
src/providers/xxdbx.rs
View File

@@ -8,9 +8,9 @@ use crate::videos::{ServerOptions, VideoItem};
use async_trait::async_trait;
use error_chain::error_chain;
use htmlentity::entity::{ICodedDataTrait, decode};
use regex::Regex;
use std::sync::{Arc, RwLock};
use std::vec;
use regex::Regex;
error_chain! {
foreign_links {
@@ -56,26 +56,24 @@ impl XxdbxProvider {
favicon: "https://www.google.com/s2/favicons?sz=64&domain=xxdbx.com".to_string(),
status: "active".to_string(),
categories: vec![],
options: vec![
ChannelOption {
id: "sort".to_string(),
title: "Sort".to_string(),
description: "Sort the Videos".to_string(),
systemImage: "list.number".to_string(),
colorName: "blue".to_string(),
options: vec![
FilterOption {
id: "new".into(),
title: "New".into(),
},
FilterOption {
id: "popular".into(),
title: "Most Popular".into(),
},
],
multiSelect: false,
},
],
options: vec![ChannelOption {
id: "sort".to_string(),
title: "Sort".to_string(),
description: "Sort the Videos".to_string(),
systemImage: "list.number".to_string(),
colorName: "blue".to_string(),
options: vec![
FilterOption {
id: "new".into(),
title: "New".into(),
},
FilterOption {
id: "popular".into(),
title: "Most Popular".into(),
},
],
multiSelect: false,
}],
nsfw: true,
cacheDuration: Some(1800),
}
@@ -92,10 +90,7 @@ impl XxdbxProvider {
"popular" => "most-popular".to_string(),
_ => "".to_string(),
};
let video_url = format!(
"{}/{}?page={}",
self.url, sort_string, page
);
let video_url = format!("{}/{}?page={}", self.url, sort_string, page);
let old_items = match cache.get(&video_url) {
Some((time, items)) => {
if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
@@ -109,7 +104,8 @@ impl XxdbxProvider {
}
};
let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let text = match requester.get(&video_url, None).await {
Ok(text) => text,
Err(e) => {
@@ -163,7 +159,7 @@ impl XxdbxProvider {
.unwrap_or(false)
{
search_type = "stars";
} else if is_valid_date(&search_string){
} else if is_valid_date(&search_string) {
search_type = "dates";
}
@@ -186,7 +182,8 @@ impl XxdbxProvider {
}
};
let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let mut requester =
crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
let text = match requester.get(&video_url, None).await {
Ok(text) => text,
@@ -210,9 +207,17 @@ impl XxdbxProvider {
return vec![];
}
let mut items: Vec<VideoItem> = Vec::new();
let raw_videos = html.split("</article>").collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
let raw_videos = html
.split("</article>")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.split("<div class=\"vids\">")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<div class=\"v\">")
.collect::<Vec<&str>>()[1..]
.to_vec();
@@ -221,47 +226,126 @@ impl XxdbxProvider {
// for (index, line) in vid.iter().enumerate() {
// println!("Line {}: {}\n\n", index, line);
// }
let video_url: String = format!("{}{}", self.url, video_segment.split("<a href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string());
let video_url: String = format!(
"{}{}",
self.url,
video_segment
.split("<a href=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string()
);
let mut title = video_segment
.split("<div class=\"v_title\">")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.trim()
.to_string();
// html decode
title = decode(title.as_bytes()).to_string().unwrap_or(title);
let id = video_url.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string();
let id = video_url
.split("/")
.collect::<Vec<&str>>()
.get(4)
.copied()
.unwrap_or_default()
.to_string();
let thumb = format!("https:{}", video_segment.split("<img ").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("src=\"").collect::<Vec<&str>>().last().copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string());
let thumb = format!(
"https:{}",
video_segment
.split("<img ")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("src=\"")
.collect::<Vec<&str>>()
.last()
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string()
);
let raw_duration = video_segment
.split("<div class=\"v_dur\">")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
let duration = parse_time_to_seconds(raw_duration.as_str()).unwrap_or(0) as u32;
let preview = format!("https:{}",video_segment
.split("data-preview=\"")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string());
let tags = video_segment.split("<div class=\"v_tags\">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("</div>").collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
let preview = format!(
"https:{}",
video_segment
.split("data-preview=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string()
);
let tags = video_segment
.split("<div class=\"v_tags\">")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("</div>")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.split("<a href=\"")
.collect::<Vec<&str>>()[1..]
.into_iter().map(|s| s.split("\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default().replace("%20"," ").to_string()).collect::<Vec<String>>();
.into_iter()
.map(|s| {
s.split("\"")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.replace("%20", " ")
.to_string()
})
.collect::<Vec<String>>();
for tag in tags.clone() {
let shorted_tag = tag.split("/").collect::<Vec<&str>>().get(2).copied().unwrap_or_default().to_string();
let shorted_tag = tag
.split("/")
.collect::<Vec<&str>>()
.get(2)
.copied()
.unwrap_or_default()
.to_string();
if tag.contains("channels")
&& self
.channels
@@ -293,7 +377,18 @@ impl XxdbxProvider {
thumb,
duration,
)
.tags(tags.into_iter().map(|s| s.split("/").collect::<Vec<&str>>().last().copied().unwrap_or_default().to_string()).collect::<Vec<String>>())
.tags(
tags.into_iter()
.map(|s| {
s.split("/")
.collect::<Vec<&str>>()
.last()
.copied()
.unwrap_or_default()
.to_string()
})
.collect::<Vec<String>>(),
)
.preview(preview);
items.push(video_item);
}

90
src/providers/xxthots.rs
View File

@@ -1,5 +1,5 @@
use crate::api::ClientVersion;
use crate::DbPool;
use crate::api::ClientVersion;
use crate::providers::Provider;
use crate::status::*;
use crate::util::cache::VideoCache;
@@ -72,7 +72,11 @@ impl XxthotsProvider {
options: ServerOptions,
) -> Result<Vec<VideoItem>> {
let (sort_path, list_str, sort_by) = match sort {
"popular" => ("/most-popular/", "list_videos_common_videos_list", "video_viewed"),
"popular" => (
"/most-popular/",
"list_videos_common_videos_list",
"video_viewed",
),
"top-rated" => ("/top-rated/", "list_videos_common_videos_list", "rating"),
_ => (
"/latest-updates/",
@@ -194,7 +198,10 @@ impl XxthotsProvider {
let mut items: Vec<VideoItem> = Vec::new();
let raw_videos: Vec<&str> = html
.split("<div class=\"pagination\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.split("<div class=\"thumb thumb_rel item \">")
.skip(1)
.collect();
@@ -203,41 +210,87 @@ impl XxthotsProvider {
// for (index, line) in vid.iter().enumerate() {
// println!("Line {}: {}", index, line);
// }
let video_url: String = video_segment.split("<a href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
let video_url: String = video_segment
.split("<a href=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
let mut title = video_segment.split("\" title=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
let mut title = video_segment
.split("\" title=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
// html decode
title = decode(title.as_bytes()).to_string().unwrap_or(title);
let id = video_url.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string();
let id = video_url
.split("/")
.collect::<Vec<&str>>()
.get(4)
.copied()
.unwrap_or_default()
.to_string();
let raw_duration = video_segment
.split("<div class=\"time\">")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
let thumb = video_segment
.split("<img class=\"lazy-load")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("data-original=\"")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
let views_part = video_segment
.split("svg-icon icon-eye")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("</i>")
.collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
let views = parse_abbreviated_number(&views_part).unwrap_or(0) as u32;
@@ -279,7 +332,10 @@ impl Provider for XxthotsProvider {
self.get(cache, page.parse::<u8>().unwrap_or(1), &sort, options)
.await
}
_ => self.get(cache, page.parse::<u8>().unwrap_or(1), &sort, options).await,
_ => {
self.get(cache, page.parse::<u8>().unwrap_or(1), &sort, options)
.await
}
};
match videos {
Ok(v) => v,

127
src/providers/youjizz.rs
View File

@@ -1,15 +1,15 @@
use crate::api::ClientVersion;
use crate::util::parse_abbreviated_number;
use crate::DbPool;
use crate::api::ClientVersion;
use crate::providers::Provider;
use crate::status::*;
use crate::util::cache::VideoCache;
use crate::util::parse_abbreviated_number;
use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoItem};
use async_trait::async_trait;
use error_chain::error_chain;
use htmlentity::entity::{ICodedDataTrait, decode};
use std::vec;
use async_trait::async_trait;
error_chain! {
foreign_links {
@@ -141,7 +141,12 @@ impl YoujizzProvider {
query: &str,
options: ServerOptions,
) -> Result<Vec<VideoItem>> {
let video_url = format!("{}/search/{}-{}.html", self.url, query.to_lowercase().trim(), page);
let video_url = format!(
"{}/search/{}-{}.html",
self.url,
query.to_lowercase().trim(),
page
);
// Check our Video Cache. If the result is younger than 1 hour, we return it.
let old_items = match cache.get(&video_url) {
Some((time, items)) => {
@@ -187,7 +192,12 @@ impl YoujizzProvider {
return vec![];
}
let mut items: Vec<VideoItem> = Vec::new();
let raw_videos = html.split("class=\"mobile-only\"").collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
let raw_videos = html
.split("class=\"mobile-only\"")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.split("class=\"default video-item\"")
.collect::<Vec<&str>>()[1..]
.to_vec();
@@ -200,32 +210,98 @@ impl YoujizzProvider {
// println!("Skipping video segment due to placeholder thumbnail");
// continue;
// }
let video_url: String = format!("{}{}",self.url, video_segment.split("href=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
let video_url: String = format!(
"{}{}",
self.url,
video_segment
.split("href=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default().to_string());
let mut title = video_segment.split("class=\"video-title\">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split(">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string()
);
let mut title = video_segment
.split("class=\"video-title\">")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split(">")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
// html decode
title = decode(title.as_bytes()).to_string().unwrap_or(title);
let id = video_url.split("/").collect::<Vec<&str>>().get(4).copied().unwrap_or_default().to_string();
let id = video_url
.split("/")
.collect::<Vec<&str>>()
.get(4)
.copied()
.unwrap_or_default()
.to_string();
let thumb = format!("https:{}",video_segment.split("<img ").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("data-original=\"").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string());
let raw_duration = video_segment.split("fa fa-clock-o\"></i>&nbsp;").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
let thumb = format!(
"https:{}",
video_segment
.split("<img ")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("data-original=\"")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("\"")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string()
);
let raw_duration = video_segment
.split("fa fa-clock-o\"></i>&nbsp;")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string();
let duration = parse_time_to_seconds(raw_duration.as_str()).unwrap_or(0) as u32;
let views = parse_abbreviated_number(video_segment.split("format-views\">").collect::<Vec<&str>>().get(1).copied().unwrap_or_default()
.split("<")
.collect::<Vec<&str>>().get(0).copied().unwrap_or_default()
.to_string().as_str()).unwrap_or(0) as u32;
let views = parse_abbreviated_number(
video_segment
.split("format-views\">")
.collect::<Vec<&str>>()
.get(1)
.copied()
.unwrap_or_default()
.split("<")
.collect::<Vec<&str>>()
.get(0)
.copied()
.unwrap_or_default()
.to_string()
.as_str(),
)
.unwrap_or(0) as u32;
let video_item = VideoItem::new(
id,
@@ -235,14 +311,11 @@ impl YoujizzProvider {
thumb,
duration,
)
.views(views)
;
.views(views);
items.push(video_item);
}
return items;
}
}
#[async_trait]
@@ -261,7 +334,7 @@ impl Provider for YoujizzProvider {
let _ = pool;
let videos: std::result::Result<Vec<VideoItem>, Error> = match query {
Some(q) => {
self.query(cache, page.parse::<u8>().unwrap_or(1), &q,options)
self.query(cache, page.parse::<u8>().unwrap_or(1), &q, options)
.await
}
None => {

2
src/proxies/hanimecdn.rs
View File

@@ -1,8 +1,8 @@
use ntex::http::header::{CONTENT_LENGTH, CONTENT_TYPE};
use ntex::{
http::Response,
web::{self, HttpRequest, error},
};
use ntex::http::header::{CONTENT_LENGTH, CONTENT_TYPE};
use crate::util::requester::Requester;

51
src/proxies/hqpornerthumb.rs Normal file
View File

@@ -0,0 +1,51 @@
use ntex::http::header::{CONTENT_LENGTH, CONTENT_TYPE};
use ntex::{
http::Response,
web::{self, HttpRequest, error},
};
use crate::util::requester::Requester;
pub async fn get_image(
req: HttpRequest,
requester: web::types::State<Requester>,
) -> Result<impl web::Responder, web::Error> {
let endpoint = req.match_info().query("endpoint").to_string();
let image_url = if endpoint.starts_with("http://") || endpoint.starts_with("https://") {
endpoint
} else {
format!("https://{}", endpoint.trim_start_matches('/'))
};
let upstream = match requester
.get_ref()
.clone()
.get_raw_with_headers(
image_url.as_str(),
vec![("Referer".to_string(), "https://hqporner.com/".to_string())],
)
.await
{
Ok(response) => response,
Err(_) => return Ok(web::HttpResponse::NotFound().finish()),
};
let status = upstream.status();
let headers = upstream.headers().clone();
let bytes = upstream.bytes().await.map_err(error::ErrorBadGateway)?;
let mut resp = Response::build(status);
if let Some(ct) = headers.get(CONTENT_TYPE) {
if let Ok(ct_str) = ct.to_str() {
resp.set_header(CONTENT_TYPE, ct_str);
}
}
if let Some(cl) = headers.get(CONTENT_LENGTH) {
if let Ok(cl_str) = cl.to_str() {
resp.set_header(CONTENT_LENGTH, cl_str);
}
}
Ok(resp.body(bytes.to_vec()))
}

28
src/proxies/javtiful.rs
View File

@@ -3,15 +3,12 @@ use wreq::Version;
use crate::util::requester::Requester;
#[derive(Debug, Clone)]
pub struct JavtifulProxy {
}
pub struct JavtifulProxy {}
impl JavtifulProxy {
pub fn new() -> Self {
JavtifulProxy {
}
JavtifulProxy {}
}
pub async fn get_video_url(
@@ -25,18 +22,15 @@ impl JavtifulProxy {
if text.is_empty() {
return "".to_string();
}
let video_id = url
.split('/')
.nth(4)
.unwrap_or("")
.to_string();
let video_id = url.split('/').nth(4).unwrap_or("").to_string();
let token = text.split("data-csrf-token=\"")
let token = text
.split("data-csrf-token=\"")
.nth(1)
.and_then(|s| s.split('"').next())
.unwrap_or("")
.to_string();
let form = wreq::multipart::Form::new()
.text("video_id", video_id.clone())
.text("pid_c", "".to_string())
@@ -54,11 +48,13 @@ impl JavtifulProxy {
Err(_) => return "".to_string(),
};
let text = resp.text().await.unwrap_or_default();
let json: serde_json::Value = serde_json::from_str(&text).unwrap_or(serde_json::Value::Null);
let video_url = json.get("playlists")
let json: serde_json::Value =
serde_json::from_str(&text).unwrap_or(serde_json::Value::Null);
let video_url = json
.get("playlists")
.map(|v| v.to_string().replace("\"", ""))
.unwrap_or_default();
return video_url;
}
}
}

18
src/proxies/mod.rs
View File

@@ -2,9 +2,10 @@ use ntex::web;
use crate::{proxies::sxyprn::SxyprnProxy, util::requester::Requester};
pub mod sxyprn;
pub mod hanimecdn;
pub mod hqpornerthumb;
pub mod javtiful;
pub mod sxyprn;
#[derive(Debug, Clone)]
pub enum AnyProxy {
@@ -13,23 +14,14 @@ pub enum AnyProxy {
}
pub trait Proxy {
async fn get_video_url(
&self,
url: String,
requester: web::types::State<Requester>,
) -> String;
async fn get_video_url(&self, url: String, requester: web::types::State<Requester>) -> String;
}
impl Proxy for AnyProxy {
async fn get_video_url(
&self,
url: String,
requester: web::types::State<Requester>,
) -> String {
async fn get_video_url(&self, url: String, requester: web::types::State<Requester>) -> String {
match self {
AnyProxy::Sxyprn(p) => p.get_video_url(url, requester).await,
AnyProxy::Javtiful(p) => p.get_video_url(url, requester).await,
}
}
}
}

41
src/proxies/sxyprn.rs
View File

@@ -1,4 +1,4 @@
use base64::{engine::general_purpose, Engine as _};
use base64::{Engine as _, engine::general_purpose};
use ntex::web;
use crate::util::requester::Requester;
@@ -24,13 +24,11 @@ fn boo(sum1: u32, sum2: u32) -> String {
}
#[derive(Debug, Clone)]
pub struct SxyprnProxy {
}
pub struct SxyprnProxy {}
impl SxyprnProxy {
pub fn new() -> Self {
SxyprnProxy {
}
SxyprnProxy {}
}
pub async fn get_video_url(
@@ -45,16 +43,23 @@ impl SxyprnProxy {
return "".to_string();
}
let data_string = text.split("data-vnfo='").collect::<Vec<&str>>()[1]
.split("\":\"").collect::<Vec<&str>>()[1]
.split("\"}").collect::<Vec<&str>>()[0].replace("\\","");
.split("\":\"")
.collect::<Vec<&str>>()[1]
.split("\"}")
.collect::<Vec<&str>>()[0]
.replace("\\", "");
//println!("src: {}",data_string);
let mut tmp = data_string
.split("/")
.map(|s| s.to_string())
.collect::<Vec<String>>();
//println!("tmp: {:?}",tmp);
tmp[1] = format!("{}8/{}", tmp[1], boo(ssut51(tmp[6].as_str()), ssut51(tmp[7].as_str())));
tmp[1] = format!(
"{}8/{}",
tmp[1],
boo(ssut51(tmp[6].as_str()), ssut51(tmp[7].as_str()))
);
//println!("tmp[1]: {:?}",tmp[1]);
//preda
tmp[5] = format!(
@@ -62,17 +67,25 @@ impl SxyprnProxy {
tmp[5].parse::<u32>().unwrap() - ssut51(tmp[6].as_str()) - ssut51(tmp[7].as_str())
);
//println!("tmp: {:?}",tmp);
let sxyprn_video_url = format!("https://sxyprn.com{}",tmp.join("/"));
let sxyprn_video_url = format!("https://sxyprn.com{}", tmp.join("/"));
let response = requester.get_raw(&sxyprn_video_url).await;
match response {
Ok(resp) => {
return format!("https:{}", resp.headers().get("Location").unwrap().to_str().unwrap_or("").to_string());
},
return format!(
"https:{}",
resp.headers()
.get("Location")
.unwrap()
.to_str()
.unwrap_or("")
.to_string()
);
}
Err(e) => {
println!("Error fetching video URL: {}", e);
}
}
return "".to_string();
}
}
}

23
src/proxy.rs
View File

@@ -2,8 +2,8 @@ use ntex::web::{self, HttpRequest};
use crate::proxies::javtiful::JavtifulProxy;
use crate::proxies::sxyprn::SxyprnProxy;
use crate::util::requester::Requester;
use crate::proxies::*;
use crate::util::requester::Requester;
pub fn config(cfg: &mut web::ServiceConfig) {
cfg.service(
@@ -21,21 +21,26 @@ pub fn config(cfg: &mut web::ServiceConfig) {
.route(web::post().to(crate::proxies::hanimecdn::get_image))
.route(web::get().to(crate::proxies::hanimecdn::get_image)),
)
;
.service(
web::resource("/hqporner-thumb/{endpoint}*")
.route(web::post().to(crate::proxies::hqpornerthumb::get_image))
.route(web::get().to(crate::proxies::hqpornerthumb::get_image)),
);
}
async fn proxy2redirect(req: HttpRequest,
requester: web::types::State<Requester>,) -> Result<impl web::Responder, web::Error> {
async fn proxy2redirect(
req: HttpRequest,
requester: web::types::State<Requester>,
) -> Result<impl web::Responder, web::Error> {
let proxy = get_proxy(req.uri().to_string().split("/").collect::<Vec<&str>>()[2]).unwrap();
let endpoint = req.match_info().query("endpoint").to_string();
let video_url = match proxy.get_video_url(endpoint, requester).await{
let video_url = match proxy.get_video_url(endpoint, requester).await {
url if url != "" => url,
_ => "Error".to_string(),
};
Ok(web::HttpResponse::Found()
.header("Location", video_url)
.finish())
.header("Location", video_url)
.finish())
}
fn get_proxy(proxy: &str) -> Option<AnyProxy> {
@@ -44,4 +49,4 @@ fn get_proxy(proxy: &str) -> Option<AnyProxy> {
"javtiful" => Some(AnyProxy::Javtiful(JavtifulProxy::new())),
_ => None,
}
}
}

18
src/status.rs
View File

@@ -24,19 +24,19 @@ pub struct Channel {
#[derive(serde::Serialize)]
pub struct ChannelOption {
pub id: String, //"channels",
pub title: String, //"Sites",
pub description: String, //"Websites included in search results.",
pub systemImage: String, //"network",
pub colorName: String, //"purple",
pub id: String, //"channels",
pub title: String, //"Sites",
pub description: String, //"Websites included in search results.",
pub systemImage: String, //"network",
pub colorName: String, //"purple",
pub options: Vec<FilterOption>, //[],
pub multiSelect: bool, //true
pub multiSelect: bool, //true
}
#[derive(serde::Serialize, Debug, Clone)]
pub struct FilterOption{
pub id: String, //"sort",
pub title: String, //"Sort",
pub struct FilterOption {
pub id: String, //"sort",
pub title: String, //"Sort",
}
#[derive(serde::Serialize)]

2
src/util/cache.rs
View File

@@ -45,7 +45,7 @@ impl VideoCache {
cache.remove(key);
}
}
pub fn entries(&self) -> Option<Vec<(String, (SystemTime, Vec<VideoItem>))>> {
if let Ok(cache) = self.cache.lock() {
// Return a cloned vector of the cache entries

16
src/util/discord.rs
View File

@@ -1,10 +1,10 @@
use std::error::Error;
use std::fmt::Write as _;
use std::time::{SystemTime, UNIX_EPOCH};
use crate::util::requester;
use dashmap::DashMap;
use once_cell::sync::Lazy;
use serde_json::json;
use crate::util::requester;
use std::error::Error;
use std::fmt::Write as _;
use std::time::{SystemTime, UNIX_EPOCH};
// Global cache: Map<ErrorSignature, LastSentTimestamp>
static ERROR_CACHE: Lazy<DashMap<String, u64>> = Lazy::new(DashMap::new);
@@ -42,11 +42,11 @@ pub async fn send_discord_error_report(
if let Some(_) = ERROR_CACHE.get(&error_signature) {
// if now - *last_sent < COOLDOWN_SECONDS {
// Error is still in cooldown, skip sending
return;
// Error is still in cooldown, skip sending
return;
// }
}
// Update the cache with the current timestamp
ERROR_CACHE.insert(error_signature, now);
// ---------------------------
@@ -104,4 +104,4 @@ pub async fn send_discord_error_report(
let mut requester = requester::Requester::new();
let _ = requester.post_json(&webhook_url, &payload, vec![]).await;
}
}

9
src/util/flaresolverr.rs
View File

@@ -57,10 +57,7 @@ pub struct Flaresolverr {
impl Flaresolverr {
pub fn new(url: String) -> Self {
Self {
url,
proxy: false,
}
Self { url, proxy: false }
}
pub fn set_proxy(&mut self, proxy: bool) {
@@ -71,9 +68,7 @@ impl Flaresolverr {
&self,
request: FlareSolverrRequest,
) -> Result<FlareSolverrResponse, Box<dyn std::error::Error>> {
let client = Client::builder()
.emulation(Emulation::Firefox136)
.build()?;
let client = Client::builder().emulation(Emulation::Firefox136).build()?;
let mut req = client
.post(&self.url)

11
src/util/mod.rs
View File

@@ -1,9 +1,9 @@
pub mod time;
pub mod flaresolverr;
pub mod cache;
pub mod requester;
pub mod discord;
pub mod flaresolverr;
pub mod proxy;
pub mod requester;
pub mod time;
pub fn parse_abbreviated_number(s: &str) -> Option<u32> {
let s = s.trim();
@@ -20,7 +20,10 @@ pub fn parse_abbreviated_number(s: &str) -> Option<u32> {
"" => 1.0,
_ => return None,
};
num_part.parse::<f64>().ok().map(|n| (n * multiplier) as u32)
num_part
.parse::<f64>()
.ok()
.map(|n| (n * multiplier) as u32)
}
pub fn interleave<T: Clone>(lists: &[Vec<T>]) -> Vec<T> {

15
src/util/proxy.rs
View File

@@ -1,6 +1,6 @@
use std::fmt;
use std::sync::Arc;
use std::str::FromStr;
use std::sync::Arc;
use serde::Serialize;
use tokio::sync::{OnceCell, RwLock};
@@ -71,11 +71,7 @@ impl fmt::Display for Proxy {
"{}://{}@{}:{}",
self.protocol, username, self.host, self.port
),
(None, Some(_)) => write!(
f,
"{}://{}:{}",
self.protocol, self.host, self.port
),
(None, Some(_)) => write!(f, "{}://{}:{}", self.protocol, self.host, self.port),
(None, None) => write!(f, "{}://{}:{}", self.protocol, self.host, self.port),
}
}
@@ -224,8 +220,7 @@ pub fn init_all_proxies_background(requester: Requester) {
for list in PROXY_LIST {
let proxy_cache = proxy_cache.clone();
let mut requester = requester.clone();
tasks
.spawn(async move { fetch_proxies(&mut requester, list, proxy_cache).await });
tasks.spawn(async move { fetch_proxies(&mut requester, list, proxy_cache).await });
}
while let Some(result) = tasks.join_next().await {
@@ -278,9 +273,7 @@ impl FromStr for Proxy {
.ok_or_else(|| ProxyParseError::new("proxy url is missing host"))?
.to_string();
let port = url
.port()
.unwrap_or(80);
let port = url.port().unwrap_or(80);
Ok(Proxy {
protocol: url.scheme().to_string(),

24
src/util/requester.rs
View File

@@ -1,17 +1,17 @@
use serde::Serialize;
use wreq::multipart::Form;
use std::env;
use wreq::Client;
use wreq::Proxy;
use wreq::Response;
use wreq::Version;
use wreq::header::HeaderValue;
use wreq::multipart::Form;
use wreq::redirect::Policy;
use wreq_util::Emulation;
use crate::util::proxy;
use crate::util::flaresolverr::FlareSolverrRequest;
use crate::util::flaresolverr::Flaresolverr;
use crate::util::proxy;
// A Send + Sync error type for all async paths
type AnyErr = Box<dyn std::error::Error + Send + Sync + 'static>;
@@ -99,7 +99,7 @@ impl Requester {
request.send().await
}
pub async fn post_json<S>(
pub async fn post_json<S>(
&mut self,
url: &str,
data: &S,
@@ -131,7 +131,11 @@ pub async fn post_json<S>(
data: &str,
headers: Vec<(&str, &str)>,
) -> Result<Response, wreq::Error> {
let mut request = self.client.post(url).version(Version::HTTP_11).body(data.to_string());
let mut request = self
.client
.post(url)
.version(Version::HTTP_11)
.body(data.to_string());
// Set custom headers
for (key, value) in headers.iter() {
@@ -154,8 +158,7 @@ pub async fn post_json<S>(
form: Form,
headers: Vec<(String, String)>,
_http_version: Option<Version>,
) -> Result<Response, wreq::Error>
{
) -> Result<Response, wreq::Error> {
let http_version = match _http_version {
Some(v) => v,
None => Version::HTTP_11,
@@ -178,7 +181,11 @@ pub async fn post_json<S>(
request.send().await
}
pub async fn get(&mut self, url: &str, _http_version: Option<Version>) -> Result<String, AnyErr> {
pub async fn get(
&mut self,
url: &str,
_http_version: Option<Version>,
) -> Result<String, AnyErr> {
let http_version = match _http_version {
Some(v) => v,
None => Version::HTTP_11,
@@ -190,7 +197,7 @@ pub async fn post_json<S>(
let proxy = Proxy::all(&proxy_url).unwrap();
request = request.proxy(proxy);
}
}
}
let response = request.send().await?;
if response.status().is_success() || response.status().as_u16() == 404 {
return Ok(response.text().await?);
@@ -208,7 +215,6 @@ pub async fn post_json<S>(
}
}
// If direct request failed, try FlareSolverr. Map its error to a Send+Sync error immediately,
// so no non-Send error value lives across later `.await`s.
let flare_url = match env::var("FLARE_URL") {

34
src/videos.rs
View File

@@ -37,17 +37,18 @@ pub struct VideosRequest {
pub channel: Option<String>, //"youtube",
pub sort: Option<String>, //"new",
pub query: Option<String>, //"kittens",
pub page: Option<FlexibleNumber>, //1,
pub page: Option<FlexibleNumber>, //1,
pub perPage: Option<FlexibleNumber>, //10,
// Your server's global options will be sent in the videos request
// pub flavor: "mint chocolate chip"
pub featured: Option<String>, // "featured",
pub category: Option<String>, // "pmv"
pub sites: Option<String>, //
pub filter: Option<String>, //
pub language: Option<String>, //
pub networks: Option<String>, //
pub stars: Option<String>, //
pub featured: Option<String>, // "featured",
pub category: Option<String>, // "pmv"
pub sites: Option<String>, //
pub all_provider_sites: Option<String>, //
pub filter: Option<String>, //
pub language: Option<String>, //
pub networks: Option<String>, //
pub stars: Option<String>, //
pub categories: Option<String>,
pub duration: Option<String>,
}
@@ -60,11 +61,11 @@ pub struct ServerOptions {
pub filter: Option<String>,
pub language: Option<String>, // "en"
pub requester: Option<Requester>,
pub network: Option<String>, //
pub stars: Option<String>, //
pub network: Option<String>, //
pub stars: Option<String>, //
pub categories: Option<String>, //
pub duration: Option<String>, //
pub sort: Option<String>, //
pub duration: Option<String>, //
pub sort: Option<String>, //
}
#[derive(serde::Serialize, Debug)]
@@ -78,11 +79,6 @@ pub struct VideoEmbed {
pub html: String,
pub source: String,
}
impl VideoEmbed {
pub fn new(html: String, source: String) -> Self {
VideoEmbed { html, source }
}
}
#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)]
pub struct VideoItem {
pub duration: u32, // 110,
@@ -148,7 +144,7 @@ impl VideoItem {
serde_json::from_str::<VideoItem>(&s)
}
pub fn tags(mut self, tags: Vec<String>) -> Self {
if tags.is_empty(){
if tags.is_empty() {
return self;
}
self.tags = Some(tags);
@@ -179,7 +175,7 @@ impl VideoItem {
self
}
pub fn formats(mut self, formats: Vec<VideoFormat>) -> Self {
if formats.is_empty(){
if formats.is_empty() {
return self;
}
self.formats = Some(formats);