487 lines
16 KiB
Rust
487 lines
16 KiB
Rust
use crate::api::ClientVersion;
|
|
use crate::DbPool;
|
|
use crate::providers::Provider;
|
|
use crate::status::*;
|
|
use crate::util::cache::VideoCache;
|
|
use crate::util::discord::format_error_chain;
|
|
use crate::util::discord::send_discord_error_report;
|
|
use crate::util::requester::Requester;
|
|
use crate::util::time::parse_time_to_seconds;
|
|
use crate::videos::ServerOptions;
|
|
use crate::videos::VideoItem;
|
|
use async_trait::async_trait;
|
|
use error_chain::error_chain;
|
|
use htmlentity::entity::{ICodedDataTrait, decode};
|
|
use scraper::{Html, Selector};
|
|
use std::vec;
|
|
|
|
error_chain! {
|
|
foreign_links {
|
|
Io(std::io::Error);
|
|
HttpRequest(wreq::Error);
|
|
JsonError(serde_json::Error);
|
|
}
|
|
errors {
|
|
Parse(msg: String) {
|
|
description("html parse error")
|
|
display("html parse error: {}", msg)
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Clone)]
|
|
pub struct SxyprnProvider {
|
|
url: String,
|
|
}
|
|
impl SxyprnProvider {
|
|
pub fn new() -> Self {
|
|
SxyprnProvider {
|
|
url: "https://sxyprn.com".to_string(),
|
|
}
|
|
}
|
|
|
|
fn build_channel(&self, _clientversion: ClientVersion) -> Channel {
|
|
Channel {
|
|
id: "sxyprn".to_string(),
|
|
name: "SexyPorn".to_string(),
|
|
description: "Free Porn Site".to_string(),
|
|
premium: false,
|
|
favicon: "https://www.google.com/s2/favicons?sz=64&domain=sxyprn.com".to_string(),
|
|
status: "active".to_string(),
|
|
categories: vec![],
|
|
options: vec![
|
|
ChannelOption {
|
|
id: "sort".to_string(),
|
|
title: "Sort".to_string(),
|
|
description: "Sort the Videos".to_string(),
|
|
systemImage: "list.number".to_string(),
|
|
colorName: "blue".to_string(),
|
|
options: vec![
|
|
FilterOption {
|
|
id: "latest".to_string(),
|
|
title: "Latest".to_string(),
|
|
},
|
|
FilterOption {
|
|
id: "views".to_string(),
|
|
title: "Views".to_string(),
|
|
},
|
|
FilterOption {
|
|
id: "rating".to_string(),
|
|
title: "Rating".to_string(),
|
|
},
|
|
FilterOption {
|
|
id: "orgasmic".to_string(),
|
|
title: "Orgasmic".to_string(),
|
|
},
|
|
],
|
|
multiSelect: false,
|
|
},
|
|
ChannelOption {
|
|
id: "filter".to_string(),
|
|
title: "Filter".to_string(),
|
|
description: "Filter the Videos".to_string(),
|
|
systemImage: "line.horizontal.3.decrease.circle".to_string(),
|
|
colorName: "green".to_string(),
|
|
options: vec![
|
|
FilterOption {
|
|
id: "top".to_string(),
|
|
title: "Top".to_string(),
|
|
},
|
|
FilterOption {
|
|
id: "other".to_string(),
|
|
title: "Other".to_string(),
|
|
},
|
|
FilterOption {
|
|
id: "all".to_string(),
|
|
title: "All".to_string(),
|
|
},
|
|
],
|
|
multiSelect: false,
|
|
},
|
|
],
|
|
nsfw: true,
|
|
cacheDuration: Some(1800),
|
|
}
|
|
}
|
|
async fn get(
|
|
&self,
|
|
cache: VideoCache,
|
|
pool: DbPool,
|
|
page: u8,
|
|
sort: String,
|
|
options: ServerOptions,
|
|
) -> Result<Vec<VideoItem>> {
|
|
let sort_string = match sort.as_str() {
|
|
"views" => "views",
|
|
"rating" => "rating",
|
|
"orgasmic" => "orgasmic",
|
|
_ => "latest",
|
|
};
|
|
// Extract needed fields from options at the start
|
|
let filter = options.filter.clone().unwrap_or_else(|| "top".to_string());
|
|
let filter_string = match filter.as_str() {
|
|
"other" => "other",
|
|
"all" => "all",
|
|
_ => "top",
|
|
};
|
|
let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
|
|
|
|
let url_str = format!(
|
|
"{}/blog/all/{}.html?fl={}&sm={}",
|
|
self.url,
|
|
((page as u32) - 1) * 20,
|
|
filter_string,
|
|
sort_string
|
|
);
|
|
|
|
let old_items = match cache.get(&url_str) {
|
|
Some((time, items)) => {
|
|
if time.elapsed().unwrap_or_default().as_secs() < 60 * 60 {
|
|
return Ok(items.clone());
|
|
} else {
|
|
items.clone()
|
|
}
|
|
}
|
|
None => {
|
|
vec![]
|
|
}
|
|
};
|
|
|
|
let text = match requester.get(&url_str, None).await {
|
|
Ok(text) => text,
|
|
Err(e) => {
|
|
crate::providers::report_provider_error(
|
|
"sxyprn",
|
|
"get.request",
|
|
&format!("url={url_str}; error={e}"),
|
|
)
|
|
.await;
|
|
return Ok(old_items);
|
|
}
|
|
};
|
|
// Pass a reference to options if needed, or reconstruct as needed
|
|
let video_items = match self
|
|
.get_video_items_from_html(text.clone(), pool, requester)
|
|
.await
|
|
{
|
|
Ok(items) => items,
|
|
Err(e) => {
|
|
println!("Error parsing video items: {}", e);
|
|
send_discord_error_report(
|
|
e.to_string(),
|
|
Some(format_error_chain(&e)),
|
|
Some("Sxyprn Provider"),
|
|
Some(&format!("URL: {}", url_str)),
|
|
file!(),
|
|
line!(),
|
|
module_path!(),
|
|
).await;
|
|
return Ok(old_items);
|
|
}
|
|
};
|
|
// let video_items: Vec<VideoItem> = self
|
|
// .get_video_items_from_html(text.clone(), pool, requester)
|
|
// .await;
|
|
if !video_items.is_empty() {
|
|
cache.remove(&url_str);
|
|
cache.insert(url_str.clone(), video_items.clone());
|
|
} else {
|
|
return Ok(old_items);
|
|
}
|
|
Ok(video_items)
|
|
}
|
|
|
|
async fn query(
|
|
&self,
|
|
cache: VideoCache,
|
|
pool: DbPool,
|
|
page: u8,
|
|
query: &str,
|
|
sort: String,
|
|
options: ServerOptions,
|
|
) -> Result<Vec<VideoItem>> {
|
|
let sort_string = match sort.as_str() {
|
|
"views" => "views",
|
|
"rating" => "trending",
|
|
"orgasmic" => "orgasmic",
|
|
_ => "latest",
|
|
};
|
|
// Extract needed fields from options at the start
|
|
let mut requester = crate::providers::requester_or_default(&options, module_path!(), "missing_requester");
|
|
let search_string = query.replace(" ", "-");
|
|
let url_str = format!(
|
|
"{}/{}.html?page={}&sm={}",
|
|
self.url,
|
|
search_string,
|
|
((page as u32) - 1) * 20,
|
|
sort_string
|
|
);
|
|
// Check our Video Cache. If the result is younger than 1 hour, we return it.
|
|
let old_items = match cache.get(&url_str) {
|
|
Some((time, items)) => {
|
|
if time.elapsed().unwrap_or_default().as_secs() < 60 * 60 {
|
|
return Ok(items.clone());
|
|
} else {
|
|
let _ = cache.check().await;
|
|
return Ok(items.clone());
|
|
}
|
|
}
|
|
None => {
|
|
vec![]
|
|
}
|
|
};
|
|
let text = match requester.get(&url_str, None).await {
|
|
Ok(text) => text,
|
|
Err(e) => {
|
|
crate::providers::report_provider_error(
|
|
"sxyprn",
|
|
"query.request",
|
|
&format!("url={url_str}; error={e}"),
|
|
)
|
|
.await;
|
|
return Ok(old_items);
|
|
}
|
|
};
|
|
|
|
let video_items = match self
|
|
.get_video_items_from_html(text.clone(), pool, requester)
|
|
.await
|
|
{
|
|
Ok(items) => items,
|
|
Err(e) => {
|
|
println!("Error parsing video items: {}", e);// 1. Convert the error to a string immediately
|
|
send_discord_error_report(
|
|
e.to_string(),
|
|
Some(format_error_chain(&e)),
|
|
Some("Sxyprn Provider"),
|
|
Some(&format!("URL: {}", url_str)),
|
|
file!(),
|
|
line!(),
|
|
module_path!(),
|
|
).await;
|
|
return Ok(old_items);
|
|
}
|
|
};
|
|
|
|
// let video_items: Vec<VideoItem> = self
|
|
// .get_video_items_from_html(text.clone(), pool, requester)
|
|
// .await;
|
|
if !video_items.is_empty() {
|
|
cache.remove(&url_str);
|
|
cache.insert(url_str.clone(), video_items.clone());
|
|
} else {
|
|
return Ok(old_items);
|
|
}
|
|
Ok(video_items)
|
|
}
|
|
|
|
async fn get_video_items_from_html(
|
|
&self,
|
|
html: String,
|
|
_pool: DbPool,
|
|
_requester: Requester,
|
|
) -> Result<Vec<VideoItem>> {
|
|
if html.is_empty() {
|
|
return Ok(vec![]);
|
|
}
|
|
|
|
// take content before "<script async"
|
|
let before_script = html
|
|
.split("<script async")
|
|
.next()
|
|
.ok_or_else(|| ErrorKind::Parse("missing '<script async' split point".into()))?;
|
|
|
|
// split into video segments (skip the first chunk)
|
|
let raw_videos: Vec<&str> = before_script.split("post_el_small'").skip(1).collect();
|
|
|
|
if raw_videos.is_empty() {
|
|
return Err(ErrorKind::Parse("no 'post_el_small\\'' segments found".into()).into());
|
|
}
|
|
|
|
let mut items = Vec::new();
|
|
|
|
for video_segment in raw_videos {
|
|
// url id
|
|
let url = video_segment
|
|
.split("/post/")
|
|
.nth(1)
|
|
.and_then(|s| s.split('\'').next())
|
|
.ok_or_else(|| ErrorKind::Parse("failed to extract /post/ url".into()))?
|
|
.to_string();
|
|
|
|
let video_url = format!("https://hottub.spacemoehre.de/proxy/sxyprn/post/{}", url);
|
|
|
|
// title parts
|
|
let title_parts = video_segment
|
|
.split("post_text")
|
|
.nth(1)
|
|
.and_then(|s| s.split("style=''>").nth(1))
|
|
.and_then(|s| s.split("</div>").next())
|
|
.ok_or_else(|| ErrorKind::Parse("failed to extract title_parts".into()))?;
|
|
|
|
let document = Html::parse_document(title_parts);
|
|
let selector = Selector::parse("*")
|
|
.map_err(|e| ErrorKind::Parse(format!("selector parse failed: {e}")))?;
|
|
|
|
let mut texts = Vec::new();
|
|
for element in document.select(&selector) {
|
|
let text = element.text().collect::<Vec<_>>().join(" ");
|
|
if !text.trim().is_empty() {
|
|
texts.push(text.trim().to_string());
|
|
}
|
|
}
|
|
|
|
let mut title = texts.join(" ");
|
|
title = decode(title.as_bytes())
|
|
.to_string()
|
|
.unwrap_or(title)
|
|
.replace(" ", " ");
|
|
|
|
title = title
|
|
.replace('\n', "")
|
|
.replace(" + ", " ")
|
|
.replace(" ", " ")
|
|
.trim()
|
|
.to_string();
|
|
|
|
if title.to_ascii_lowercase().starts_with("new ") {
|
|
title = title[4..].to_string();
|
|
}
|
|
|
|
// id (DON'T index [6])
|
|
let id = video_url
|
|
.split('/')
|
|
.last()
|
|
.ok_or_else(|| ErrorKind::Parse("failed to extract id from video_url".into()))?
|
|
.split('?')
|
|
.next()
|
|
.unwrap_or("")
|
|
.to_string();
|
|
|
|
// thumb
|
|
let thumb_path = video_segment
|
|
.split("<img class='mini_post_vid_thumb lazyload'")
|
|
.nth(1)
|
|
.and_then(|s| s.split("data-src='").nth(1))
|
|
.and_then(|s| s.split('\'').next())
|
|
.ok_or_else(|| ErrorKind::Parse("failed to extract thumb".into()))?;
|
|
|
|
let thumb = format!("https:{thumb_path}");
|
|
|
|
// preview
|
|
let preview = if video_segment.contains("class='hvp_player'") {
|
|
Some(format!(
|
|
"https:{}",
|
|
video_segment
|
|
.split("class='hvp_player'")
|
|
.nth(1)
|
|
.and_then(|s| s.split(" src='").nth(1))
|
|
.and_then(|s| s.split('\'').next())
|
|
.ok_or_else(|| ErrorKind::Parse("failed to extract preview src".into()))?
|
|
))
|
|
} else {
|
|
None
|
|
};
|
|
|
|
// views
|
|
let views = video_segment
|
|
.split("<strong>·</strong> ")
|
|
.nth(1)
|
|
.and_then(|s| s.split_whitespace().next())
|
|
.ok_or_else(|| ErrorKind::Parse("failed to extract views".into()))?
|
|
.to_string();
|
|
|
|
// duration
|
|
let raw_duration = video_segment
|
|
.split("duration_small")
|
|
.nth(1)
|
|
.and_then(|s| s.split("title='").nth(1))
|
|
.and_then(|s| s.split('\'').nth(1))
|
|
.and_then(|s| s.split('>').nth(1))
|
|
.and_then(|s| s.split('<').next())
|
|
.ok_or_else(|| ErrorKind::Parse("failed to extract duration".into()))?
|
|
.to_string();
|
|
|
|
let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
|
|
|
|
// stream urls (your filter condition looks suspicious; leaving as-is)
|
|
let stream_urls = video_segment
|
|
.split("extlink_icon extlink")
|
|
.filter_map(|part| {
|
|
part.split("href='")
|
|
.last()
|
|
.and_then(|s| s.split('\'').next())
|
|
.map(|u| u.to_string())
|
|
})
|
|
.filter(|url| url.starts_with("https://lulustream."))
|
|
.collect::<Vec<String>>();
|
|
|
|
let video_item_url = stream_urls.first().cloned().unwrap_or_else(|| {
|
|
format!("https://hottub.spacemoehre.de/proxy/sxyprn/post/{}", id)
|
|
});
|
|
|
|
let mut video_item = VideoItem::new(
|
|
id,
|
|
title,
|
|
video_item_url,
|
|
"sxyprn".to_string(),
|
|
thumb,
|
|
duration,
|
|
)
|
|
.views(views.parse::<u32>().unwrap_or(0));
|
|
|
|
if let Some(p) = preview {
|
|
video_item = video_item.preview(p);
|
|
}
|
|
|
|
items.push(video_item);
|
|
}
|
|
|
|
Ok(items)
|
|
}
|
|
}
|
|
|
|
#[async_trait]
|
|
impl Provider for SxyprnProvider {
|
|
async fn get_videos(
|
|
&self,
|
|
cache: VideoCache,
|
|
pool: DbPool,
|
|
sort: String,
|
|
query: Option<String>,
|
|
page: String,
|
|
per_page: String,
|
|
options: ServerOptions,
|
|
) -> Vec<VideoItem> {
|
|
let _ = per_page;
|
|
let videos: std::result::Result<Vec<VideoItem>, Error> = match query {
|
|
Some(q) => {
|
|
self.query(
|
|
cache,
|
|
pool,
|
|
page.parse::<u8>().unwrap_or(1),
|
|
&q,
|
|
sort,
|
|
options,
|
|
)
|
|
.await
|
|
}
|
|
None => {
|
|
self.get(cache, pool, page.parse::<u8>().unwrap_or(1), sort, options)
|
|
.await
|
|
}
|
|
};
|
|
match videos {
|
|
Ok(v) => v,
|
|
Err(e) => {
|
|
println!("Error fetching videos: {}", e);
|
|
vec![]
|
|
}
|
|
}
|
|
}
|
|
|
|
fn get_channel(&self, clientversion: ClientVersion) -> Option<Channel> {
|
|
Some(self.build_channel(clientversion))
|
|
}
|
|
}
|