This commit is contained in:
Simon
2026-01-14 11:49:27 +00:00
parent 602dbe50f0
commit cacd45d893
5 changed files with 278 additions and 277 deletions

View File

@@ -3,6 +3,7 @@ use crate::api::ClientVersion;
use crate::providers::Provider;
use crate::status::*;
use crate::util::cache::VideoCache;
use crate::util::discord::{format_error_chain, send_discord_error_report};
use crate::util::requester::Requester;
use crate::util::time::parse_time_to_seconds;
use crate::videos::{ServerOptions, VideoFormat, VideoItem};
@@ -18,6 +19,13 @@ error_chain! {
foreign_links {
Io(std::io::Error);
HttpRequest(wreq::Error);
Json(serde_json::Error);
}
errors {
Parse(msg: String) {
description("parse error")
display("parse error: {}", msg)
}
}
}
@@ -27,6 +35,7 @@ pub struct HqpornerProvider {
stars: Arc<RwLock<Vec<FilterOption>>>,
categories: Arc<RwLock<Vec<FilterOption>>>,
}
impl HqpornerProvider {
pub fn new() -> Self {
let provider = HqpornerProvider {
@@ -38,95 +47,101 @@ impl HqpornerProvider {
provider
}
fn spawn_initial_load(&self) {
fn spawn_initial_load(&self) {
let url = self.url.clone();
let stars = Arc::clone(&self.stars);
let categories = Arc::clone(&self.categories);
thread::spawn(move || {
// Create a tiny runtime just for these async tasks
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("build tokio runtime");
.build();
rt.block_on(async move {
if let Err(e) = Self::load_stars(&url, stars).await {
eprintln!("load_stars failed: {e}");
}
if let Err(e) = Self::load_categories(&url, categories).await {
eprintln!("load_categories failed: {e}");
}
});
if let Ok(runtime) = rt {
runtime.block_on(async move {
if let Err(e) = Self::load_stars(&url, stars).await {
eprintln!("load_stars failed: {e}");
}
if let Err(e) = Self::load_categories(&url, categories).await {
eprintln!("load_categories failed: {e}");
}
});
}
});
}
async fn load_stars(base_url: &str, stars: Arc<RwLock<Vec<FilterOption>>>) -> Result<()> {
let mut requester = Requester::new();
let text = requester
.get(format!("{}/girls", &base_url).as_str(), None)
.get(&format!("{}/girls", base_url), None)
.await
.unwrap();
.map_err(|e| Error::from(format!("Request failed: {}", e)))?;
let stars_div = text
.split("<span>Girls</span>")
.collect::<Vec<&str>>().last().unwrap()
.split("</ul>")
.collect::<Vec<&str>>()[0];
for stars_element in stars_div.split("<li ").collect::<Vec<&str>>()[1..].to_vec() {
let star_id = stars_element.split("href=\"/actress/").collect::<Vec<&str>>()[1]
.split("\"")
.collect::<Vec<&str>>()[0]
.to_string();
let star_name = stars_element.split("<a ").collect::<Vec<&str>>()[1]
.split(">").collect::<Vec<&str>>()[1]
.split("<")
.collect::<Vec<&str>>()[0]
.to_string();
Self::push_unique(
&stars,
FilterOption {
id: star_id,
title: star_name,
},
);
.last()
.and_then(|s| s.split("</ul>").next())
.ok_or_else(|| Error::from("Could not find stars div"))?;
for stars_element in stars_div.split("<li ").skip(1) {
let star_id = stars_element
.split("href=\"/actress/")
.nth(1)
.and_then(|s| s.split('"').next())
.map(|s| s.to_string());
let star_name = stars_element
.split("<a ")
.nth(1)
.and_then(|s| s.split('>').nth(1))
.and_then(|s| s.split('<').next())
.map(|s| s.to_string());
if let (Some(id), Some(name)) = (star_id, star_name) {
Self::push_unique(&stars, FilterOption { id, title: name });
}
}
return Ok(());
Ok(())
}
async fn load_categories(base_url: &str, categories: Arc<RwLock<Vec<FilterOption>>>) -> Result<()> {
async fn load_categories(
base_url: &str,
categories: Arc<RwLock<Vec<FilterOption>>>,
) -> Result<()> {
let mut requester = Requester::new();
let text = requester
.get(format!("{}/categories", &base_url).as_str(), None)
.get(&format!("{}/categories", base_url), None)
.await
.unwrap();
.map_err(|e| Error::from(format!("Request failed: {}", e)))?;
let categories_div = text
.split("<span>Categories</span>")
.collect::<Vec<&str>>().last().unwrap()
.split("</ul>")
.collect::<Vec<&str>>()[0];
for categories_element in categories_div.split("<li ").collect::<Vec<&str>>()[1..].to_vec() {
let category_id = categories_element.split("href=\"/category/").collect::<Vec<&str>>()[1]
.split("\"")
.collect::<Vec<&str>>()[0]
.to_string();
let category_name = categories_element.split("<a ").collect::<Vec<&str>>()[1]
.split(">").collect::<Vec<&str>>()[1]
.split("<")
.collect::<Vec<&str>>()[0]
.titlecase();
Self::push_unique(
&categories,
FilterOption {
id: category_id,
title: category_name,
},
);
.last()
.and_then(|s| s.split("</ul>").next())
.ok_or_else(|| Error::from("Could not find categories div"))?;
for categories_element in categories_div.split("<li ").skip(1) {
let category_id = categories_element
.split("href=\"/category/")
.nth(1)
.and_then(|s| s.split('"').next())
.map(|s| s.to_string());
let category_name = categories_element
.split("<a ")
.nth(1)
.and_then(|s| s.split('>').nth(1))
.and_then(|s| s.split('<').next())
.map(|s| s.titlecase());
if let (Some(id), Some(name)) = (category_id, category_name) {
Self::push_unique(&categories, FilterOption { id, title: name });
}
}
return Ok(());
Ok(())
}
fn build_channel(&self, clientversion: ClientVersion) -> Channel {
let _ = clientversion;
fn build_channel(&self, _clientversion: ClientVersion) -> Channel {
Channel {
id: "hqporner".to_string(),
name: "HQPorner".to_string(),
@@ -134,20 +149,21 @@ impl HqpornerProvider {
premium: false,
favicon: "https://www.google.com/s2/favicons?sz=64&domain=hqporner.com".to_string(),
status: "active".to_string(),
categories: self.categories.read().unwrap().iter().map(|c| c.title.clone()).collect(),
categories: self
.categories
.read()
.map(|c| c.iter().map(|o| o.title.clone()).collect())
.unwrap_or_default(),
options: vec![],
nsfw: true,
cacheDuration: None,
}
}
// Push one item with minimal lock time and dedup by id
fn push_unique(target: &Arc<RwLock<Vec<FilterOption>>>, item: FilterOption) {
if let Ok(mut vec) = target.write() {
if !vec.iter().any(|x| x.id == item.id) {
vec.push(item);
// Optional: keep it sorted for nicer UX
// vec.sort_by(|a,b| a.title.cmp(&b.title));
}
}
}
@@ -156,34 +172,25 @@ impl HqpornerProvider {
&self,
cache: VideoCache,
page: u8,
sort: &str,
_sort: &str,
options: ServerOptions,
) -> Result<Vec<VideoItem>> {
let _ = sort;
let video_url = format!("{}/hdporn/{}", self.url, page);
let old_items = match cache.get(&video_url) {
Some((time, items)) => {
if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
return Ok(items.clone());
} else {
items.clone()
}
if let Some((time, items)) = cache.get(&video_url) {
if time.elapsed().unwrap_or_default().as_secs() < 300 {
return Ok(items.clone());
}
None => {
vec![]
}
};
}
let mut requester = options.requester.clone().unwrap();
let text = requester.get(&video_url, None).await.unwrap();
let video_items: Vec<VideoItem> = self
.get_video_items_from_html(text.clone(), &mut requester)
.await;
let mut requester = options.requester.clone().ok_or("No requester")?;
let text = requester
.get(&video_url, None)
.await
.map_err(|e| Error::from(format!("Request failed: {}", e)))?;
let video_items = self.get_video_items_from_html(text, &mut requester).await;
if !video_items.is_empty() {
cache.remove(&video_url);
cache.insert(video_url.clone(), video_items.clone());
} else {
return Ok(old_items);
cache.insert(video_url, video_items.clone());
}
Ok(video_items)
}
@@ -195,42 +202,41 @@ impl HqpornerProvider {
query: &str,
options: ServerOptions,
) -> Result<Vec<VideoItem>> {
let search_string = query.trim().to_string();
let search_string = query.trim().to_lowercase();
let mut video_url = format!("{}/?q={}&p={}", self.url, search_string, page);
if let Some(star) = self.stars.read().unwrap().iter().find(|s| s.title.to_ascii_lowercase() == search_string.to_ascii_lowercase()) {
video_url = format!("{}/actress/{}/{}", self.url, star.id, page);
}
if let Some(cat) = self.categories.read().unwrap().iter().find(|c| c.title.to_ascii_lowercase() == search_string.to_ascii_lowercase()) {
video_url = format!("{}/category/{}/{}", self.url, cat.id, page);
}
// Check our Video Cache. If the result is younger than 1 hour, we return it.
let old_items = match cache.get(&video_url) {
Some((time, items)) => {
if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
return Ok(items.clone());
} else {
let _ = cache.check().await;
return Ok(items.clone());
}
if let Ok(stars) = self.stars.read() {
if let Some(star) = stars
.iter()
.find(|s| s.title.to_lowercase() == search_string)
{
video_url = format!("{}/actress/{}/{}", self.url, star.id, page);
}
None => {
vec![]
}
if let Ok(cats) = self.categories.read() {
if let Some(cat) = cats
.iter()
.find(|c| c.title.to_lowercase() == search_string)
{
video_url = format!("{}/category/{}/{}", self.url, cat.id, page);
}
};
}
let mut requester = options.requester.clone().unwrap();
if let Some((time, items)) = cache.get(&video_url) {
if time.elapsed().unwrap_or_default().as_secs() < 300 {
return Ok(items.clone());
}
}
let text = requester.get(&video_url, None).await.unwrap();
let video_items: Vec<VideoItem> = self
.get_video_items_from_html(text.clone(), &mut requester)
.await;
let mut requester = options.requester.clone().ok_or("No requester")?;
let text = requester
.get(&video_url, None)
.await
.map_err(|e| Error::from(format!("Request failed: {}", e)))?;
let video_items = self.get_video_items_from_html(text, &mut requester).await;
if !video_items.is_empty() {
cache.remove(&video_url);
cache.insert(video_url.clone(), video_items.clone());
} else {
return Ok(old_items);
cache.insert(video_url, video_items.clone());
}
Ok(video_items)
}
@@ -243,185 +249,188 @@ impl HqpornerProvider {
if html.is_empty() || html.contains("404 Not Found") {
return vec![];
}
let raw_videos = html.split("id=\"footer\"").collect::<Vec<&str>>()[0]
.split("<section class=\"box features\">")
.collect::<Vec<&str>>()[2]
.split("<section class=\"box feature\">")
.collect::<Vec<&str>>()[1..]
.to_vec();
let raw_videos: Vec<String> = html
.split("id=\"footer\"")
.next()
.and_then(|s| s.split("<section class=\"box features\">").nth(2))
.map(|s| {
s.split("<section class=\"box feature\">")
.skip(1)
.map(|v| v.to_string())
.collect()
})
.unwrap_or_default();
let futures = raw_videos
.into_iter()
.map(|el| self.get_video_item(el.to_string(), requester.clone()));
let results: Vec<Result<VideoItem>> = join_all(futures).await;
let video_items: Vec<VideoItem> = results.into_iter().filter_map(Result::ok).collect();
return video_items;
.map(|el| self.get_video_item(el, requester.clone()));
join_all(futures)
.await
.into_iter()
.inspect(|r| {
if let Err(e) = r {
let msg = e.to_string();
let chain = format_error_chain(e);
tokio::spawn(async move {
let _ = send_discord_error_report(
msg,
Some(chain),
Some("Hqporner Provider"),
None,
file!(),
line!(),
module_path!(),
)
.await;
});
}
})
.filter_map(Result::ok)
.collect()
}
async fn get_video_item(
&self,
video_segment: String,
mut requester: Requester,
) -> Result<VideoItem> {
let video_url: String = format!(
async fn get_video_item(&self, seg: String, mut requester: Requester) -> Result<VideoItem> {
let video_url = format!(
"{}{}",
self.url,
video_segment.split("<a href=\"").collect::<Vec<&str>>()[1]
.split("\"")
.collect::<Vec<&str>>()[0]
.to_string()
seg.split("<a href=\"")
.nth(1)
.and_then(|s| s.split('"').next())
.ok_or_else(|| ErrorKind::Parse(format!("url \n{seg}").into()))?
);
let mut title = video_segment
let title_raw = seg
.split("<h3 class=\"meta-data-title\">")
.collect::<Vec<&str>>()[1]
.split(">")
.collect::<Vec<&str>>()[1]
.split("<")
.collect::<Vec<&str>>()[0]
.trim()
.to_string();
// html decode
title = decode(title.as_bytes())
.nth(1)
.and_then(|s| s.split('>').nth(1))
.and_then(|s| s.split('<').next())
.ok_or_else(|| ErrorKind::Parse(format!("title \n{seg}").into()))?;
let title = decode(title_raw.as_bytes())
.to_string()
.unwrap_or(title)
.unwrap_or_else(|_| title_raw.to_string())
.titlecase();
let id = video_url.split("/").collect::<Vec<&str>>()[4]
.split(".")
.collect::<Vec<&str>>()[0]
.to_string();
let id = video_url
.split('/')
.nth(4)
.and_then(|s| s.split('.').next())
.ok_or_else(|| ErrorKind::Parse(format!("id \n{seg}").into()))?
.to_string();
let thumb = format!(
"https:{}",
video_segment
.split("onmouseleave='defaultImage(\"")
.collect::<Vec<&str>>()[1]
.split("\"")
.collect::<Vec<&str>>()[0]
.to_string()
seg.split("onmouseleave='defaultImage(\"")
.nth(1)
.and_then(|s| s.split('"').next())
.ok_or_else(|| ErrorKind::Parse(format!("thumb \n{seg}").into()))?
);
let raw_duration = video_segment
let raw_duration = seg
.split("<span class=\"icon fa-clock-o meta-data\">")
.collect::<Vec<&str>>()[1]
.split("s<")
.collect::<Vec<&str>>()[0]
.replace("m ", ":")
.to_string();
let duration = parse_time_to_seconds(raw_duration.as_str()).unwrap_or(0) as u32;
.nth(1)
.and_then(|s| s.split("s<").next())
.map(|s| s.replace("m ", ":"))
.unwrap_or_default();
let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
let (tags, formats) = match self.extract_media(&video_url, &mut requester).await{
Ok((t, f)) => (t, f),
Err(_) => return Err(Error::from("Video media extraction failed")),
};
if formats.is_empty() {
return Err(Error::from("No formats found for video"));
}
let video_item = VideoItem::new(
id,
title,
video_url,
"hqporner".to_string(),
thumb,
duration,
let (tags, formats) = self.extract_media(&video_url, &mut requester).await?;
Ok(
VideoItem::new(id, title, video_url, "hqporner".into(), thumb, duration)
.formats(formats)
.tags(tags),
)
.formats(formats)
.tags(tags);
return Ok(video_item);
}
async fn extract_media(
&self,
video_page_url: &str,
url: &str,
requester: &mut Requester,
) -> Result<(Vec<String>, Vec<VideoFormat>)> {
let mut formats = vec![];
let mut tags = vec![];
let text = requester.get_raw_with_headers(&video_page_url, vec![("Referer".to_string(), "https://hqporner.com".to_string())])
.await.unwrap().text().await.unwrap();
let resp = requester
.get_raw_with_headers(
url,
vec![("Referer".to_string(), "https://hqporner.com".into())],
)
.await
.map_err(|e| Error::from(format!("Request failed: {}", e)))?;
let text = resp
.text()
.await
.map_err(|e| Error::from(format!("Text conversion failed: {}", e)))?;
if text.contains("Why do I see it?") {
return Ok((tags, formats));
}
let stars_elements = text.split("icon fa-star-o").collect::<Vec<&str>>()[1]
.split("</li>")
.collect::<Vec<&str>>()[0]
.split("href=\"/actress/")
.collect::<Vec<&str>>()[1..]
.to_vec();
for star_el in stars_elements {
let star_id = star_el.split("\"").collect::<Vec<&str>>()[0].to_string();
let star_name = star_el.split("\">").collect::<Vec<&str>>()[1]
.split("<")
.collect::<Vec<&str>>()[0]
.to_string();
tags.push(star_name.clone());
Self::push_unique(&self.stars, FilterOption {
id: star_id,
title: star_name.clone(),
});
}
let categories_elements = text.split("This video belongs to the following categories").collect::<Vec<&str>>()[1]
.split("</p>")
.collect::<Vec<&str>>()[0]
.split("href=\"/category/")
.collect::<Vec<&str>>()[1..]
.to_vec();
for categories_el in categories_elements {
let category_id = categories_el.split("\"").collect::<Vec<&str>>()[0].to_string();
let category_name = categories_el.split("\">").collect::<Vec<&str>>()[1]
.split("<")
.collect::<Vec<&str>>()[0].titlecase();
tags.push(category_name.clone());
Self::push_unique(&self.categories, FilterOption {
id: category_id,
title: category_name.clone(),
});
// Extract Stars & Tags
if let Some(stars_block) = text
.split("icon fa-star-o")
.nth(1)
.and_then(|s| s.split("</li>").next())
{
for star_el in stars_block.split("href=\"/actress/").skip(1) {
let id = star_el.split('"').next().unwrap_or("").to_string();
let name = star_el
.split("\">")
.nth(1)
.and_then(|s| s.split('<').next())
.unwrap_or("")
.to_string();
if !name.is_empty() {
tags.push(name.clone());
Self::push_unique(&self.stars, FilterOption { id, title: name });
}
}
}
let video_url = format!(
// Player / Video Extraction
let player_url = format!(
"https:{}",
text.split("url: '/blocks/altplayer.php?i=")
.collect::<Vec<&str>>()[1]
.split("'")
.collect::<Vec<&str>>()[0]
.nth(1)
.and_then(|s| s.split('\'').next())
.ok_or("No player link")?
);
let text2 = requester
.get_raw_with_headers(
&video_url,
vec![("Referer".to_string(), "https://hqporner.com/".to_string())],
&player_url,
vec![("Referer".to_string(), "https://hqporner.com/".into())],
)
.await
.unwrap()
.await?
.text()
.await
.unwrap();
match text2.split("<video ").collect::<Vec<&str>>().len() > 2 {
false => return Err(Error::from("No video element found")),
true => (),
}
let video_element = text2.split("<video ").collect::<Vec<&str>>()[2]
.split("</video>")
.collect::<Vec<&str>>()[0];
let sources = video_element.split("<source ").collect::<Vec<&str>>()[1..].to_vec();
for source in sources {
let title = source.split("title=\\\"").collect::<Vec<&str>>()[1]
.split("\\\"")
.collect::<Vec<&str>>()[0]
.await?;
let video_element = text2
.split("<video ")
.nth(2)
.and_then(|s| s.split("</video>").next())
.ok_or("No video element")?;
for source in video_element.split("<source ").skip(1) {
let title = source
.split("title=\\\"")
.nth(1)
.and_then(|s| s.split("\\\"").next())
.unwrap_or("")
.to_string();
let quality = title.split(" ").collect::<Vec<&str>>()[0].to_string();
let format = "mp4".to_string();
let quality = title.split(' ').next().unwrap_or("HD").to_string();
let media_url = format!(
"https:{}",
source.split("src=\\\"").collect::<Vec<&str>>()[1]
.split("\\\"")
.collect::<Vec<&str>>()[0]
source
.split("src=\\\"")
.nth(1)
.and_then(|s| s.split("\\\"").next())
.unwrap_or("")
);
formats.push(
VideoFormat::new(media_url, quality, format)
VideoFormat::new(media_url, quality, "mp4".into())
.format_id(title.clone())
.format_note(title.clone()),
.format_note(title),
);
}
Ok((tags, formats))
}
}
@@ -431,34 +440,26 @@ impl Provider for HqpornerProvider {
async fn get_videos(
&self,
cache: VideoCache,
pool: DbPool,
_pool: DbPool,
sort: String,
query: Option<String>,
page: String,
per_page: String,
_per_page: String,
options: ServerOptions,
) -> Vec<VideoItem> {
let _ = per_page;
let _ = pool;
let videos: std::result::Result<Vec<VideoItem>, Error> = match query {
Some(q) => {
self.query(cache, page.parse::<u8>().unwrap_or(1), &q, options)
.await
}
None => {
self.get(cache, page.parse::<u8>().unwrap_or(1), &sort, options)
.await
}
let page_num = page.parse::<u8>().unwrap_or(1);
let res = match query {
Some(q) => self.query(cache, page_num, &q, options).await,
None => self.get(cache, page_num, &sort, options).await,
};
match videos {
Ok(v) => v,
Err(e) => {
println!("Error fetching videos: {}", e);
vec![]
}
}
res.unwrap_or_else(|e| {
eprintln!("Hqporner error: {e}");
let _ = send_discord_error_report(e.to_string(), Some(format_error_chain(&e)), None, None, file!(), line!(), module_path!());
vec![]
})
}
fn get_channel(&self, clientversion: ClientVersion) -> Option<crate::status::Channel> {
Some(self.build_channel(clientversion))
fn get_channel(&self, v: ClientVersion) -> Option<Channel> {
Some(self.build_channel(v))
}
}

View File

@@ -272,7 +272,7 @@ impl JavtifulProvider {
.split(" href=\"")
.nth(1)
.and_then(|s| s.split('"').next())
.ok_or_else(|| ErrorKind::Parse("video url".into()))?
.ok_or_else(|| ErrorKind::Parse("video url\n\n{seg}".into()))?
.to_string();
let mut title = seg
@@ -288,12 +288,12 @@ impl JavtifulProvider {
.split('/')
.nth(5)
.and_then(|s| s.split('.').next())
.ok_or_else(|| ErrorKind::Parse("video id".into()))?
.ok_or_else(|| ErrorKind::Parse("video id\n\n{seg}".into()))?
.to_string();
let thumb_block = seg
.split("<img ")
.nth(1)
.ok_or_else(|| ErrorKind::Parse("thumb block".into()))?;
.ok_or_else(|| ErrorKind::Parse("thumb block\n\n{seg}".into()))?;
let thumb = thumb_block
.split("data-src=\"")

View File

@@ -103,7 +103,7 @@ fn build_channel(&self, clientversion: ClientVersion) -> Channel {
old_items = match cache.get(&index) {
Some((time, items)) => {
if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
println!("Cache hit for URL: {}", url);
// println!("Cache hit for URL: {}", url);
return Ok(items.clone());
} else {
items.clone()

View File

@@ -55,7 +55,7 @@ impl Rule34videoProvider {
old_items = match cache.get(&index) {
Some((time, items)) => {
if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
println!("Cache hit for URL: {}", url);
// println!("Cache hit for URL: {}", url);
return Ok(items.clone());
} else {
items.clone()