clapdat
This commit is contained in:
5
build.rs
5
build.rs
@@ -281,6 +281,11 @@ const PROVIDERS: &[ProviderDef] = &[
|
|||||||
module: "chaturbate",
|
module: "chaturbate",
|
||||||
ty: "ChaturbateProvider",
|
ty: "ChaturbateProvider",
|
||||||
},
|
},
|
||||||
|
ProviderDef {
|
||||||
|
id: "clapdat",
|
||||||
|
module: "clapdat",
|
||||||
|
ty: "ClapdatProvider",
|
||||||
|
},
|
||||||
ProviderDef {
|
ProviderDef {
|
||||||
id: "archivebate",
|
id: "archivebate",
|
||||||
module: "archivebate",
|
module: "archivebate",
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ This is the current implementation inventory as of this snapshot of the repo. Us
|
|||||||
| `beeg` | `mainstream-tube` | no | no | Basic mainstream tube pattern. |
|
| `beeg` | `mainstream-tube` | no | no | Basic mainstream tube pattern. |
|
||||||
| `blowjobspro` | `mainstream-tube` | no | no | KVS-style HTML provider with async search pagination and category shortcut routing. |
|
| `blowjobspro` | `mainstream-tube` | no | no | KVS-style HTML provider with async search pagination and category shortcut routing. |
|
||||||
| `chaturbate` | `live-cams` | no | no | Live cam channel. |
|
| `chaturbate` | `live-cams` | no | no | Live cam channel. |
|
||||||
|
| `clapdat` | `amateur-homemade` | no | yes | Svelte/JSON-hydrated provider using home/recent/trending routes, Meilisearch keyword search, and `/proxy/clapdat/...` redirect playback resolution. |
|
||||||
| `erome` | `amateur-homemade` | no | no | HTML album scraper with hot/new feeds, keyword search, and uploader-slug shortcuts (`uploader:<name>`). |
|
| `erome` | `amateur-homemade` | no | no | HTML album scraper with hot/new feeds, keyword search, and uploader-slug shortcuts (`uploader:<name>`). |
|
||||||
| `freepornvideosxxx` | `studio-network` | no | no | Studio-style scraper. |
|
| `freepornvideosxxx` | `studio-network` | no | no | Studio-style scraper. |
|
||||||
| `freeuseporn` | `fetish-kink` | no | no | Fetish archive pattern. |
|
| `freeuseporn` | `fetish-kink` | no | no | Fetish archive pattern. |
|
||||||
|
|||||||
525
src/providers/clapdat.rs
Normal file
525
src/providers/clapdat.rs
Normal file
@@ -0,0 +1,525 @@
|
|||||||
|
use crate::DbPool;
|
||||||
|
use crate::api::ClientVersion;
|
||||||
|
use crate::providers::{
|
||||||
|
Provider, build_proxy_url, report_provider_error, requester_or_default, strip_url_scheme,
|
||||||
|
};
|
||||||
|
use crate::status::*;
|
||||||
|
use crate::util::time::parse_time_to_seconds;
|
||||||
|
use crate::videos::{ServerOptions, VideoItem};
|
||||||
|
|
||||||
|
use async_trait::async_trait;
|
||||||
|
use chrono::NaiveDate;
|
||||||
|
use error_chain::error_chain;
|
||||||
|
use htmlentity::entity::{ICodedDataTrait, decode};
|
||||||
|
use regex::Regex;
|
||||||
|
use scraper::{ElementRef, Html, Selector};
|
||||||
|
use serde::Deserialize;
|
||||||
|
use std::collections::HashSet;
|
||||||
|
use wreq::Version;
|
||||||
|
|
||||||
|
pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata =
|
||||||
|
crate::providers::ProviderChannelMetadata {
|
||||||
|
group_id: "amateur-homemade",
|
||||||
|
tags: &["amateur", "homemade", "interracial"],
|
||||||
|
};
|
||||||
|
|
||||||
|
const BASE_URL: &str = "https://www.clapdat.com";
|
||||||
|
const SEARCH_URL: &str = "https://search.clapdat.com/indexes/videos/search";
|
||||||
|
const SEARCH_KEY: &str = "36ce9a190ca0e797debc3f0a2a311749dbd76262c389531c3a37e9dd74ab9df5";
|
||||||
|
const CHANNEL_ID: &str = "clapdat";
|
||||||
|
|
||||||
|
error_chain! {
|
||||||
|
foreign_links {
|
||||||
|
Io(std::io::Error);
|
||||||
|
HttpRequest(wreq::Error);
|
||||||
|
Json(serde_json::Error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct ClapdatProvider {
|
||||||
|
url: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
enum Target {
|
||||||
|
Trending,
|
||||||
|
Recent,
|
||||||
|
Search { query: String },
|
||||||
|
Tag { slug: String },
|
||||||
|
User { username: String },
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
struct StubVideo {
|
||||||
|
id: String,
|
||||||
|
title: String,
|
||||||
|
url: String,
|
||||||
|
thumb: String,
|
||||||
|
duration: u32,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
struct SearchResponse {
|
||||||
|
#[serde(default)]
|
||||||
|
hits: Vec<SearchHit>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
struct SearchHit {
|
||||||
|
#[serde(rename = "_id", default)]
|
||||||
|
id: String,
|
||||||
|
#[serde(default)]
|
||||||
|
title: String,
|
||||||
|
#[serde(default)]
|
||||||
|
slug: String,
|
||||||
|
#[serde(default)]
|
||||||
|
image: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ClapdatProvider {
|
||||||
|
pub fn new() -> Self {
|
||||||
|
Self {
|
||||||
|
url: BASE_URL.to_string(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn build_channel(&self, _clientversion: ClientVersion) -> Channel {
|
||||||
|
Channel {
|
||||||
|
id: CHANNEL_ID.to_string(),
|
||||||
|
name: "ClapDat".to_string(),
|
||||||
|
description: "ClapDat trending/recent feeds with tag and uploader shortcuts.".to_string(),
|
||||||
|
premium: false,
|
||||||
|
favicon: "https://www.google.com/s2/favicons?sz=64&domain=clapdat.com".to_string(),
|
||||||
|
status: "active".to_string(),
|
||||||
|
categories: vec![],
|
||||||
|
options: vec![ChannelOption {
|
||||||
|
id: "sort".to_string(),
|
||||||
|
title: "Sort".to_string(),
|
||||||
|
description: "Trending or latest ClapDat feed.".to_string(),
|
||||||
|
systemImage: "list.number".to_string(),
|
||||||
|
colorName: "blue".to_string(),
|
||||||
|
options: vec![
|
||||||
|
FilterOption {
|
||||||
|
id: "trending".to_string(),
|
||||||
|
title: "Trending".to_string(),
|
||||||
|
},
|
||||||
|
FilterOption {
|
||||||
|
id: "new".to_string(),
|
||||||
|
title: "Recent".to_string(),
|
||||||
|
},
|
||||||
|
],
|
||||||
|
multiSelect: false,
|
||||||
|
}],
|
||||||
|
nsfw: true,
|
||||||
|
cacheDuration: Some(1800),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn resolve_target(&self, query: &str, sort: &str) -> Target {
|
||||||
|
let q = query.trim();
|
||||||
|
if let Some(value) = q.strip_prefix("tag:").or_else(|| q.strip_prefix('#')) {
|
||||||
|
let slug = value.trim().to_lowercase().replace(' ', "-");
|
||||||
|
if !slug.is_empty() {
|
||||||
|
return Target::Tag { slug };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some(value) = q
|
||||||
|
.strip_prefix("user:")
|
||||||
|
.or_else(|| q.strip_prefix("uploader:"))
|
||||||
|
{
|
||||||
|
let username = value.trim().to_lowercase().replace(' ', "-");
|
||||||
|
if !username.is_empty() {
|
||||||
|
return Target::User { username };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if !q.is_empty() {
|
||||||
|
return Target::Search {
|
||||||
|
query: q.to_string(),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
match sort {
|
||||||
|
"recent" | "new" | "latest" => Target::Recent,
|
||||||
|
_ => Target::Trending,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn listing_url(&self, target: &Target, page: u16) -> Option<String> {
|
||||||
|
let page = page.max(1);
|
||||||
|
match target {
|
||||||
|
Target::Trending => Some(if page == 1 {
|
||||||
|
self.url.clone()
|
||||||
|
} else {
|
||||||
|
format!("{}/trending/{page}", self.url)
|
||||||
|
}),
|
||||||
|
Target::Recent => Some(if page == 1 {
|
||||||
|
self.url.clone()
|
||||||
|
} else {
|
||||||
|
format!("{}/recent/{page}", self.url)
|
||||||
|
}),
|
||||||
|
Target::Tag { slug } => Some(if page == 1 {
|
||||||
|
format!("{}/tag/{slug}", self.url)
|
||||||
|
} else {
|
||||||
|
format!("{}/tag/{slug}/{page}", self.url)
|
||||||
|
}),
|
||||||
|
Target::User { username } => Some(if page == 1 {
|
||||||
|
format!("{}/user/{username}", self.url)
|
||||||
|
} else {
|
||||||
|
format!("{}/user/{username}/{page}", self.url)
|
||||||
|
}),
|
||||||
|
Target::Search { .. } => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn selector(value: &str) -> Result<Selector> {
|
||||||
|
Selector::parse(value).map_err(|error| Error::from(format!("selector `{value}`: {error}")))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn regex(value: &str) -> Result<Regex> {
|
||||||
|
Regex::new(value).map_err(|error| Error::from(format!("regex `{value}`: {error}")))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn normalize_text(value: &str) -> String {
|
||||||
|
decode(value.as_bytes())
|
||||||
|
.to_string()
|
||||||
|
.unwrap_or_else(|_| value.to_string())
|
||||||
|
.replace('\u{a0}', " ")
|
||||||
|
.split_whitespace()
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(" ")
|
||||||
|
.trim()
|
||||||
|
.to_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn normalize_url(&self, value: &str) -> String {
|
||||||
|
let trimmed = value.trim();
|
||||||
|
if trimmed.starts_with("http://") || trimmed.starts_with("https://") {
|
||||||
|
return trimmed.to_string();
|
||||||
|
}
|
||||||
|
if trimmed.starts_with("//") {
|
||||||
|
return format!("https:{trimmed}");
|
||||||
|
}
|
||||||
|
format!(
|
||||||
|
"{}/{}",
|
||||||
|
self.url.trim_end_matches('/'),
|
||||||
|
trimmed.trim_start_matches('/')
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn extract_video_id(url: &str) -> Option<String> {
|
||||||
|
let re = Regex::new(r"-([a-z0-9]+)(?:/|$)").ok()?;
|
||||||
|
re.captures(url)
|
||||||
|
.and_then(|caps| caps.get(1).map(|m| m.as_str().to_string()))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_duration(value: &str) -> u32 {
|
||||||
|
parse_time_to_seconds(value)
|
||||||
|
.and_then(|seconds| u32::try_from(seconds).ok())
|
||||||
|
.unwrap_or(0)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_card(&self, card: &ElementRef<'_>, link_sel: &Selector, img_sel: &Selector) -> Option<StubVideo> {
|
||||||
|
let link = card.select(link_sel).next()?;
|
||||||
|
let href = link.value().attr("href")?;
|
||||||
|
let url = self.normalize_url(href);
|
||||||
|
let id = Self::extract_video_id(&url)?;
|
||||||
|
|
||||||
|
let title = card
|
||||||
|
.select(&Self::selector("h3").ok()?)
|
||||||
|
.next()
|
||||||
|
.map(|node| Self::normalize_text(&node.text().collect::<Vec<_>>().join(" ")))
|
||||||
|
.unwrap_or_default();
|
||||||
|
if title.is_empty() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
let duration_text = card
|
||||||
|
.select(&Self::selector("span").ok()?)
|
||||||
|
.filter_map(|node| {
|
||||||
|
let value = Self::normalize_text(&node.text().collect::<Vec<_>>().join(" "));
|
||||||
|
if value.contains(':') { Some(value) } else { None }
|
||||||
|
})
|
||||||
|
.next()
|
||||||
|
.unwrap_or_default();
|
||||||
|
|
||||||
|
let thumb = card
|
||||||
|
.select(img_sel)
|
||||||
|
.find_map(|img| img.value().attr("src").or_else(|| img.value().attr("data-src")))
|
||||||
|
.map(|value| self.normalize_url(value))
|
||||||
|
.unwrap_or_default();
|
||||||
|
|
||||||
|
Some(StubVideo {
|
||||||
|
id,
|
||||||
|
title,
|
||||||
|
url,
|
||||||
|
thumb,
|
||||||
|
duration: Self::parse_duration(&duration_text),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_listing_html(&self, html: &str) -> Result<Vec<StubVideo>> {
|
||||||
|
let doc = Html::parse_document(html);
|
||||||
|
let card_sel = Self::selector("div.video-card")?;
|
||||||
|
let link_sel = Self::selector("a[href*='/video/']")?;
|
||||||
|
let img_sel = Self::selector("img")?;
|
||||||
|
let mut out = Vec::new();
|
||||||
|
let mut seen = HashSet::new();
|
||||||
|
|
||||||
|
for card in doc.select(&card_sel) {
|
||||||
|
if let Some(stub) = self.parse_card(&card, &link_sel, &img_sel) {
|
||||||
|
if seen.insert(stub.id.clone()) {
|
||||||
|
out.push(stub);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(out)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_home_section_html(&self, html: &str, section_id: &str) -> Result<Vec<StubVideo>> {
|
||||||
|
let doc = Html::parse_document(html);
|
||||||
|
let section_sel = Self::selector(&format!("section#{section_id}"))?;
|
||||||
|
let card_sel = Self::selector("div.video-card")?;
|
||||||
|
let link_sel = Self::selector("a[href*='/video/']")?;
|
||||||
|
let img_sel = Self::selector("img")?;
|
||||||
|
let mut out = Vec::new();
|
||||||
|
let mut seen = HashSet::new();
|
||||||
|
|
||||||
|
if let Some(section) = doc.select(§ion_sel).next() {
|
||||||
|
for card in section.select(&card_sel) {
|
||||||
|
if let Some(stub) = self.parse_card(&card, &link_sel, &img_sel) {
|
||||||
|
if seen.insert(stub.id.clone()) {
|
||||||
|
out.push(stub);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(out)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn html_headers(&self) -> Vec<(String, String)> {
|
||||||
|
vec![
|
||||||
|
("accept".to_string(), "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8".to_string()),
|
||||||
|
("accept-language".to_string(), "en-US,en;q=0.8".to_string()),
|
||||||
|
("user-agent".to_string(), "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36".to_string()),
|
||||||
|
("referer".to_string(), self.url.clone()),
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn fetch_html(&self, options: &ServerOptions, url: &str) -> Result<String> {
|
||||||
|
let mut requester = requester_or_default(options, CHANNEL_ID, "fetch_html");
|
||||||
|
requester
|
||||||
|
.get_with_headers(url, self.html_headers(), Some(Version::HTTP_11))
|
||||||
|
.await
|
||||||
|
.map_err(|error| Error::from(format!("request failed for {url}: {error}")))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn search_videos(
|
||||||
|
&self,
|
||||||
|
options: &ServerOptions,
|
||||||
|
query: &str,
|
||||||
|
page: u16,
|
||||||
|
per_page: usize,
|
||||||
|
) -> Result<Vec<StubVideo>> {
|
||||||
|
let mut requester = requester_or_default(options, CHANNEL_ID, "search_videos");
|
||||||
|
let offset = page.saturating_sub(1) as usize * per_page;
|
||||||
|
let query_encoded = url::form_urlencoded::byte_serialize(query.as_bytes()).collect::<String>();
|
||||||
|
let search_url = format!(
|
||||||
|
"{SEARCH_URL}?q={query_encoded}&limit={per_page}&offset={offset}"
|
||||||
|
);
|
||||||
|
let auth_header = format!("Bearer {SEARCH_KEY}");
|
||||||
|
let headers = vec![
|
||||||
|
("accept".to_string(), "application/json".to_string()),
|
||||||
|
("authorization".to_string(), auth_header),
|
||||||
|
(
|
||||||
|
"user-agent".to_string(),
|
||||||
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36".to_string(),
|
||||||
|
),
|
||||||
|
];
|
||||||
|
|
||||||
|
let text = requester
|
||||||
|
.get_with_headers(&search_url, headers, Some(Version::HTTP_11))
|
||||||
|
.await
|
||||||
|
.map_err(|error| Error::from(format!("search request failed: {error}")))?;
|
||||||
|
|
||||||
|
let parsed: SearchResponse = serde_json::from_str(&text)?;
|
||||||
|
Ok(parsed
|
||||||
|
.hits
|
||||||
|
.into_iter()
|
||||||
|
.filter_map(|hit| {
|
||||||
|
let slug = hit.slug.trim();
|
||||||
|
if hit.id.is_empty() || slug.is_empty() || hit.title.trim().is_empty() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
Some(StubVideo {
|
||||||
|
id: hit.id,
|
||||||
|
title: Self::normalize_text(&hit.title),
|
||||||
|
url: format!("{}/video/{}", self.url, slug),
|
||||||
|
thumb: hit.image,
|
||||||
|
duration: 0,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn extract_detail_metadata(
|
||||||
|
&self,
|
||||||
|
html: &str,
|
||||||
|
) -> (
|
||||||
|
Vec<String>,
|
||||||
|
Option<String>,
|
||||||
|
Option<String>,
|
||||||
|
Option<String>,
|
||||||
|
Option<u64>,
|
||||||
|
) {
|
||||||
|
let uploader_name = Self::regex(r#"<a href="/user/([^"]+)"[^>]*>[^<]*<img[^>]*>\s*<p[^>]*>([^<]+)</p>"#)
|
||||||
|
.ok()
|
||||||
|
.and_then(|re| re.captures(html))
|
||||||
|
.and_then(|caps| {
|
||||||
|
let slug = caps.get(1)?.as_str().to_string();
|
||||||
|
let name = Self::normalize_text(caps.get(2)?.as_str());
|
||||||
|
if name.is_empty() { return None; }
|
||||||
|
Some((name, slug))
|
||||||
|
});
|
||||||
|
|
||||||
|
let uploader = uploader_name.as_ref().map(|v| v.0.clone());
|
||||||
|
let uploader_url = uploader_name
|
||||||
|
.as_ref()
|
||||||
|
.map(|v| format!("{}/user/{}", self.url, v.1));
|
||||||
|
let uploader_id = uploader_name
|
||||||
|
.as_ref()
|
||||||
|
.map(|v| format!("{CHANNEL_ID}:{}", v.1));
|
||||||
|
|
||||||
|
let uploaded_at = Self::regex(r#"<p class="s-1he9h8j">([A-Za-z]{3}\s+\d{1,2},\s+\d{4})</p>"#)
|
||||||
|
.ok()
|
||||||
|
.and_then(|re| re.captures(html))
|
||||||
|
.and_then(|caps| caps.get(1).map(|m| m.as_str().to_string()))
|
||||||
|
.and_then(|value| NaiveDate::parse_from_str(&value, "%b %e, %Y").ok())
|
||||||
|
.and_then(|date| date.and_hms_opt(0, 0, 0))
|
||||||
|
.and_then(|dt| u64::try_from(dt.and_utc().timestamp()).ok());
|
||||||
|
|
||||||
|
let tag_re = Self::regex(r#"<a href="/tag/[^"]+"[^>]*>([^<]+)</a>"#).ok();
|
||||||
|
let tags = tag_re
|
||||||
|
.map(|re| {
|
||||||
|
re.captures_iter(html)
|
||||||
|
.filter_map(|caps| caps.get(1).map(|m| Self::normalize_text(m.as_str())))
|
||||||
|
.filter(|t| !t.is_empty())
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
})
|
||||||
|
.unwrap_or_default();
|
||||||
|
|
||||||
|
(tags, uploader, uploader_url, uploader_id, uploaded_at)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn enrich_video(&self, options: &ServerOptions, stub: StubVideo) -> VideoItem {
|
||||||
|
let proxy_url = build_proxy_url(&options, CHANNEL_ID, &strip_url_scheme(&stub.url));
|
||||||
|
let mut item = VideoItem::new(
|
||||||
|
stub.id,
|
||||||
|
stub.title,
|
||||||
|
proxy_url,
|
||||||
|
CHANNEL_ID.to_string(),
|
||||||
|
stub.thumb,
|
||||||
|
stub.duration,
|
||||||
|
);
|
||||||
|
|
||||||
|
if let Ok(detail_html) = self.fetch_html(options, &stub.url).await {
|
||||||
|
let (tags, uploader, uploader_url, uploader_id, uploaded_at) =
|
||||||
|
self.extract_detail_metadata(&detail_html);
|
||||||
|
|
||||||
|
if !tags.is_empty() {
|
||||||
|
item.tags = Some(tags);
|
||||||
|
}
|
||||||
|
if let Some(value) = uploader {
|
||||||
|
item = item.uploader(value);
|
||||||
|
}
|
||||||
|
if let Some(value) = uploader_url {
|
||||||
|
item = item.uploader_url(value);
|
||||||
|
}
|
||||||
|
if let Some(value) = uploader_id {
|
||||||
|
item.uploaderId = Some(value);
|
||||||
|
}
|
||||||
|
if let Some(value) = uploaded_at {
|
||||||
|
item.uploadedAt = Some(value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
item
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl Provider for ClapdatProvider {
|
||||||
|
async fn get_videos(
|
||||||
|
&self,
|
||||||
|
_cache: crate::util::cache::VideoCache,
|
||||||
|
_pool: DbPool,
|
||||||
|
sort: String,
|
||||||
|
query: Option<String>,
|
||||||
|
page: String,
|
||||||
|
per_page: String,
|
||||||
|
options: ServerOptions,
|
||||||
|
) -> Vec<VideoItem> {
|
||||||
|
let page_num = page.parse::<u16>().unwrap_or(1).max(1);
|
||||||
|
let per_page_num = per_page.parse::<usize>().unwrap_or(20).clamp(1, 60);
|
||||||
|
let sort_value = if sort.trim().is_empty() {
|
||||||
|
options.sort.as_deref().unwrap_or("trending").to_string()
|
||||||
|
} else {
|
||||||
|
sort
|
||||||
|
};
|
||||||
|
let query_value = query.unwrap_or_default();
|
||||||
|
let target = self.resolve_target(&query_value, &sort_value);
|
||||||
|
|
||||||
|
let stubs = match &target {
|
||||||
|
Target::Search { query } => match self.search_videos(&options, query, page_num, per_page_num).await {
|
||||||
|
Ok(items) => items,
|
||||||
|
Err(error) => {
|
||||||
|
report_provider_error(CHANNEL_ID, "search_videos", &error.to_string()).await;
|
||||||
|
vec![]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
_ => {
|
||||||
|
let Some(url) = self.listing_url(&target, page_num) else {
|
||||||
|
return vec![];
|
||||||
|
};
|
||||||
|
match self.fetch_html(&options, &url).await {
|
||||||
|
Ok(html) => {
|
||||||
|
let parsed = match (&target, page_num) {
|
||||||
|
(Target::Trending, 1) => {
|
||||||
|
self.parse_home_section_html(&html, "trending-videos")
|
||||||
|
}
|
||||||
|
(Target::Recent, 1) => {
|
||||||
|
self.parse_home_section_html(&html, "recent-videos")
|
||||||
|
}
|
||||||
|
_ => self.parse_listing_html(&html),
|
||||||
|
};
|
||||||
|
match parsed {
|
||||||
|
Ok(items) => items,
|
||||||
|
Err(error) => {
|
||||||
|
report_provider_error(CHANNEL_ID, "parse_listing_html", &error.to_string()).await;
|
||||||
|
vec![]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(error) => {
|
||||||
|
report_provider_error(CHANNEL_ID, "fetch_html", &error.to_string()).await;
|
||||||
|
vec![]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut output = Vec::with_capacity(stubs.len());
|
||||||
|
for stub in stubs.into_iter().take(per_page_num) {
|
||||||
|
output.push(self.enrich_video(&options, stub).await);
|
||||||
|
}
|
||||||
|
output
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_channel(&self, clientversion: ClientVersion) -> Option<Channel> {
|
||||||
|
Some(self.build_channel(clientversion))
|
||||||
|
}
|
||||||
|
}
|
||||||
113
src/proxies/clapdat.rs
Normal file
113
src/proxies/clapdat.rs
Normal file
@@ -0,0 +1,113 @@
|
|||||||
|
use ntex::web;
|
||||||
|
use regex::Regex;
|
||||||
|
|
||||||
|
use crate::util::requester::Requester;
|
||||||
|
|
||||||
|
const BASE_URL: &str = "https://www.clapdat.com";
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct ClapdatProxy {}
|
||||||
|
|
||||||
|
impl ClapdatProxy {
|
||||||
|
pub fn new() -> Self {
|
||||||
|
Self {}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn normalize_detail_url(endpoint: &str) -> Option<String> {
|
||||||
|
let value = endpoint.trim().trim_start_matches('/');
|
||||||
|
if value.is_empty() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
let detail_url = if value.starts_with("http://") || value.starts_with("https://") {
|
||||||
|
value.to_string()
|
||||||
|
} else {
|
||||||
|
format!("https://{}", value)
|
||||||
|
};
|
||||||
|
|
||||||
|
let detail_url = detail_url.replacen("http://", "https://", 1);
|
||||||
|
let parsed = url::Url::parse(&detail_url).ok()?;
|
||||||
|
let host = parsed.host_str()?;
|
||||||
|
if !(host == "www.clapdat.com" || host == "clapdat.com") {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
if !parsed.path().starts_with("/video/") {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
Some(detail_url)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn clapdat_decode(input: &str) -> Option<Vec<u8>> {
|
||||||
|
let compact = if input.len() > 209 {
|
||||||
|
format!("{}{}", &input[..19], &input[209..])
|
||||||
|
} else {
|
||||||
|
input.to_string()
|
||||||
|
};
|
||||||
|
|
||||||
|
let cleaned: String = compact
|
||||||
|
.chars()
|
||||||
|
.filter(|c| c.is_ascii_alphanumeric() || *c == '+' || *c == '/')
|
||||||
|
.collect();
|
||||||
|
if cleaned.is_empty() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut padded = cleaned;
|
||||||
|
while padded.len() % 4 != 0 {
|
||||||
|
padded.push('=');
|
||||||
|
}
|
||||||
|
|
||||||
|
base64::Engine::decode(&base64::engine::general_purpose::STANDARD, padded.as_bytes()).ok()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn extract_media_url(html: &str) -> Option<String> {
|
||||||
|
let domain_re = Regex::new(r#"file_domain:"([^"]+)""#).ok()?;
|
||||||
|
let file_re = Regex::new(r#"file:"([^"]+)""#).ok()?;
|
||||||
|
let domain = domain_re
|
||||||
|
.captures(html)
|
||||||
|
.and_then(|caps| caps.get(1).map(|m| m.as_str().trim().to_string()))?;
|
||||||
|
let encoded = file_re
|
||||||
|
.captures(html)
|
||||||
|
.and_then(|caps| caps.get(1).map(|m| m.as_str().trim().to_string()))?;
|
||||||
|
|
||||||
|
let decoded = Self::clapdat_decode(&encoded)?;
|
||||||
|
let path: String = decoded.into_iter().map(char::from).collect();
|
||||||
|
if path.is_empty() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
Some(format!("https://{}/{}", domain, path.trim_start_matches('/')))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl crate::proxies::Proxy for ClapdatProxy {
|
||||||
|
async fn get_video_url(&self, url: String, requester: web::types::State<Requester>) -> String {
|
||||||
|
let Some(detail_url) = Self::normalize_detail_url(&url) else {
|
||||||
|
return String::new();
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut requester = requester.get_ref().clone();
|
||||||
|
let headers = vec![
|
||||||
|
(
|
||||||
|
"accept".to_string(),
|
||||||
|
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8".to_string(),
|
||||||
|
),
|
||||||
|
("accept-language".to_string(), "en-US,en;q=0.8".to_string()),
|
||||||
|
(
|
||||||
|
"user-agent".to_string(),
|
||||||
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36".to_string(),
|
||||||
|
),
|
||||||
|
("referer".to_string(), BASE_URL.to_string()),
|
||||||
|
];
|
||||||
|
|
||||||
|
let html = requester
|
||||||
|
.get_with_headers(&detail_url, headers, Some(wreq::Version::HTTP_11))
|
||||||
|
.await
|
||||||
|
.unwrap_or_default();
|
||||||
|
if html.is_empty() {
|
||||||
|
return String::new();
|
||||||
|
}
|
||||||
|
|
||||||
|
Self::extract_media_url(&html).unwrap_or_default()
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -61,23 +61,29 @@ impl LulustreamProxy {
|
|||||||
) -> String {
|
) -> String {
|
||||||
let mut requester = requester.get_ref().clone();
|
let mut requester = requester.get_ref().clone();
|
||||||
let Some((detail_url, video_id)) = Self::normalize_detail_request(&url) else {
|
let Some((detail_url, video_id)) = Self::normalize_detail_request(&url) else {
|
||||||
println!("LulustreamProxy: Invalid detail URL: {url}");
|
|
||||||
return String::new();
|
return String::new();
|
||||||
};
|
};
|
||||||
let mut text = requester.get(&detail_url, None).await.unwrap_or_default();
|
println!("LulustreamProxy: Normalized detail URL: {:?}", format!("https://luluvid.com/e/{video_id}"));
|
||||||
|
let mut text = requester.get(format!("https://luluvid.com/e/{video_id}").as_str(), None).await.unwrap_or_default();
|
||||||
if !text.contains("[{file:\"") {
|
if !text.contains("[{file:\"") {
|
||||||
let packedtext = text.split("<script type='text/javascript'>").nth(1).and_then(|t| t.split("</script>").next()).unwrap_or_default();
|
let packedtext = text.split("<script type='text/javascript'>").nth(1).and_then(|t| t.split("</script>").next()).unwrap_or_default();
|
||||||
println!("LulustreamProxy: Found packed text: {packedtext}");
|
|
||||||
text = dean_edwards::unpack(&packedtext).unwrap_or_default();
|
text = dean_edwards::unpack(&packedtext).unwrap_or_default();
|
||||||
println!("LulustreamProxy: Unpacked text: {text}");
|
|
||||||
}
|
}
|
||||||
let video_url = text.split("[{file:\"")
|
let video_url = text.split("[{file:\"")
|
||||||
.nth(1)
|
.nth(1)
|
||||||
.and_then(|s| s.split('"').next())
|
.and_then(|s| s.split('"').next())
|
||||||
.unwrap_or_default()
|
.unwrap_or_default()
|
||||||
.to_string();
|
.to_string();
|
||||||
println!("LulustreamProxy: Extracted video URL: {video_url}");
|
println!("LulustreamProxy: Extracted video URL: {}", video_url);
|
||||||
|
let test_request = requester.get_raw_with_headers(video_url.as_str(), vec![
|
||||||
|
("Accept-Language".to_string(), "en-US,en;q=0.9".to_string()),
|
||||||
|
("Referer".to_string(), detail_url.clone()),
|
||||||
|
("User-Agent".to_string(), "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36".to_string())
|
||||||
|
]).await.unwrap();
|
||||||
|
println!("LulustreamProxy: Test request status: {}", test_request.status());
|
||||||
|
|
||||||
video_url
|
video_url
|
||||||
|
// return "https://cdn1004.cdn-tnmr.org/hls2/01/03256/cssckmym0ibf_h/master.m3u8?t=Y2jXSIPERwSec0L6RSAOIPFAW53dQ0UgslngqGnF0go&s=1778507711&e=28800&f=16283923&i=0.3&sp=0".to_string();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
use crate::proxies::archivebate::ArchivebateProxy;
|
use crate::proxies::archivebate::ArchivebateProxy;
|
||||||
|
use crate::proxies::clapdat::ClapdatProxy;
|
||||||
use crate::proxies::doodstream::DoodstreamProxy;
|
use crate::proxies::doodstream::DoodstreamProxy;
|
||||||
use crate::proxies::heavyfetish::HeavyfetishProxy;
|
use crate::proxies::heavyfetish::HeavyfetishProxy;
|
||||||
use crate::proxies::hqporner::HqpornerProxy;
|
use crate::proxies::hqporner::HqpornerProxy;
|
||||||
@@ -15,6 +16,7 @@ use crate::proxies::vidara::VidaraProxy;
|
|||||||
use crate::proxies::lulustream::LulustreamProxy;
|
use crate::proxies::lulustream::LulustreamProxy;
|
||||||
|
|
||||||
pub mod archivebate;
|
pub mod archivebate;
|
||||||
|
pub mod clapdat;
|
||||||
pub mod doodstream;
|
pub mod doodstream;
|
||||||
pub mod hanimecdn;
|
pub mod hanimecdn;
|
||||||
pub mod heavyfetish;
|
pub mod heavyfetish;
|
||||||
@@ -50,6 +52,7 @@ pub enum AnyProxy {
|
|||||||
Heavyfetish(HeavyfetishProxy),
|
Heavyfetish(HeavyfetishProxy),
|
||||||
Vjav(VjavProxy),
|
Vjav(VjavProxy),
|
||||||
Vidara(VidaraProxy),
|
Vidara(VidaraProxy),
|
||||||
|
Clapdat(ClapdatProxy),
|
||||||
}
|
}
|
||||||
|
|
||||||
pub trait Proxy {
|
pub trait Proxy {
|
||||||
@@ -73,6 +76,7 @@ impl Proxy for AnyProxy {
|
|||||||
AnyProxy::Heavyfetish(p) => p.get_video_url(url, requester).await,
|
AnyProxy::Heavyfetish(p) => p.get_video_url(url, requester).await,
|
||||||
AnyProxy::Vjav(p) => p.get_video_url(url, requester).await,
|
AnyProxy::Vjav(p) => p.get_video_url(url, requester).await,
|
||||||
AnyProxy::Vidara(p) => p.get_video_url(url, requester).await,
|
AnyProxy::Vidara(p) => p.get_video_url(url, requester).await,
|
||||||
|
AnyProxy::Clapdat(p) => p.get_video_url(url, requester).await,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
use ntex::web::{self, HttpRequest};
|
use ntex::web::{self, HttpRequest};
|
||||||
|
|
||||||
use crate::proxies::archivebate::ArchivebateProxy;
|
use crate::proxies::archivebate::ArchivebateProxy;
|
||||||
|
use crate::proxies::clapdat::ClapdatProxy;
|
||||||
use crate::proxies::doodstream::DoodstreamProxy;
|
use crate::proxies::doodstream::DoodstreamProxy;
|
||||||
use crate::proxies::heavyfetish::HeavyfetishProxy;
|
use crate::proxies::heavyfetish::HeavyfetishProxy;
|
||||||
use crate::proxies::hqporner::HqpornerProxy;
|
use crate::proxies::hqporner::HqpornerProxy;
|
||||||
@@ -23,6 +24,11 @@ pub fn config(cfg: &mut web::ServiceConfig) {
|
|||||||
.route(web::post().to(proxy2redirect))
|
.route(web::post().to(proxy2redirect))
|
||||||
.route(web::get().to(proxy2redirect)),
|
.route(web::get().to(proxy2redirect)),
|
||||||
)
|
)
|
||||||
|
.service(
|
||||||
|
web::resource("/clapdat/{endpoint}*")
|
||||||
|
.route(web::post().to(proxy2redirect))
|
||||||
|
.route(web::get().to(proxy2redirect)),
|
||||||
|
)
|
||||||
.service(
|
.service(
|
||||||
web::resource("/doodstream/{endpoint}*")
|
web::resource("/doodstream/{endpoint}*")
|
||||||
.route(web::post().to(proxy2redirect))
|
.route(web::post().to(proxy2redirect))
|
||||||
@@ -143,6 +149,7 @@ async fn proxy2redirect(
|
|||||||
fn get_proxy(proxy: &str) -> Option<AnyProxy> {
|
fn get_proxy(proxy: &str) -> Option<AnyProxy> {
|
||||||
match proxy {
|
match proxy {
|
||||||
"archivebate" => Some(AnyProxy::Archivebate(ArchivebateProxy::new())),
|
"archivebate" => Some(AnyProxy::Archivebate(ArchivebateProxy::new())),
|
||||||
|
"clapdat" => Some(AnyProxy::Clapdat(ClapdatProxy::new())),
|
||||||
"doodstream" => Some(AnyProxy::Doodstream(DoodstreamProxy::new())),
|
"doodstream" => Some(AnyProxy::Doodstream(DoodstreamProxy::new())),
|
||||||
"sxyprn" => Some(AnyProxy::Sxyprn(SxyprnProxy::new())),
|
"sxyprn" => Some(AnyProxy::Sxyprn(SxyprnProxy::new())),
|
||||||
"javtiful" => Some(AnyProxy::Javtiful(JavtifulProxy::new())),
|
"javtiful" => Some(AnyProxy::Javtiful(JavtifulProxy::new())),
|
||||||
|
|||||||
Reference in New Issue
Block a user