1147 lines
40 KiB
Rust
1147 lines
40 KiB
Rust
use crate::DbPool;
|
|
use crate::api::ClientVersion;
|
|
use crate::providers::{
|
|
Provider, report_provider_error, report_provider_error_background, requester_or_default,
|
|
};
|
|
use crate::status::*;
|
|
use crate::util::cache::VideoCache;
|
|
use crate::util::requester::Requester;
|
|
use crate::videos::{ServerOptions, VideoFormat, VideoItem};
|
|
|
|
use async_trait::async_trait;
|
|
use error_chain::error_chain;
|
|
use futures::stream::{self, StreamExt};
|
|
use htmlentity::entity::{ICodedDataTrait, decode};
|
|
use percent_encoding::{NON_ALPHANUMERIC, utf8_percent_encode};
|
|
use scraper::{ElementRef, Html, Selector};
|
|
use std::sync::{Arc, RwLock};
|
|
use std::thread;
|
|
use std::time::Duration as StdDuration;
|
|
use tokio::time::timeout;
|
|
|
|
pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata =
|
|
crate::providers::ProviderChannelMetadata {
|
|
group_id: "onlyfans",
|
|
tags: &["onlyfans", "leaks", "sextape"],
|
|
};
|
|
|
|
const CHANNEL_ID: &str = "borntobefuck";
|
|
|
|
// The apex domain 301-redirects to a geo-picked locale subdomain and drops the
|
|
// request path while doing so (`/watch/40297` -> `https://eu.borntobefuck.com/`).
|
|
// `uk.` is the English locale and answers every path directly with 200, so it is
|
|
// the default. The canonical link on the page still points at the apex, but only
|
|
// a locale subdomain is actually fetchable.
|
|
const BASE_URL: &str = "https://uk.borntobefuck.com";
|
|
|
|
/// Locale subdomains that were probed directly (curl, no redirect) and returned
|
|
/// 200 for `/`, `/videos`, `/search/results`, `/tags/*`, `/categories/*`, and
|
|
/// `/@*`. Every other two-letter guess (`en`, `us`, `asia`, ...) failed to
|
|
/// resolve at all. Exposed to users as a "Region" filter.
|
|
const REGIONS: &[(&str, &str)] = &[
|
|
("uk", "English"),
|
|
("fr", "French"),
|
|
("de", "German"),
|
|
("es", "Spanish"),
|
|
("it", "Italian"),
|
|
("pt", "Portuguese"),
|
|
];
|
|
const DEFAULT_REGION: &str = "uk";
|
|
|
|
// Media and thumbnails live on the site's own CDN and are hotlinkable without a
|
|
// referer or token, so no `/proxy/...` route is required.
|
|
const CDN_PREFIX: &str = "https://content.zavastream.com/";
|
|
const THUMB_SUFFIX: &str = "/thumbnail.jpg";
|
|
|
|
const FIREFOX_UA: &str = "Mozilla/5.0 (X11; Linux x86_64; rv:120.0) Gecko/20100101 Firefox/120.0";
|
|
const HTML_ACCEPT: &str =
|
|
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8";
|
|
|
|
const LIST_CACHE_SECS: u64 = 300;
|
|
const ENRICH_CONCURRENCY: usize = 6;
|
|
const ENRICH_BUDGET_SECS: u64 = 8;
|
|
|
|
error_chain! {
|
|
foreign_links {
|
|
Io(std::io::Error);
|
|
}
|
|
errors {
|
|
Parse(msg: String) {
|
|
description("parse error")
|
|
display("parse error: {}", msg)
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Sort ids are the literal `data-sort` values the site's own filter buttons post
|
|
/// back as `?sort=`. Order matches the on-site button order.
|
|
const SORTS: &[(&str, &str)] = &[
|
|
("latest", "Latest"),
|
|
("trending", "Trending"),
|
|
("topModels", "Top Models"),
|
|
("mostWatched", "Most Watched"),
|
|
("mostLiked", "Most Liked"),
|
|
("oldest", "Oldest"),
|
|
];
|
|
|
|
/// Scraped from https://uk.borntobefuck.com/categories (46 entries). The site
|
|
/// mixes fetish categories and nationality archives in the same namespace.
|
|
const CATEGORIES: &[&str] = &[
|
|
"american",
|
|
"anal",
|
|
"arab",
|
|
"asian",
|
|
"australia",
|
|
"bbc",
|
|
"bbw",
|
|
"big-ass",
|
|
"big-tits",
|
|
"black",
|
|
"blonde",
|
|
"blowjob",
|
|
"brazil",
|
|
"brunette",
|
|
"canada",
|
|
"colorful",
|
|
"columbia",
|
|
"cosplay",
|
|
"couple",
|
|
"ebony",
|
|
"france",
|
|
"gangbang",
|
|
"germany",
|
|
"hardcore",
|
|
"india",
|
|
"interracial",
|
|
"italy",
|
|
"japan",
|
|
"lesbians",
|
|
"masturbation",
|
|
"mexico",
|
|
"panama",
|
|
"redhead",
|
|
"rough",
|
|
"russia",
|
|
"sextape",
|
|
"spain",
|
|
"sudan",
|
|
"switzerland",
|
|
"teen",
|
|
"threesome",
|
|
"uk",
|
|
"vaginal",
|
|
"venezuela",
|
|
"white",
|
|
"milf",
|
|
];
|
|
|
|
/// Slug fragments the generic title-caser gets wrong.
|
|
const ACRONYMS: &[(&str, &str)] = &[
|
|
("bbc", "BBC"),
|
|
("bbw", "BBW"),
|
|
("uk", "UK"),
|
|
("milf", "MILF"),
|
|
("pov", "POV"),
|
|
("bwc", "BWC"),
|
|
("ai", "AI"),
|
|
];
|
|
|
|
#[derive(Debug, Clone, PartialEq)]
|
|
enum Target {
|
|
/// `/?sort=<id>` on the selected region's host — the home feed, which is
|
|
/// the only route exposing the site's six sort modes.
|
|
Feed { sort: String, region: String },
|
|
/// Region-aware: each locale accepts free-text search in its own language.
|
|
Search { query: String, region: String },
|
|
/// Region-aware: uploader slugs are shared across every locale.
|
|
Uploader { slug: String, region: String },
|
|
/// Always resolved against the English (`uk`) host: tag/category slugs are
|
|
/// localized per region and only the English catalog is loaded/known here.
|
|
Tag(String),
|
|
Category(String),
|
|
}
|
|
|
|
#[derive(Debug, Clone)]
|
|
pub struct BorntobefuckProvider {
|
|
tags: Arc<RwLock<Vec<FilterOption>>>,
|
|
}
|
|
|
|
impl BorntobefuckProvider {
|
|
pub fn new() -> Self {
|
|
let provider = Self {
|
|
tags: Arc::new(RwLock::new(vec![FilterOption {
|
|
id: "all".to_string(),
|
|
title: "All".to_string(),
|
|
}])),
|
|
};
|
|
provider.spawn_initial_load();
|
|
provider
|
|
}
|
|
|
|
/// The site has 412 tags on a single `/tags` page. Load them off the request
|
|
/// path so startup is not blocked; until they arrive, unknown queries simply
|
|
/// fall through to site search.
|
|
fn spawn_initial_load(&self) {
|
|
let tags = Arc::clone(&self.tags);
|
|
thread::spawn(move || {
|
|
let runtime = match tokio::runtime::Builder::new_current_thread()
|
|
.enable_all()
|
|
.build()
|
|
{
|
|
Ok(runtime) => runtime,
|
|
Err(error) => {
|
|
report_provider_error_background(
|
|
CHANNEL_ID,
|
|
"spawn_initial_load.runtime_build",
|
|
&error.to_string(),
|
|
);
|
|
return;
|
|
}
|
|
};
|
|
runtime.block_on(async move {
|
|
if let Err(error) = Self::load_tags(Arc::clone(&tags)).await {
|
|
report_provider_error_background(
|
|
CHANNEL_ID,
|
|
"load_tags",
|
|
&error.to_string(),
|
|
);
|
|
}
|
|
});
|
|
});
|
|
}
|
|
|
|
async fn load_tags(store: Arc<RwLock<Vec<FilterOption>>>) -> Result<()> {
|
|
let url = format!("{BASE_URL}/tags");
|
|
// Runs at startup, before any request exists to borrow a requester from.
|
|
let mut requester = Requester::new();
|
|
let html = requester
|
|
.get_with_headers(&url, Self::html_headers(BASE_URL), None)
|
|
.await
|
|
.map_err(|e| Error::from(format!("request failed for {url}: {e}")))?;
|
|
|
|
let mut options = vec![FilterOption {
|
|
id: "all".to_string(),
|
|
title: "All".to_string(),
|
|
}];
|
|
{
|
|
let document = Html::parse_document(&html);
|
|
let link_sel = Self::selector("a.active-video-tag[href]")?;
|
|
for link in document.select(&link_sel) {
|
|
let href = link.value().attr("href").unwrap_or_default();
|
|
let Some(slug) = Self::slug_after(href, "/tags/") else {
|
|
continue;
|
|
};
|
|
let title = Self::decode_html(&Self::text_of(&link));
|
|
let title = title.trim_start_matches('#').trim().to_string();
|
|
if title.is_empty() || options.iter().any(|o| o.id == slug) {
|
|
continue;
|
|
}
|
|
options.push(FilterOption { id: slug, title });
|
|
}
|
|
}
|
|
|
|
if options.len() <= 1 {
|
|
return Err(Error::from(format!("no tags parsed from {url}")));
|
|
}
|
|
if let Ok(mut guard) = store.write() {
|
|
*guard = options;
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn build_channel(&self, _clientversion: ClientVersion) -> Channel {
|
|
let tags = self
|
|
.tags
|
|
.read()
|
|
.map(|value| value.clone())
|
|
.unwrap_or_default();
|
|
|
|
let mut category_options = vec![FilterOption {
|
|
id: "all".to_string(),
|
|
title: "All".to_string(),
|
|
}];
|
|
for slug in CATEGORIES {
|
|
category_options.push(FilterOption {
|
|
id: (*slug).to_string(),
|
|
title: Self::titleize(slug),
|
|
});
|
|
}
|
|
|
|
Channel {
|
|
id: CHANNEL_ID.to_string(),
|
|
name: "BornToBeFuck".to_string(),
|
|
description:
|
|
"BornToBeFuck — OnlyFans leaks and sextapes with model channels, categories, and tag archives."
|
|
.to_string(),
|
|
premium: false,
|
|
favicon: "https://www.google.com/s2/favicons?sz=64&domain=borntobefuck.com".to_string(),
|
|
status: "active".to_string(),
|
|
categories: CATEGORIES.iter().map(|slug| Self::titleize(slug)).collect(),
|
|
options: vec![
|
|
ChannelOption {
|
|
id: "sort".to_string(),
|
|
title: "Sort".to_string(),
|
|
description: "Order the BornToBeFuck home feed.".to_string(),
|
|
systemImage: "arrow.up.arrow.down".to_string(),
|
|
colorName: "blue".to_string(),
|
|
options: SORTS
|
|
.iter()
|
|
.map(|(id, title)| FilterOption {
|
|
id: (*id).to_string(),
|
|
title: (*title).to_string(),
|
|
})
|
|
.collect(),
|
|
multiSelect: false,
|
|
},
|
|
ChannelOption {
|
|
id: "category".to_string(),
|
|
title: "Category".to_string(),
|
|
description: "Browse a BornToBeFuck category archive.".to_string(),
|
|
systemImage: "square.grid.2x2".to_string(),
|
|
colorName: "orange".to_string(),
|
|
options: category_options,
|
|
multiSelect: false,
|
|
},
|
|
ChannelOption {
|
|
id: "filter".to_string(),
|
|
title: "Tags".to_string(),
|
|
description: "Browse a BornToBeFuck tag archive.".to_string(),
|
|
systemImage: "number".to_string(),
|
|
colorName: "purple".to_string(),
|
|
options: tags,
|
|
multiSelect: false,
|
|
},
|
|
ChannelOption {
|
|
id: "language".to_string(),
|
|
title: "Region".to_string(),
|
|
description:
|
|
"Pick a BornToBeFuck locale. Feed, search, and uploader pages are \
|
|
localized per region; tags and categories always use the English \
|
|
(UK) catalog."
|
|
.to_string(),
|
|
systemImage: "globe".to_string(),
|
|
colorName: "green".to_string(),
|
|
options: REGIONS
|
|
.iter()
|
|
.map(|(id, title)| FilterOption {
|
|
id: (*id).to_string(),
|
|
title: (*title).to_string(),
|
|
})
|
|
.collect(),
|
|
multiSelect: false,
|
|
},
|
|
],
|
|
nsfw: true,
|
|
cacheDuration: Some(1800),
|
|
ytdlpCommand: None,
|
|
}
|
|
}
|
|
|
|
fn selector(value: &str) -> Result<Selector> {
|
|
Selector::parse(value)
|
|
.map_err(|e| Error::from(format!("selector `{value}` parse failed: {e}")))
|
|
}
|
|
|
|
fn decode_html(text: &str) -> String {
|
|
decode(text.as_bytes())
|
|
.to_string()
|
|
.unwrap_or_else(|_| text.to_string())
|
|
}
|
|
|
|
fn text_of(el: &ElementRef<'_>) -> String {
|
|
el.text()
|
|
.collect::<Vec<_>>()
|
|
.join(" ")
|
|
.split_whitespace()
|
|
.collect::<Vec<_>>()
|
|
.join(" ")
|
|
}
|
|
|
|
fn html_headers(referer: &str) -> Vec<(String, String)> {
|
|
vec![
|
|
("User-Agent".to_string(), FIREFOX_UA.to_string()),
|
|
("Accept".to_string(), HTML_ACCEPT.to_string()),
|
|
("Referer".to_string(), referer.to_string()),
|
|
]
|
|
}
|
|
|
|
/// Returns the path segment following `marker`, stripped of any query string.
|
|
fn slug_after(href: &str, marker: &str) -> Option<String> {
|
|
let rest = href.split(marker).nth(1)?;
|
|
let slug = rest
|
|
.split(['?', '#', '/'])
|
|
.next()
|
|
.unwrap_or_default()
|
|
.trim()
|
|
.to_ascii_lowercase();
|
|
if slug.is_empty() { None } else { Some(slug) }
|
|
}
|
|
|
|
fn titleize(slug: &str) -> String {
|
|
slug.split('-')
|
|
.map(|word| {
|
|
if let Some((_, label)) = ACRONYMS.iter().find(|(k, _)| *k == word) {
|
|
return (*label).to_string();
|
|
}
|
|
let mut chars = word.chars();
|
|
match chars.next() {
|
|
Some(first) => first.to_uppercase().collect::<String>() + chars.as_str(),
|
|
None => String::new(),
|
|
}
|
|
})
|
|
.collect::<Vec<_>>()
|
|
.join(" ")
|
|
}
|
|
|
|
fn normalize_key(value: &str) -> String {
|
|
value
|
|
.trim()
|
|
.trim_start_matches(['#', '@'])
|
|
.replace(['-', '_'], " ")
|
|
.split_whitespace()
|
|
.collect::<Vec<_>>()
|
|
.join(" ")
|
|
.to_ascii_lowercase()
|
|
}
|
|
|
|
fn slugify(value: &str) -> String {
|
|
Self::normalize_key(value).replace(' ', "-")
|
|
}
|
|
|
|
/// Region subdomain -> full origin. Falls back to the default (uk) region
|
|
/// for anything not in `REGIONS` — callers are expected to have already
|
|
/// validated the value, but this keeps `target_url` infallible.
|
|
fn region_base(region: &str) -> String {
|
|
let region = REGIONS
|
|
.iter()
|
|
.find(|(id, _)| *id == region)
|
|
.map(|(id, _)| *id)
|
|
.unwrap_or(DEFAULT_REGION);
|
|
format!("https://{region}.borntobefuck.com")
|
|
}
|
|
|
|
/// Every listing route on the site is `?page=N` based; page 1 omits the
|
|
/// parameter. Only `/` carries the `sort` parameter — archives ignore it.
|
|
/// `Tag`/`Category` are always resolved against the English (uk) host:
|
|
/// their slug catalogs are fully localized per region and only the
|
|
/// English catalog is scraped/known here, so cross-region slugs would
|
|
/// silently 404.
|
|
fn target_url(target: &Target, page: u16) -> String {
|
|
let (base, has_query) = match target {
|
|
Target::Feed { sort, region } => {
|
|
(format!("{}/?sort={sort}", Self::region_base(region)), true)
|
|
}
|
|
Target::Search { query, region } => {
|
|
let encoded = utf8_percent_encode(query.trim(), NON_ALPHANUMERIC).to_string();
|
|
(
|
|
format!(
|
|
"{}/search/results?search={encoded}",
|
|
Self::region_base(region)
|
|
),
|
|
true,
|
|
)
|
|
}
|
|
Target::Tag(slug) => (format!("{BASE_URL}/tags/{slug}"), false),
|
|
Target::Category(slug) => (format!("{BASE_URL}/categories/{slug}"), false),
|
|
Target::Uploader { slug, region } => {
|
|
(format!("{}/@{slug}", Self::region_base(region)), false)
|
|
}
|
|
};
|
|
if page <= 1 {
|
|
base
|
|
} else if has_query {
|
|
format!("{base}&page={page}")
|
|
} else {
|
|
format!("{base}?page={page}")
|
|
}
|
|
}
|
|
|
|
/// The listing card thumbnail and the HLS manifest are siblings in the same
|
|
/// CDN folder: `.../{account}/{uuid}/thumbnail.jpg` and
|
|
/// `.../{account}/{uuid}/index.m3u8`. The watch page only exposes the
|
|
/// manifest through a nested `/videos/{id}/player` iframe, so deriving it
|
|
/// from the card avoids one extra request per item.
|
|
fn hls_from_thumb(thumb: &str) -> Option<String> {
|
|
if !thumb.starts_with(CDN_PREFIX) || !thumb.ends_with(THUMB_SUFFIX) {
|
|
return None;
|
|
}
|
|
Some(format!(
|
|
"{}/index.m3u8",
|
|
&thumb[..thumb.len() - THUMB_SUFFIX.len()]
|
|
))
|
|
}
|
|
|
|
fn parse_list_page(html: &str) -> Result<Vec<VideoItem>> {
|
|
let document = Html::parse_document(html);
|
|
|
|
let card_sel = Self::selector("div.video")?;
|
|
let link_sel = Self::selector("a.cardVideo-top-link[href]")?;
|
|
let img_sel = Self::selector("img.thumbnail")?;
|
|
let duration_sel = Self::selector("div.time[data-duration]")?;
|
|
let title_sel = Self::selector("h3.video-title")?;
|
|
let uploader_link_sel = Self::selector("a.channel_avatar_video_card[href]")?;
|
|
let uploader_name_sel = Self::selector("p.video-channel")?;
|
|
let date_sel = Self::selector("p.date[data-published]")?;
|
|
let views_sel = Self::selector("p.views[data-views]")?;
|
|
let country_sel = Self::selector("a.card-country[href]")?;
|
|
|
|
let mut items: Vec<VideoItem> = Vec::new();
|
|
|
|
for card in document.select(&card_sel) {
|
|
let Some(link) = card.select(&link_sel).next() else {
|
|
continue;
|
|
};
|
|
let href = link.value().attr("href").unwrap_or_default();
|
|
let Some(id) = Self::slug_after(href, "/watch/") else {
|
|
continue;
|
|
};
|
|
if items.iter().any(|existing| existing.id == id) {
|
|
continue;
|
|
}
|
|
let page_url = format!("{BASE_URL}/watch/{id}");
|
|
|
|
let title = card
|
|
.select(&title_sel)
|
|
.next()
|
|
.map(|el| Self::decode_html(&Self::text_of(&el)))
|
|
.filter(|value| !value.is_empty());
|
|
let Some(title) = title else {
|
|
continue;
|
|
};
|
|
|
|
let thumb = card
|
|
.select(&img_sel)
|
|
.next()
|
|
.and_then(|el| el.value().attr("src"))
|
|
.unwrap_or_default()
|
|
.to_string();
|
|
|
|
let duration = card
|
|
.select(&duration_sel)
|
|
.next()
|
|
.and_then(|el| el.value().attr("data-duration"))
|
|
.and_then(|value| value.parse::<u32>().ok())
|
|
.unwrap_or(0);
|
|
|
|
let mut item = VideoItem::new(
|
|
id,
|
|
title,
|
|
page_url.clone(),
|
|
CHANNEL_ID.to_string(),
|
|
thumb.clone(),
|
|
duration,
|
|
);
|
|
|
|
item.views = card
|
|
.select(&views_sel)
|
|
.next()
|
|
.and_then(|el| el.value().attr("data-views"))
|
|
.and_then(|value| value.parse::<u32>().ok());
|
|
|
|
item.uploadedAt = card
|
|
.select(&date_sel)
|
|
.next()
|
|
.and_then(|el| el.value().attr("data-published"))
|
|
.and_then(|value| value.parse::<u64>().ok());
|
|
|
|
if let Some(uploader_link) = card.select(&uploader_link_sel).next() {
|
|
let href = uploader_link.value().attr("href").unwrap_or_default();
|
|
if let Some(slug) = Self::slug_after(href, "/@") {
|
|
item.uploaderUrl = Some(format!("{BASE_URL}/@{slug}"));
|
|
item.uploaderId = Some(format!("{CHANNEL_ID}:{slug}"));
|
|
}
|
|
}
|
|
item.uploader = card
|
|
.select(&uploader_name_sel)
|
|
.next()
|
|
.map(|el| Self::decode_html(&Self::text_of(&el)))
|
|
.filter(|value| !value.is_empty());
|
|
|
|
// The card only carries the nationality archive link; the full tag
|
|
// list lives on the watch page and is added during enrichment.
|
|
if let Some(slug) = card
|
|
.select(&country_sel)
|
|
.next()
|
|
.and_then(|el| el.value().attr("href"))
|
|
.and_then(|href| Self::slug_after(href, "/categories/"))
|
|
{
|
|
item.tags = Some(vec![Self::titleize(&slug)]);
|
|
}
|
|
|
|
if let Some(hls) = Self::hls_from_thumb(&thumb) {
|
|
item.formats = Some(vec![
|
|
VideoFormat::m3u8(hls, "master".to_string(), "hls".to_string())
|
|
.http_header("Referer".to_string(), page_url)
|
|
.http_header("User-Agent".to_string(), FIREFOX_UA.to_string()),
|
|
]);
|
|
}
|
|
|
|
items.push(item);
|
|
}
|
|
|
|
Ok(items)
|
|
}
|
|
|
|
fn parse_detail_tags(html: &str) -> Result<Vec<String>> {
|
|
let document = Html::parse_document(html);
|
|
let tag_sel = Self::selector("a.video_metadata_box_tag")?;
|
|
let mut tags = Vec::new();
|
|
for tag in document.select(&tag_sel) {
|
|
let title = Self::decode_html(&Self::text_of(&tag));
|
|
let title = title.trim_start_matches('#').trim().to_string();
|
|
if !title.is_empty() && !tags.contains(&title) {
|
|
tags.push(title);
|
|
}
|
|
}
|
|
Ok(tags)
|
|
}
|
|
|
|
/// The listing card already supplies everything needed for playback, so the
|
|
/// only reason to touch detail pages is the per-video tag list. Failures are
|
|
/// swallowed: an item without tags is still a good item.
|
|
async fn enrich_tags(items: &mut [VideoItem], options: &ServerOptions) {
|
|
let requester = requester_or_default(options, CHANNEL_ID, "enrich_tags");
|
|
let jobs: Vec<(usize, String)> = items
|
|
.iter()
|
|
.enumerate()
|
|
.map(|(index, item)| (index, item.url.clone()))
|
|
.collect();
|
|
|
|
let fetched = stream::iter(jobs)
|
|
.map(|(index, url)| {
|
|
let mut requester = requester.clone();
|
|
async move {
|
|
let html = requester
|
|
.get_with_headers(&url, Self::html_headers(BASE_URL), None)
|
|
.await
|
|
.ok()?;
|
|
Some((index, Self::parse_detail_tags(&html).ok()?))
|
|
}
|
|
})
|
|
.buffer_unordered(ENRICH_CONCURRENCY)
|
|
.collect::<Vec<_>>();
|
|
|
|
let Ok(fetched) = timeout(StdDuration::from_secs(ENRICH_BUDGET_SECS), fetched).await else {
|
|
return;
|
|
};
|
|
|
|
for (index, tags) in fetched.into_iter().flatten() {
|
|
if tags.is_empty() {
|
|
continue;
|
|
}
|
|
let Some(item) = items.get_mut(index) else {
|
|
continue;
|
|
};
|
|
let mut merged = item.tags.take().unwrap_or_default();
|
|
for tag in tags {
|
|
if !merged.iter().any(|existing| existing.eq_ignore_ascii_case(&tag)) {
|
|
merged.push(tag);
|
|
}
|
|
}
|
|
item.tags = Some(merged);
|
|
}
|
|
}
|
|
|
|
fn known_tag_slug(&self, value: &str) -> Option<String> {
|
|
let key = Self::normalize_key(value);
|
|
if key.is_empty() {
|
|
return None;
|
|
}
|
|
let guard = self.tags.read().ok()?;
|
|
guard
|
|
.iter()
|
|
.find(|option| {
|
|
option.id != "all"
|
|
&& (Self::normalize_key(&option.id) == key
|
|
|| Self::normalize_key(&option.title) == key)
|
|
})
|
|
.map(|option| option.id.clone())
|
|
}
|
|
|
|
fn known_category_slug(value: &str) -> Option<String> {
|
|
let key = Self::normalize_key(value);
|
|
CATEGORIES
|
|
.iter()
|
|
.find(|slug| Self::normalize_key(slug) == key)
|
|
.map(|slug| (*slug).to_string())
|
|
}
|
|
|
|
/// Resolves `options.language` against the known `REGIONS` list, falling
|
|
/// back to `DEFAULT_REGION` (uk) for anything unset or unrecognized.
|
|
fn resolve_region(options: &ServerOptions) -> String {
|
|
options
|
|
.language
|
|
.as_deref()
|
|
.and_then(|value| {
|
|
REGIONS
|
|
.iter()
|
|
.find(|(id, _)| id.eq_ignore_ascii_case(value))
|
|
.map(|(id, _)| (*id).to_string())
|
|
})
|
|
.unwrap_or_else(|| DEFAULT_REGION.to_string())
|
|
}
|
|
|
|
fn resolve_query_target(&self, query: &str, options: &ServerOptions) -> Target {
|
|
let trimmed = query.trim();
|
|
let region = Self::resolve_region(options);
|
|
|
|
// `/@slug` is the site's model route; `@slug` is the natural shorthand.
|
|
if let Some(rest) = trimmed.strip_prefix('@') {
|
|
let slug = Self::slugify(rest);
|
|
if !slug.is_empty() {
|
|
return Target::Uploader { slug, region };
|
|
}
|
|
}
|
|
|
|
if let Some((kind, value)) = trimmed.split_once(':') {
|
|
let slug = Self::slugify(value);
|
|
if !slug.is_empty() {
|
|
match kind.trim().to_ascii_lowercase().as_str() {
|
|
"tag" | "tags" => return Target::Tag(slug),
|
|
"cat" | "category" | "categories" => return Target::Category(slug),
|
|
"model" | "models" | "channel" | "uploader" => {
|
|
return Target::Uploader { slug, region };
|
|
}
|
|
_ => {}
|
|
}
|
|
}
|
|
}
|
|
|
|
// The server strips a leading `#` before dispatch, so bare tag names
|
|
// arrive here and are matched against the loaded catalog first.
|
|
if let Some(slug) = Self::known_category_slug(trimmed) {
|
|
return Target::Category(slug);
|
|
}
|
|
if let Some(slug) = self.known_tag_slug(trimmed) {
|
|
return Target::Tag(slug);
|
|
}
|
|
|
|
Target::Search {
|
|
query: trimmed.to_string(),
|
|
region,
|
|
}
|
|
}
|
|
|
|
fn resolve_option_target(&self, sort: &str, options: &ServerOptions) -> Target {
|
|
if let Some(slug) = options
|
|
.category
|
|
.as_deref()
|
|
.filter(|value| !value.is_empty() && *value != "all")
|
|
{
|
|
return Target::Category(slug.to_string());
|
|
}
|
|
// `options.filter` defaults to the literal string `"new"` when the
|
|
// client doesn't pick a tag (see `api.rs`'s `unwrap_or("new")`), so it
|
|
// must be validated against the loaded tag catalog rather than
|
|
// trusted as-is — otherwise every unfiltered request would 404
|
|
// against `/tags/new`.
|
|
if let Some(slug) = options
|
|
.filter
|
|
.as_deref()
|
|
.filter(|value| !value.is_empty() && *value != "all")
|
|
.filter(|value| {
|
|
self.tags
|
|
.read()
|
|
.map(|guard| guard.iter().any(|option| option.id == *value))
|
|
.unwrap_or(false)
|
|
})
|
|
{
|
|
return Target::Tag(slug.to_string());
|
|
}
|
|
|
|
let requested = options
|
|
.sort
|
|
.as_deref()
|
|
.filter(|value| !value.is_empty())
|
|
.unwrap_or(sort);
|
|
let sort = SORTS
|
|
.iter()
|
|
.find(|(id, _)| id.eq_ignore_ascii_case(requested))
|
|
.map(|(id, _)| (*id).to_string())
|
|
.unwrap_or_else(|| "latest".to_string());
|
|
Target::Feed {
|
|
sort,
|
|
region: Self::resolve_region(options),
|
|
}
|
|
}
|
|
|
|
async fn fetch_target(
|
|
&self,
|
|
cache: VideoCache,
|
|
target: Target,
|
|
page: u16,
|
|
per_page: usize,
|
|
options: ServerOptions,
|
|
) -> Result<Vec<VideoItem>> {
|
|
let url = Self::target_url(&target, page);
|
|
let cache_key = format!("{url}#per={per_page}");
|
|
|
|
if let Some((ts, cached)) = cache.get(&cache_key) {
|
|
if ts.elapsed().unwrap_or_default().as_secs() < LIST_CACHE_SECS {
|
|
return Ok(cached.clone());
|
|
}
|
|
}
|
|
|
|
// Referer should match the region host actually being requested, not
|
|
// always the default (uk) origin.
|
|
let referer = match url
|
|
.split_once("://")
|
|
.and_then(|(_, rest)| rest.split_once('/'))
|
|
{
|
|
Some((host, _)) => format!("https://{host}"),
|
|
None => BASE_URL.to_string(),
|
|
};
|
|
|
|
let mut requester = requester_or_default(&options, CHANNEL_ID, "fetch_target");
|
|
let html = requester
|
|
.get_with_headers(&url, Self::html_headers(&referer), None)
|
|
.await
|
|
.map_err(|e| Error::from(format!("request failed for {url}: {e}")))?;
|
|
|
|
if html.trim().is_empty() {
|
|
return Err(Error::from(format!("empty response for {url}")));
|
|
}
|
|
|
|
let mut items: Vec<VideoItem> = Self::parse_list_page(&html)?
|
|
.into_iter()
|
|
.take(per_page.max(1))
|
|
.collect();
|
|
|
|
Self::enrich_tags(&mut items, &options).await;
|
|
|
|
if !items.is_empty() {
|
|
cache.insert(cache_key, items.clone());
|
|
}
|
|
Ok(items)
|
|
}
|
|
}
|
|
|
|
#[async_trait]
|
|
impl Provider for BorntobefuckProvider {
|
|
async fn get_videos(
|
|
&self,
|
|
cache: VideoCache,
|
|
_pool: DbPool,
|
|
sort: String,
|
|
query: Option<String>,
|
|
page: String,
|
|
per_page: String,
|
|
options: ServerOptions,
|
|
) -> Vec<VideoItem> {
|
|
let page = page.parse::<u16>().unwrap_or(1).max(1);
|
|
let per_page = per_page.parse::<usize>().unwrap_or(10).clamp(1, 12);
|
|
|
|
let target = match query {
|
|
Some(value) if !value.trim().is_empty() => {
|
|
self.resolve_query_target(value.trim(), &options)
|
|
}
|
|
_ => self.resolve_option_target(&sort, &options),
|
|
};
|
|
|
|
match self
|
|
.fetch_target(cache, target, page, per_page, options)
|
|
.await
|
|
{
|
|
Ok(items) => items,
|
|
Err(error) => {
|
|
report_provider_error(CHANNEL_ID, "get_videos", &error.to_string()).await;
|
|
vec![]
|
|
}
|
|
}
|
|
}
|
|
|
|
fn get_channel(&self, clientversion: ClientVersion) -> Option<Channel> {
|
|
Some(self.build_channel(clientversion))
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn builds_paginated_urls_for_every_route() {
|
|
assert_eq!(
|
|
BorntobefuckProvider::target_url(
|
|
&Target::Feed {
|
|
sort: "latest".to_string(),
|
|
region: "uk".to_string()
|
|
},
|
|
1
|
|
),
|
|
"https://uk.borntobefuck.com/?sort=latest"
|
|
);
|
|
assert_eq!(
|
|
BorntobefuckProvider::target_url(
|
|
&Target::Feed {
|
|
sort: "mostWatched".to_string(),
|
|
region: "fr".to_string()
|
|
},
|
|
2
|
|
),
|
|
"https://fr.borntobefuck.com/?sort=mostWatched&page=2"
|
|
);
|
|
assert_eq!(
|
|
BorntobefuckProvider::target_url(
|
|
&Target::Search {
|
|
query: "big tits".to_string(),
|
|
region: "uk".to_string()
|
|
},
|
|
2
|
|
),
|
|
"https://uk.borntobefuck.com/search/results?search=big%20tits&page=2"
|
|
);
|
|
assert_eq!(
|
|
BorntobefuckProvider::target_url(&Target::Tag("cosplay".to_string()), 3),
|
|
"https://uk.borntobefuck.com/tags/cosplay?page=3"
|
|
);
|
|
assert_eq!(
|
|
BorntobefuckProvider::target_url(&Target::Category("milf".to_string()), 1),
|
|
"https://uk.borntobefuck.com/categories/milf"
|
|
);
|
|
assert_eq!(
|
|
BorntobefuckProvider::target_url(
|
|
&Target::Uploader {
|
|
slug: "tru-kait".to_string(),
|
|
region: "de".to_string()
|
|
},
|
|
2
|
|
),
|
|
"https://de.borntobefuck.com/@tru-kait?page=2"
|
|
);
|
|
// Unknown region codes fall back to the default (uk) host rather than
|
|
// producing a broken URL.
|
|
assert_eq!(
|
|
BorntobefuckProvider::target_url(
|
|
&Target::Feed {
|
|
sort: "latest".to_string(),
|
|
region: "xx".to_string()
|
|
},
|
|
1
|
|
),
|
|
"https://uk.borntobefuck.com/?sort=latest"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn derives_hls_url_from_cdn_thumbnail() {
|
|
assert_eq!(
|
|
BorntobefuckProvider::hls_from_thumb(
|
|
"https://content.zavastream.com/621742/644d7359-a256-40d5-a7ed-72a08d590ee7/thumbnail.jpg"
|
|
)
|
|
.as_deref(),
|
|
Some(
|
|
"https://content.zavastream.com/621742/644d7359-a256-40d5-a7ed-72a08d590ee7/index.m3u8"
|
|
)
|
|
);
|
|
assert_eq!(
|
|
BorntobefuckProvider::hls_from_thumb("https://example.com/other.jpg"),
|
|
None
|
|
);
|
|
}
|
|
|
|
fn blank_options() -> ServerOptions {
|
|
ServerOptions {
|
|
client_version: None,
|
|
language: None,
|
|
sort: None,
|
|
featured: None,
|
|
category: None,
|
|
sites: None,
|
|
filter: None,
|
|
public_url_base: None,
|
|
requester: None,
|
|
network: None,
|
|
stars: None,
|
|
categories: None,
|
|
duration: None,
|
|
sexuality: None,
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn routes_explicit_and_catalog_shortcuts() {
|
|
let provider = BorntobefuckProvider::new();
|
|
let options = blank_options();
|
|
assert_eq!(
|
|
provider.resolve_query_target("@tru-kait", &options),
|
|
Target::Uploader {
|
|
slug: "tru-kait".to_string(),
|
|
region: DEFAULT_REGION.to_string()
|
|
}
|
|
);
|
|
assert_eq!(
|
|
provider.resolve_query_target("model:Tru Kait", &options),
|
|
Target::Uploader {
|
|
slug: "tru-kait".to_string(),
|
|
region: DEFAULT_REGION.to_string()
|
|
}
|
|
);
|
|
assert_eq!(
|
|
provider.resolve_query_target("tag:pussy-licking", &options),
|
|
Target::Tag("pussy-licking".to_string())
|
|
);
|
|
// `milf` is a static category, so it beats the generic search path even
|
|
// before the background tag catalog has loaded.
|
|
assert_eq!(
|
|
provider.resolve_query_target("MILF", &options),
|
|
Target::Category("milf".to_string())
|
|
);
|
|
assert_eq!(
|
|
provider.resolve_query_target("some unknown thing", &options),
|
|
Target::Search {
|
|
query: "some unknown thing".to_string(),
|
|
region: DEFAULT_REGION.to_string()
|
|
}
|
|
);
|
|
|
|
// A selected region carries through to region-aware targets.
|
|
let fr_options = ServerOptions {
|
|
language: Some("fr".to_string()),
|
|
..blank_options()
|
|
};
|
|
assert_eq!(
|
|
provider.resolve_query_target("@tru-kait", &fr_options),
|
|
Target::Uploader {
|
|
slug: "tru-kait".to_string(),
|
|
region: "fr".to_string()
|
|
}
|
|
);
|
|
assert_eq!(
|
|
provider.resolve_query_target("some unknown thing", &fr_options),
|
|
Target::Search {
|
|
query: "some unknown thing".to_string(),
|
|
region: "fr".to_string()
|
|
}
|
|
);
|
|
// But tag/category shortcuts stay pinned to the English catalog
|
|
// regardless of the selected region.
|
|
assert_eq!(
|
|
provider.resolve_query_target("tag:pussy-licking", &fr_options),
|
|
Target::Tag("pussy-licking".to_string())
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn maps_sort_option_to_site_sort_id() {
|
|
let provider = BorntobefuckProvider::new();
|
|
let mut options = blank_options();
|
|
assert_eq!(
|
|
provider.resolve_option_target("date", &options),
|
|
Target::Feed {
|
|
sort: "latest".to_string(),
|
|
region: DEFAULT_REGION.to_string()
|
|
}
|
|
);
|
|
options.sort = Some("mostLiked".to_string());
|
|
assert_eq!(
|
|
provider.resolve_option_target("date", &options),
|
|
Target::Feed {
|
|
sort: "mostLiked".to_string(),
|
|
region: DEFAULT_REGION.to_string()
|
|
}
|
|
);
|
|
options.category = Some("cosplay".to_string());
|
|
assert_eq!(
|
|
provider.resolve_option_target("date", &options),
|
|
Target::Category("cosplay".to_string())
|
|
);
|
|
|
|
// An unrecognized region falls back to the default rather than being
|
|
// passed through verbatim.
|
|
options.category = None;
|
|
options.language = Some("xx".to_string());
|
|
assert_eq!(
|
|
provider.resolve_option_target("date", &options),
|
|
Target::Feed {
|
|
sort: "mostLiked".to_string(),
|
|
region: DEFAULT_REGION.to_string()
|
|
}
|
|
);
|
|
|
|
options.language = Some("DE".to_string());
|
|
assert_eq!(
|
|
provider.resolve_option_target("date", &options),
|
|
Target::Feed {
|
|
sort: "mostLiked".to_string(),
|
|
region: "de".to_string()
|
|
}
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn parses_listing_card() {
|
|
let html = r#"
|
|
<html><body>
|
|
<div class="video ranked ">
|
|
<div class="top">
|
|
<div class="infos">
|
|
<a class="card-country flag_country_video" href="https://uk.borntobefuck.com/categories/american">
|
|
<img class="card-img" src="https://static.zavastream.com/images/flags/US.svg" alt="Country US">
|
|
</a>
|
|
</div>
|
|
<a href="https://uk.borntobefuck.com/watch/40390" class="cardVideo-top-link">
|
|
<img src="https://content.zavastream.com/621742/644d7359-a256-40d5-a7ed-72a08d590ee7/thumbnail.jpg"
|
|
class="thumbnail" width="360" height="202">
|
|
<div id="video-duration" class="time" data-duration="2539"></div>
|
|
</a>
|
|
</div>
|
|
<div class="bottom">
|
|
<a href="https://uk.borntobefuck.com/@tru-kait" class="profil channel_avatar_video_card"></a>
|
|
<div class="details">
|
|
<div class="tlwr">
|
|
<a href="https://uk.borntobefuck.com/watch/40390">
|
|
<h3 class="video-title">Tru Kait Sextape Gets Railed In Threesome BGG Porn Video</h3>
|
|
</a>
|
|
</div>
|
|
<a href="https://uk.borntobefuck.com/@tru-kait">
|
|
<p class="video-channel channel_name_video_card">Tru Kait</p>
|
|
</a>
|
|
<div class="data">
|
|
<p class="date" data-published="1788703680"></p>
|
|
<p class="views" data-views="2822"></p>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</body></html>
|
|
"#;
|
|
|
|
let items = BorntobefuckProvider::parse_list_page(html).expect("parse should succeed");
|
|
assert_eq!(items.len(), 1);
|
|
let item = &items[0];
|
|
assert_eq!(item.id, "40390");
|
|
assert_eq!(
|
|
item.title,
|
|
"Tru Kait Sextape Gets Railed In Threesome BGG Porn Video"
|
|
);
|
|
assert_eq!(item.url, "https://uk.borntobefuck.com/watch/40390");
|
|
assert_eq!(item.duration, 2539);
|
|
assert_eq!(item.views, Some(2822));
|
|
assert_eq!(item.uploadedAt, Some(1788703680));
|
|
assert_eq!(item.uploader.as_deref(), Some("Tru Kait"));
|
|
assert_eq!(
|
|
item.uploaderUrl.as_deref(),
|
|
Some("https://uk.borntobefuck.com/@tru-kait")
|
|
);
|
|
assert_eq!(item.uploaderId.as_deref(), Some("borntobefuck:tru-kait"));
|
|
assert_eq!(item.tags.as_deref(), Some(&["American".to_string()][..]));
|
|
let formats = item.formats.as_ref().expect("formats");
|
|
assert_eq!(
|
|
formats[0].url,
|
|
"https://content.zavastream.com/621742/644d7359-a256-40d5-a7ed-72a08d590ee7/index.m3u8"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn parses_detail_page_tags() {
|
|
let html = r#"
|
|
<a href="https://uk.borntobefuck.com/tags/blowjob" class="active-video-tag video_metadata_box_tag">#Blowjob</a>
|
|
<a href="https://uk.borntobefuck.com/tags/big-tits" class="active-video-tag video_metadata_box_tag">#Big tits</a>
|
|
<a href="https://uk.borntobefuck.com/tags/blowjob" class="active-video-tag video_metadata_box_tag">#Blowjob</a>
|
|
"#;
|
|
let tags = BorntobefuckProvider::parse_detail_tags(html).expect("parse should succeed");
|
|
assert_eq!(tags, vec!["Blowjob".to_string(), "Big tits".to_string()]);
|
|
}
|
|
}
|