531 lines
18 KiB
Rust
531 lines
18 KiB
Rust
use crate::DbPool;
|
||
use crate::api::ClientVersion;
|
||
use crate::providers::{Provider, report_provider_error, requester_or_default};
|
||
use crate::status::*;
|
||
use crate::util::cache::VideoCache;
|
||
use crate::util::parse_abbreviated_number;
|
||
use crate::util::time::parse_time_to_seconds;
|
||
use crate::videos::{ServerOptions, VideoItem};
|
||
use async_trait::async_trait;
|
||
use error_chain::error_chain;
|
||
use htmlentity::entity::{ICodedDataTrait, decode};
|
||
use regex::Regex;
|
||
use url::form_urlencoded::Serializer;
|
||
|
||
pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata =
|
||
crate::providers::ProviderChannelMetadata {
|
||
group_id: "jav",
|
||
tags: &["japanese", "amateur", "jav"],
|
||
};
|
||
|
||
error_chain! {
|
||
foreign_links {
|
||
Io(std::io::Error);
|
||
HttpRequest(wreq::Error);
|
||
}
|
||
}
|
||
|
||
#[derive(Debug, Clone)]
|
||
pub struct TokyomotionProvider {
|
||
url: String,
|
||
}
|
||
|
||
impl TokyomotionProvider {
|
||
pub fn new() -> Self {
|
||
Self {
|
||
url: "https://www.tokyomotion.net".to_string(),
|
||
}
|
||
}
|
||
|
||
fn build_channel(&self, _clientversion: ClientVersion) -> Channel {
|
||
Channel {
|
||
id: "tokyomotion".to_string(),
|
||
name: "Tokyo Motion".to_string(),
|
||
description: "Japanese porn videos.".to_string(),
|
||
premium: false,
|
||
favicon: "https://www.google.com/s2/favicons?sz=64&domain=www.tokyomotion.net"
|
||
.to_string(),
|
||
status: "active".to_string(),
|
||
categories: vec![],
|
||
options: vec![ChannelOption {
|
||
id: "sort".to_string(),
|
||
title: "Sort".to_string(),
|
||
description: "Sort the videos".to_string(),
|
||
systemImage: "list.number".to_string(),
|
||
colorName: "blue".to_string(),
|
||
options: vec![
|
||
FilterOption {
|
||
id: "being-watched".to_string(),
|
||
title: "Being Watched".to_string(),
|
||
},
|
||
FilterOption {
|
||
id: "most-recent".to_string(),
|
||
title: "Most Recent".to_string(),
|
||
},
|
||
FilterOption {
|
||
id: "most-viewed".to_string(),
|
||
title: "Most Viewed".to_string(),
|
||
},
|
||
FilterOption {
|
||
id: "most-commented".to_string(),
|
||
title: "Most Commented".to_string(),
|
||
},
|
||
FilterOption {
|
||
id: "top-rated".to_string(),
|
||
title: "Top Rated".to_string(),
|
||
},
|
||
FilterOption {
|
||
id: "top-favorites".to_string(),
|
||
title: "Top Favorites".to_string(),
|
||
},
|
||
FilterOption {
|
||
id: "longest".to_string(),
|
||
title: "Longest".to_string(),
|
||
},
|
||
],
|
||
multiSelect: false,
|
||
}],
|
||
nsfw: true,
|
||
cacheDuration: Some(1800),
|
||
}
|
||
}
|
||
|
||
fn sort_code_for_get(sort: &str) -> &'static str {
|
||
match sort {
|
||
"being-watched" => "bw",
|
||
"most-recent" => "mr",
|
||
"most-commented" => "md",
|
||
"top-rated" => "tr",
|
||
"top-favorites" => "tf",
|
||
"longest" => "lg",
|
||
_ => "mv",
|
||
}
|
||
}
|
||
|
||
fn sort_code_for_query(sort: &str) -> &'static str {
|
||
match sort {
|
||
"being-watched" => "bw",
|
||
"most-viewed" => "mv",
|
||
"most-commented" => "md",
|
||
"top-rated" => "tr",
|
||
"top-favorites" => "tf",
|
||
"longest" => "lg",
|
||
_ => "mr",
|
||
}
|
||
}
|
||
|
||
fn build_get_url(&self, page: u32, sort: &str) -> String {
|
||
format!(
|
||
"{}/videos?t=a&o={}&page={page}",
|
||
self.url,
|
||
Self::sort_code_for_get(sort)
|
||
)
|
||
}
|
||
|
||
fn build_query_url(&self, query: &str, page: u32, sort: &str) -> String {
|
||
let mut serializer = Serializer::new(String::new());
|
||
serializer.append_pair("search_query", query);
|
||
serializer.append_pair("search_type", "videos");
|
||
serializer.append_pair("o", Self::sort_code_for_query(sort));
|
||
serializer.append_pair("page", &page.to_string());
|
||
|
||
format!("{}/search?{}", self.url, serializer.finish())
|
||
}
|
||
|
||
async fn get(
|
||
&self,
|
||
cache: VideoCache,
|
||
page: u32,
|
||
sort: &str,
|
||
options: ServerOptions,
|
||
) -> Result<Vec<VideoItem>> {
|
||
let video_url = self.build_get_url(page, sort);
|
||
let old_items = match cache.get(&video_url) {
|
||
Some((time, items)) => {
|
||
if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
|
||
return Ok(items.clone());
|
||
}
|
||
items.clone()
|
||
}
|
||
None => vec![],
|
||
};
|
||
|
||
let mut requester =
|
||
requester_or_default(&options, "tokyomotion", "tokyomotion.get.missing_requester");
|
||
let text = match requester.get(&video_url, None).await {
|
||
Ok(text) => text,
|
||
Err(e) => {
|
||
report_provider_error(
|
||
"tokyomotion",
|
||
"get.request",
|
||
&format!("url={video_url}; error={e}"),
|
||
)
|
||
.await;
|
||
return Ok(old_items);
|
||
}
|
||
};
|
||
|
||
if text.trim().is_empty() {
|
||
report_provider_error(
|
||
"tokyomotion",
|
||
"get.empty_response",
|
||
&format!("url={video_url}"),
|
||
)
|
||
.await;
|
||
return Ok(old_items);
|
||
}
|
||
|
||
let video_items = self.get_video_items_from_html(text);
|
||
if !video_items.is_empty() {
|
||
cache.remove(&video_url);
|
||
cache.insert(video_url.clone(), video_items.clone());
|
||
return Ok(video_items);
|
||
}
|
||
|
||
Ok(old_items)
|
||
}
|
||
|
||
async fn query(
|
||
&self,
|
||
cache: VideoCache,
|
||
page: u32,
|
||
query: &str,
|
||
sort: &str,
|
||
options: ServerOptions,
|
||
) -> Result<Vec<VideoItem>> {
|
||
let video_url = self.build_query_url(query, page, sort);
|
||
let old_items = match cache.get(&video_url) {
|
||
Some((time, items)) => {
|
||
if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
|
||
return Ok(items.clone());
|
||
}
|
||
items.clone()
|
||
}
|
||
None => vec![],
|
||
};
|
||
|
||
let mut requester = requester_or_default(
|
||
&options,
|
||
"tokyomotion",
|
||
"tokyomotion.query.missing_requester",
|
||
);
|
||
let text = match requester.get(&video_url, None).await {
|
||
Ok(text) => text,
|
||
Err(e) => {
|
||
report_provider_error(
|
||
"tokyomotion",
|
||
"query.request",
|
||
&format!("url={video_url}; error={e}"),
|
||
)
|
||
.await;
|
||
return Ok(old_items);
|
||
}
|
||
};
|
||
|
||
if text.trim().is_empty() {
|
||
report_provider_error(
|
||
"tokyomotion",
|
||
"query.empty_response",
|
||
&format!("url={video_url}"),
|
||
)
|
||
.await;
|
||
return Ok(old_items);
|
||
}
|
||
|
||
let video_items = self.get_video_items_from_html(text);
|
||
if !video_items.is_empty() {
|
||
cache.remove(&video_url);
|
||
cache.insert(video_url.clone(), video_items.clone());
|
||
return Ok(video_items);
|
||
}
|
||
|
||
Ok(old_items)
|
||
}
|
||
|
||
fn extract_between<'a>(text: &'a str, start: &str, end: &str) -> Option<&'a str> {
|
||
text.split(start).nth(1)?.split(end).next()
|
||
}
|
||
|
||
fn normalize_url(&self, url: &str) -> String {
|
||
if url.starts_with("http://") || url.starts_with("https://") {
|
||
return url.to_string();
|
||
}
|
||
if url.starts_with("//") {
|
||
return format!("https:{url}");
|
||
}
|
||
if url.starts_with('/') {
|
||
return format!("{}{}", self.url, url);
|
||
}
|
||
format!("{}/{}", self.url, url.trim_start_matches("./"))
|
||
}
|
||
|
||
fn parse_views(raw: &str) -> Option<u32> {
|
||
let cleaned = raw
|
||
.replace("views", "")
|
||
.replace("view", "")
|
||
.replace(',', "")
|
||
.trim()
|
||
.to_string();
|
||
parse_abbreviated_number(&cleaned)
|
||
}
|
||
|
||
fn parse_rating(raw: &str) -> Option<f32> {
|
||
let cleaned = raw.replace('%', "").trim().to_string();
|
||
if cleaned == "-" || cleaned.is_empty() {
|
||
return None;
|
||
}
|
||
cleaned.parse::<f32>().ok()
|
||
}
|
||
|
||
fn extract_id_from_url(url: &str) -> String {
|
||
url.trim_end_matches('/')
|
||
.split('/')
|
||
.find_map(|part| {
|
||
if part.chars().all(|c| c.is_ascii_digit()) {
|
||
Some(part.to_string())
|
||
} else {
|
||
None
|
||
}
|
||
})
|
||
.unwrap_or_default()
|
||
}
|
||
|
||
fn get_video_items_from_html(&self, html: String) -> Vec<VideoItem> {
|
||
if html.trim().is_empty() {
|
||
return vec![];
|
||
}
|
||
|
||
let Ok(card_re) = Regex::new(
|
||
r#"(?is)<a href="(?P<href>/video/(?P<id>\d+)/[^"]+)"\s+class="thumb-popu">(?P<body>.*?)</a>\s*<div class="video-added">.*?</div>\s*<div class="video-views pull-left">\s*(?P<views>.*?)\s*</div>\s*<div class="video-rating pull-right[^"]*">\s*.*?<b>(?P<rating>[^<]+)</b>"#,
|
||
) else {
|
||
return vec![];
|
||
};
|
||
|
||
let mut items = Vec::new();
|
||
|
||
for captures in card_re.captures_iter(&html) {
|
||
let href = captures
|
||
.name("href")
|
||
.map(|m| m.as_str())
|
||
.unwrap_or_default();
|
||
let video_url = self.normalize_url(href);
|
||
let id = captures
|
||
.name("id")
|
||
.map(|m| m.as_str().to_string())
|
||
.unwrap_or_else(|| Self::extract_id_from_url(&video_url));
|
||
if id.is_empty() {
|
||
continue;
|
||
}
|
||
|
||
let body = captures
|
||
.name("body")
|
||
.map(|m| m.as_str())
|
||
.unwrap_or_default();
|
||
let title_raw = Self::extract_between(
|
||
body,
|
||
"<span class=\"video-title title-truncate m-t-5\">",
|
||
"<",
|
||
)
|
||
.or_else(|| Self::extract_between(body, "title=\"", "\""))
|
||
.unwrap_or_default()
|
||
.trim()
|
||
.to_string();
|
||
let title = decode(title_raw.as_bytes())
|
||
.to_string()
|
||
.unwrap_or(title_raw);
|
||
if title.trim().is_empty() {
|
||
continue;
|
||
}
|
||
|
||
let thumb = Self::extract_between(body, "<img src=\"", "\"")
|
||
.map(|thumb| self.normalize_url(thumb))
|
||
.unwrap_or_default();
|
||
|
||
let duration_raw = Self::extract_between(body, "<div class=\"duration\">", "<")
|
||
.unwrap_or_default()
|
||
.trim()
|
||
.to_string();
|
||
let duration = parse_time_to_seconds(&duration_raw).unwrap_or(0) as u32;
|
||
|
||
let views_raw = captures
|
||
.name("views")
|
||
.map(|m| m.as_str())
|
||
.unwrap_or_default()
|
||
.trim()
|
||
.to_string();
|
||
let views = Self::parse_views(&views_raw);
|
||
|
||
let rating_raw = captures
|
||
.name("rating")
|
||
.map(|m| m.as_str())
|
||
.unwrap_or_default()
|
||
.trim()
|
||
.to_string();
|
||
let rating = Self::parse_rating(&rating_raw);
|
||
|
||
let mut item = VideoItem::new(
|
||
id,
|
||
title,
|
||
video_url,
|
||
"tokyomotion".to_string(),
|
||
thumb,
|
||
duration,
|
||
);
|
||
if let Some(views) = views {
|
||
item = item.views(views);
|
||
}
|
||
if let Some(rating) = rating {
|
||
item = item.rating(rating);
|
||
}
|
||
items.push(item);
|
||
}
|
||
|
||
items
|
||
}
|
||
}
|
||
|
||
#[async_trait]
|
||
impl Provider for TokyomotionProvider {
|
||
async fn get_videos(
|
||
&self,
|
||
cache: VideoCache,
|
||
pool: DbPool,
|
||
sort: String,
|
||
query: Option<String>,
|
||
page: String,
|
||
per_page: String,
|
||
options: ServerOptions,
|
||
) -> Vec<VideoItem> {
|
||
let _ = pool;
|
||
let _ = per_page;
|
||
let page = page.parse::<u32>().unwrap_or(1);
|
||
|
||
let videos = match query {
|
||
Some(query) if !query.trim().is_empty() => {
|
||
self.query(cache, page, &query, &sort, options).await
|
||
}
|
||
_ => self.get(cache, page, &sort, options).await,
|
||
};
|
||
|
||
match videos {
|
||
Ok(videos) => videos,
|
||
Err(e) => {
|
||
report_provider_error(
|
||
"tokyomotion",
|
||
"get_videos",
|
||
&format!("page={page}; error={e}"),
|
||
)
|
||
.await;
|
||
vec![]
|
||
}
|
||
}
|
||
}
|
||
|
||
fn get_channel(&self, clientversion: ClientVersion) -> Option<Channel> {
|
||
Some(self.build_channel(clientversion))
|
||
}
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::TokyomotionProvider;
|
||
|
||
#[test]
|
||
fn builds_get_url_with_requested_sort() {
|
||
let provider = TokyomotionProvider::new();
|
||
assert_eq!(
|
||
provider.build_get_url(2, "most-viewed"),
|
||
"https://www.tokyomotion.net/videos?t=a&o=mv&page=2"
|
||
);
|
||
assert_eq!(
|
||
provider.build_get_url(2, "top-rated"),
|
||
"https://www.tokyomotion.net/videos?t=a&o=tr&page=2"
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn builds_query_url_with_requested_sort() {
|
||
let provider = TokyomotionProvider::new();
|
||
assert_eq!(
|
||
provider.build_query_url("cute girl", 2, "most-recent"),
|
||
"https://www.tokyomotion.net/search?search_query=cute+girl&search_type=videos&o=mr&page=2"
|
||
);
|
||
assert_eq!(
|
||
provider.build_query_url("cute girl", 2, "top-favorites"),
|
||
"https://www.tokyomotion.net/search?search_query=cute+girl&search_type=videos&o=tf&page=2"
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn parses_tokyomotion_cards() {
|
||
let provider = TokyomotionProvider::new();
|
||
let html = r##"
|
||
<div class="row">
|
||
<div class="col-sm-4 col-md-3 col-lg-3">
|
||
<div class="well well-sm">
|
||
<a href="/video/6225200/いのりちゃん 着エロ iv-日本美女-cute-japanese-girl" class="thumb-popu">
|
||
<div class="thumb-overlay">
|
||
<img src="https://cdn.tokyo-motion.net/media/videos/tmb194/6225200/16.jpg" title="いのりちゃん 着エロ IV 日本美女 Cute Japanese Girl" alt="いのりちゃん 着エロ IV 日本美女 Cute Japanese Girl" class="img-responsive "/>
|
||
<div class="hd-text-icon">HD</div>
|
||
<div class="duration">
|
||
01:55:27
|
||
</div>
|
||
</div>
|
||
<span class="video-title title-truncate m-t-5">いのりちゃん 着エロ IV 日本美女 Cute Japanese Girl</span>
|
||
</a>
|
||
<div class="video-added">4 days ago</div>
|
||
<div class="video-views pull-left">
|
||
4000 views
|
||
</div>
|
||
<div class="video-rating pull-right ">
|
||
<i class="fa fa-heart video-rating-heart "></i> <b>57%</b>
|
||
</div>
|
||
<div class="clearfix"></div>
|
||
</div>
|
||
</div>
|
||
<div class="col-sm-4 col-md-3 col-lg-3">
|
||
<div class="well well-sm">
|
||
<a href="/video/6222401/tattooed-trans-tease-jerking-on-cam" class="thumb-popu">
|
||
<div class="thumb-overlay">
|
||
<img src="https://cdn.tokyo-motion.net/media/videos/tmb194/6222401/1.jpg" title="Tattooed Trans Tease Jerking On Cam" alt="Tattooed Trans Tease Jerking On Cam" class="img-responsive "/>
|
||
<div class="hd-text-icon">HD</div>
|
||
<div class="duration">
|
||
10:33
|
||
</div>
|
||
</div>
|
||
<span class="video-title title-truncate m-t-5">Tattooed Trans Tease Jerking On Cam</span>
|
||
</a>
|
||
<div class="video-added">4 days ago</div>
|
||
<div class="video-views pull-left">
|
||
0 views
|
||
</div>
|
||
<div class="video-rating pull-right no-rating">
|
||
<i class="fa fa-heart video-rating-heart no-rating"></i> <b>-</b>
|
||
</div>
|
||
<div class="clearfix"></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
"##;
|
||
|
||
let items = provider.get_video_items_from_html(html.to_string());
|
||
assert_eq!(items.len(), 2);
|
||
assert_eq!(items[0].id, "6225200");
|
||
assert_eq!(
|
||
items[0].url,
|
||
"https://www.tokyomotion.net/video/6225200/いのりちゃん 着エロ iv-日本美女-cute-japanese-girl"
|
||
);
|
||
assert_eq!(
|
||
items[0].thumb,
|
||
"https://cdn.tokyo-motion.net/media/videos/tmb194/6225200/16.jpg"
|
||
);
|
||
assert_eq!(items[0].duration, 6927);
|
||
assert_eq!(items[0].views, Some(4000));
|
||
assert_eq!(items[0].rating, Some(57.0));
|
||
assert_eq!(items[1].id, "6222401");
|
||
assert_eq!(items[1].duration, 633);
|
||
assert_eq!(items[1].views, Some(0));
|
||
assert_eq!(items[1].rating, None);
|
||
}
|
||
}
|