use crate::DbPool; use crate::api::ClientVersion; use crate::providers::{Provider, report_provider_error, requester_or_default}; use crate::status::*; use crate::util::cache::VideoCache; use crate::util::parse_abbreviated_number; use crate::util::time::parse_time_to_seconds; use crate::videos::{ServerOptions, VideoItem}; use async_trait::async_trait; use error_chain::error_chain; use htmlentity::entity::{ICodedDataTrait, decode}; use regex::Regex; use url::form_urlencoded::Serializer; pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata = crate::providers::ProviderChannelMetadata { group_id: "asian-jav", tags: &["japanese", "amateur", "jav"], }; error_chain! { foreign_links { Io(std::io::Error); HttpRequest(wreq::Error); } } #[derive(Debug, Clone)] pub struct TokyomotionProvider { url: String, } impl TokyomotionProvider { pub fn new() -> Self { Self { url: "https://www.tokyomotion.net".to_string(), } } fn build_channel(&self, _clientversion: ClientVersion) -> Channel { Channel { id: "tokyomotion".to_string(), name: "Tokyo Motion".to_string(), description: "Japanese porn videos.".to_string(), premium: false, favicon: "https://www.google.com/s2/favicons?sz=64&domain=www.tokyomotion.net" .to_string(), status: "active".to_string(), categories: vec![], options: vec![ChannelOption { id: "sort".to_string(), title: "Sort".to_string(), description: "Sort the videos".to_string(), systemImage: "list.number".to_string(), colorName: "blue".to_string(), options: vec![ FilterOption { id: "being-watched".to_string(), title: "Being Watched".to_string(), }, FilterOption { id: "most-recent".to_string(), title: "Most Recent".to_string(), }, FilterOption { id: "most-viewed".to_string(), title: "Most Viewed".to_string(), }, FilterOption { id: "most-commented".to_string(), title: "Most Commented".to_string(), }, FilterOption { id: "top-rated".to_string(), title: "Top Rated".to_string(), }, FilterOption { id: "top-favorites".to_string(), title: "Top Favorites".to_string(), }, FilterOption { id: "longest".to_string(), title: "Longest".to_string(), }, ], multiSelect: false, }], nsfw: true, cacheDuration: Some(1800), } } fn sort_code_for_get(sort: &str) -> &'static str { match sort { "being-watched" => "bw", "most-recent" => "mr", "most-commented" => "md", "top-rated" => "tr", "top-favorites" => "tf", "longest" => "lg", _ => "mv", } } fn sort_code_for_query(sort: &str) -> &'static str { match sort { "being-watched" => "bw", "most-viewed" => "mv", "most-commented" => "md", "top-rated" => "tr", "top-favorites" => "tf", "longest" => "lg", _ => "mr", } } fn build_get_url(&self, page: u32, sort: &str) -> String { format!( "{}/videos?t=a&o={}&page={page}", self.url, Self::sort_code_for_get(sort) ) } fn build_query_url(&self, query: &str, page: u32, sort: &str) -> String { let mut serializer = Serializer::new(String::new()); serializer.append_pair("search_query", query); serializer.append_pair("search_type", "videos"); serializer.append_pair("o", Self::sort_code_for_query(sort)); serializer.append_pair("page", &page.to_string()); format!("{}/search?{}", self.url, serializer.finish()) } async fn get( &self, cache: VideoCache, page: u32, sort: &str, options: ServerOptions, ) -> Result> { let video_url = self.build_get_url(page, sort); let old_items = match cache.get(&video_url) { Some((time, items)) => { if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { return Ok(items.clone()); } items.clone() } None => vec![], }; let mut requester = requester_or_default(&options, "tokyomotion", "tokyomotion.get.missing_requester"); let text = match requester.get(&video_url, None).await { Ok(text) => text, Err(e) => { report_provider_error( "tokyomotion", "get.request", &format!("url={video_url}; error={e}"), ) .await; return Ok(old_items); } }; if text.trim().is_empty() { report_provider_error( "tokyomotion", "get.empty_response", &format!("url={video_url}"), ) .await; return Ok(old_items); } let video_items = self.get_video_items_from_html(text); if !video_items.is_empty() { cache.remove(&video_url); cache.insert(video_url.clone(), video_items.clone()); return Ok(video_items); } Ok(old_items) } async fn query( &self, cache: VideoCache, page: u32, query: &str, sort: &str, options: ServerOptions, ) -> Result> { let video_url = self.build_query_url(query, page, sort); let old_items = match cache.get(&video_url) { Some((time, items)) => { if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 { return Ok(items.clone()); } items.clone() } None => vec![], }; let mut requester = requester_or_default( &options, "tokyomotion", "tokyomotion.query.missing_requester", ); let text = match requester.get(&video_url, None).await { Ok(text) => text, Err(e) => { report_provider_error( "tokyomotion", "query.request", &format!("url={video_url}; error={e}"), ) .await; return Ok(old_items); } }; if text.trim().is_empty() { report_provider_error( "tokyomotion", "query.empty_response", &format!("url={video_url}"), ) .await; return Ok(old_items); } let video_items = self.get_video_items_from_html(text); if !video_items.is_empty() { cache.remove(&video_url); cache.insert(video_url.clone(), video_items.clone()); return Ok(video_items); } Ok(old_items) } fn extract_between<'a>(text: &'a str, start: &str, end: &str) -> Option<&'a str> { text.split(start).nth(1)?.split(end).next() } fn normalize_url(&self, url: &str) -> String { if url.starts_with("http://") || url.starts_with("https://") { return url.to_string(); } if url.starts_with("//") { return format!("https:{url}"); } if url.starts_with('/') { return format!("{}{}", self.url, url); } format!("{}/{}", self.url, url.trim_start_matches("./")) } fn parse_views(raw: &str) -> Option { let cleaned = raw .replace("views", "") .replace("view", "") .replace(',', "") .trim() .to_string(); parse_abbreviated_number(&cleaned) } fn parse_rating(raw: &str) -> Option { let cleaned = raw.replace('%', "").trim().to_string(); if cleaned == "-" || cleaned.is_empty() { return None; } cleaned.parse::().ok() } fn extract_id_from_url(url: &str) -> String { url.trim_end_matches('/') .split('/') .find_map(|part| { if part.chars().all(|c| c.is_ascii_digit()) { Some(part.to_string()) } else { None } }) .unwrap_or_default() } fn get_video_items_from_html(&self, html: String) -> Vec { if html.trim().is_empty() { return vec![]; } let Ok(card_re) = Regex::new( r#"(?is)(?P.*?)\s*
.*?
\s*
\s*(?P.*?)\s*
\s*
\s*.*?(?P[^<]+)"#, ) else { return vec![]; }; let mut items = Vec::new(); for captures in card_re.captures_iter(&html) { let href = captures .name("href") .map(|m| m.as_str()) .unwrap_or_default(); let video_url = self.normalize_url(href); let id = captures .name("id") .map(|m| m.as_str().to_string()) .unwrap_or_else(|| Self::extract_id_from_url(&video_url)); if id.is_empty() { continue; } let body = captures .name("body") .map(|m| m.as_str()) .unwrap_or_default(); let title_raw = Self::extract_between( body, "", "<", ) .or_else(|| Self::extract_between(body, "title=\"", "\"")) .unwrap_or_default() .trim() .to_string(); let title = decode(title_raw.as_bytes()) .to_string() .unwrap_or(title_raw); if title.trim().is_empty() { continue; } let thumb = Self::extract_between(body, "", "<") .unwrap_or_default() .trim() .to_string(); let duration = parse_time_to_seconds(&duration_raw).unwrap_or(0) as u32; let views_raw = captures .name("views") .map(|m| m.as_str()) .unwrap_or_default() .trim() .to_string(); let views = Self::parse_views(&views_raw); let rating_raw = captures .name("rating") .map(|m| m.as_str()) .unwrap_or_default() .trim() .to_string(); let rating = Self::parse_rating(&rating_raw); let mut item = VideoItem::new( id, title, video_url, "tokyomotion".to_string(), thumb, duration, ); if let Some(views) = views { item = item.views(views); } if let Some(rating) = rating { item = item.rating(rating); } items.push(item); } items } } #[async_trait] impl Provider for TokyomotionProvider { async fn get_videos( &self, cache: VideoCache, pool: DbPool, sort: String, query: Option, page: String, per_page: String, options: ServerOptions, ) -> Vec { let _ = pool; let _ = per_page; let page = page.parse::().unwrap_or(1); let videos = match query { Some(query) if !query.trim().is_empty() => { self.query(cache, page, &query, &sort, options).await } _ => self.get(cache, page, &sort, options).await, }; match videos { Ok(videos) => videos, Err(e) => { report_provider_error( "tokyomotion", "get_videos", &format!("page={page}; error={e}"), ) .await; vec![] } } } fn get_channel(&self, clientversion: ClientVersion) -> Option { Some(self.build_channel(clientversion)) } } #[cfg(test)] mod tests { use super::TokyomotionProvider; #[test] fn builds_get_url_with_requested_sort() { let provider = TokyomotionProvider::new(); assert_eq!( provider.build_get_url(2, "most-viewed"), "https://www.tokyomotion.net/videos?t=a&o=mv&page=2" ); assert_eq!( provider.build_get_url(2, "top-rated"), "https://www.tokyomotion.net/videos?t=a&o=tr&page=2" ); } #[test] fn builds_query_url_with_requested_sort() { let provider = TokyomotionProvider::new(); assert_eq!( provider.build_query_url("cute girl", 2, "most-recent"), "https://www.tokyomotion.net/search?search_query=cute+girl&search_type=videos&o=mr&page=2" ); assert_eq!( provider.build_query_url("cute girl", 2, "top-favorites"), "https://www.tokyomotion.net/search?search_query=cute+girl&search_type=videos&o=tf&page=2" ); } #[test] fn parses_tokyomotion_cards() { let provider = TokyomotionProvider::new(); let html = r##" "##; let items = provider.get_video_items_from_html(html.to_string()); assert_eq!(items.len(), 2); assert_eq!(items[0].id, "6225200"); assert_eq!( items[0].url, "https://www.tokyomotion.net/video/6225200/いのりちゃん 着エロ iv-日本美女-cute-japanese-girl" ); assert_eq!( items[0].thumb, "https://cdn.tokyo-motion.net/media/videos/tmb194/6225200/16.jpg" ); assert_eq!(items[0].duration, 6927); assert_eq!(items[0].views, Some(4000)); assert_eq!(items[0].rating, Some(57.0)); assert_eq!(items[1].id, "6222401"); assert_eq!(items[1].duration, 633); assert_eq!(items[1].views, Some(0)); assert_eq!(items[1].rating, None); } }