tokyomotion added

src/providers/mod.rs

@@ -37,6 +37,7 @@ pub mod porn4fans;
 pub mod pornzog;
 pub mod sxyprn;
 pub mod tnaflix;
+pub mod tokyomotion;
 pub mod viralxxxporn;
 pub mod xfree;
 pub mod xxthots;
@@ -141,6 +142,10 @@ pub static ALL_PROVIDERS: Lazy<HashMap<&'static str, DynProvider>> = Lazy::new(|
         "tnaflix",
         Arc::new(tnaflix::TnaflixProvider::new()) as DynProvider,
     );
+    m.insert(
+        "tokyomotion",
+        Arc::new(tokyomotion::TokyomotionProvider::new()) as DynProvider,
+    );
     m.insert(
         "viralxxxporn",
         Arc::new(viralxxxporn::ViralxxxpornProvider::new()) as DynProvider,

src/providers/tokyomotion.rs

@@ -0,0 +1,524 @@
+use crate::DbPool;
+use crate::api::ClientVersion;
+use crate::providers::{Provider, report_provider_error, requester_or_default};
+use crate::status::*;
+use crate::util::cache::VideoCache;
+use crate::util::parse_abbreviated_number;
+use crate::util::time::parse_time_to_seconds;
+use crate::videos::{ServerOptions, VideoItem};
+use async_trait::async_trait;
+use error_chain::error_chain;
+use htmlentity::entity::{ICodedDataTrait, decode};
+use regex::Regex;
+use url::form_urlencoded::Serializer;
+
+error_chain! {
+    foreign_links {
+        Io(std::io::Error);
+        HttpRequest(wreq::Error);
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct TokyomotionProvider {
+    url: String,
+}
+
+impl TokyomotionProvider {
+    pub fn new() -> Self {
+        Self {
+            url: "https://www.tokyomotion.net".to_string(),
+        }
+    }
+
+    fn build_channel(&self, _clientversion: ClientVersion) -> Channel {
+        Channel {
+            id: "tokyomotion".to_string(),
+            name: "Tokyo Motion".to_string(),
+            description: "Japanese porn videos.".to_string(),
+            premium: false,
+            favicon: "https://www.google.com/s2/favicons?sz=64&domain=www.tokyomotion.net"
+                .to_string(),
+            status: "active".to_string(),
+            categories: vec![],
+            options: vec![ChannelOption {
+                id: "sort".to_string(),
+                title: "Sort".to_string(),
+                description: "Sort the videos".to_string(),
+                systemImage: "list.number".to_string(),
+                colorName: "blue".to_string(),
+                options: vec![
+                    FilterOption {
+                        id: "being-watched".to_string(),
+                        title: "Being Watched".to_string(),
+                    },
+                    FilterOption {
+                        id: "most-recent".to_string(),
+                        title: "Most Recent".to_string(),
+                    },
+                    FilterOption {
+                        id: "most-viewed".to_string(),
+                        title: "Most Viewed".to_string(),
+                    },
+                    FilterOption {
+                        id: "most-commented".to_string(),
+                        title: "Most Commented".to_string(),
+                    },
+                    FilterOption {
+                        id: "top-rated".to_string(),
+                        title: "Top Rated".to_string(),
+                    },
+                    FilterOption {
+                        id: "top-favorites".to_string(),
+                        title: "Top Favorites".to_string(),
+                    },
+                    FilterOption {
+                        id: "longest".to_string(),
+                        title: "Longest".to_string(),
+                    },
+                ],
+                multiSelect: false,
+            }],
+            nsfw: true,
+            cacheDuration: Some(1800),
+        }
+    }
+
+    fn sort_code_for_get(sort: &str) -> &'static str {
+        match sort {
+            "being-watched" => "bw",
+            "most-recent" => "mr",
+            "most-commented" => "md",
+            "top-rated" => "tr",
+            "top-favorites" => "tf",
+            "longest" => "lg",
+            _ => "mv",
+        }
+    }
+
+    fn sort_code_for_query(sort: &str) -> &'static str {
+        match sort {
+            "being-watched" => "bw",
+            "most-viewed" => "mv",
+            "most-commented" => "md",
+            "top-rated" => "tr",
+            "top-favorites" => "tf",
+            "longest" => "lg",
+            _ => "mr",
+        }
+    }
+
+    fn build_get_url(&self, page: u32, sort: &str) -> String {
+        format!(
+            "{}/videos?t=a&o={}&page={page}",
+            self.url,
+            Self::sort_code_for_get(sort)
+        )
+    }
+
+    fn build_query_url(&self, query: &str, page: u32, sort: &str) -> String {
+        let mut serializer = Serializer::new(String::new());
+        serializer.append_pair("search_query", query);
+        serializer.append_pair("search_type", "videos");
+        serializer.append_pair("o", Self::sort_code_for_query(sort));
+        serializer.append_pair("page", &page.to_string());
+
+        format!("{}/search?{}", self.url, serializer.finish())
+    }
+
+    async fn get(
+        &self,
+        cache: VideoCache,
+        page: u32,
+        sort: &str,
+        options: ServerOptions,
+    ) -> Result<Vec<VideoItem>> {
+        let video_url = self.build_get_url(page, sort);
+        let old_items = match cache.get(&video_url) {
+            Some((time, items)) => {
+                if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
+                    return Ok(items.clone());
+                }
+                items.clone()
+            }
+            None => vec![],
+        };
+
+        let mut requester =
+            requester_or_default(&options, "tokyomotion", "tokyomotion.get.missing_requester");
+        let text = match requester.get(&video_url, None).await {
+            Ok(text) => text,
+            Err(e) => {
+                report_provider_error(
+                    "tokyomotion",
+                    "get.request",
+                    &format!("url={video_url}; error={e}"),
+                )
+                .await;
+                return Ok(old_items);
+            }
+        };
+
+        if text.trim().is_empty() {
+            report_provider_error(
+                "tokyomotion",
+                "get.empty_response",
+                &format!("url={video_url}"),
+            )
+            .await;
+            return Ok(old_items);
+        }
+
+        let video_items = self.get_video_items_from_html(text);
+        if !video_items.is_empty() {
+            cache.remove(&video_url);
+            cache.insert(video_url.clone(), video_items.clone());
+            return Ok(video_items);
+        }
+
+        Ok(old_items)
+    }
+
+    async fn query(
+        &self,
+        cache: VideoCache,
+        page: u32,
+        query: &str,
+        sort: &str,
+        options: ServerOptions,
+    ) -> Result<Vec<VideoItem>> {
+        let video_url = self.build_query_url(query, page, sort);
+        let old_items = match cache.get(&video_url) {
+            Some((time, items)) => {
+                if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
+                    return Ok(items.clone());
+                }
+                items.clone()
+            }
+            None => vec![],
+        };
+
+        let mut requester = requester_or_default(
+            &options,
+            "tokyomotion",
+            "tokyomotion.query.missing_requester",
+        );
+        let text = match requester.get(&video_url, None).await {
+            Ok(text) => text,
+            Err(e) => {
+                report_provider_error(
+                    "tokyomotion",
+                    "query.request",
+                    &format!("url={video_url}; error={e}"),
+                )
+                .await;
+                return Ok(old_items);
+            }
+        };
+
+        if text.trim().is_empty() {
+            report_provider_error(
+                "tokyomotion",
+                "query.empty_response",
+                &format!("url={video_url}"),
+            )
+            .await;
+            return Ok(old_items);
+        }
+
+        let video_items = self.get_video_items_from_html(text);
+        if !video_items.is_empty() {
+            cache.remove(&video_url);
+            cache.insert(video_url.clone(), video_items.clone());
+            return Ok(video_items);
+        }
+
+        Ok(old_items)
+    }
+
+    fn extract_between<'a>(text: &'a str, start: &str, end: &str) -> Option<&'a str> {
+        text.split(start).nth(1)?.split(end).next()
+    }
+
+    fn normalize_url(&self, url: &str) -> String {
+        if url.starts_with("http://") || url.starts_with("https://") {
+            return url.to_string();
+        }
+        if url.starts_with("//") {
+            return format!("https:{url}");
+        }
+        if url.starts_with('/') {
+            return format!("{}{}", self.url, url);
+        }
+        format!("{}/{}", self.url, url.trim_start_matches("./"))
+    }
+
+    fn parse_views(raw: &str) -> Option<u32> {
+        let cleaned = raw
+            .replace("views", "")
+            .replace("view", "")
+            .replace(',', "")
+            .trim()
+            .to_string();
+        parse_abbreviated_number(&cleaned)
+    }
+
+    fn parse_rating(raw: &str) -> Option<f32> {
+        let cleaned = raw.replace('%', "").trim().to_string();
+        if cleaned == "-" || cleaned.is_empty() {
+            return None;
+        }
+        cleaned.parse::<f32>().ok()
+    }
+
+    fn extract_id_from_url(url: &str) -> String {
+        url.trim_end_matches('/')
+            .split('/')
+            .find_map(|part| {
+                if part.chars().all(|c| c.is_ascii_digit()) {
+                    Some(part.to_string())
+                } else {
+                    None
+                }
+            })
+            .unwrap_or_default()
+    }
+
+    fn get_video_items_from_html(&self, html: String) -> Vec<VideoItem> {
+        if html.trim().is_empty() {
+            return vec![];
+        }
+
+        let Ok(card_re) = Regex::new(
+            r#"(?is)<a href="(?P<href>/video/(?P<id>\d+)/[^"]+)"\s+class="thumb-popu">(?P<body>.*?)</a>\s*<div class="video-added">.*?</div>\s*<div class="video-views pull-left">\s*(?P<views>.*?)\s*</div>\s*<div class="video-rating pull-right[^"]*">\s*.*?<b>(?P<rating>[^<]+)</b>"#,
+        ) else {
+            return vec![];
+        };
+
+        let mut items = Vec::new();
+
+        for captures in card_re.captures_iter(&html) {
+            let href = captures
+                .name("href")
+                .map(|m| m.as_str())
+                .unwrap_or_default();
+            let video_url = self.normalize_url(href);
+            let id = captures
+                .name("id")
+                .map(|m| m.as_str().to_string())
+                .unwrap_or_else(|| Self::extract_id_from_url(&video_url));
+            if id.is_empty() {
+                continue;
+            }
+
+            let body = captures
+                .name("body")
+                .map(|m| m.as_str())
+                .unwrap_or_default();
+            let title_raw = Self::extract_between(
+                body,
+                "<span class=\"video-title title-truncate m-t-5\">",
+                "<",
+            )
+            .or_else(|| Self::extract_between(body, "title=\"", "\""))
+            .unwrap_or_default()
+            .trim()
+            .to_string();
+            let title = decode(title_raw.as_bytes())
+                .to_string()
+                .unwrap_or(title_raw);
+            if title.trim().is_empty() {
+                continue;
+            }
+
+            let thumb = Self::extract_between(body, "<img src=\"", "\"")
+                .map(|thumb| self.normalize_url(thumb))
+                .unwrap_or_default();
+
+            let duration_raw = Self::extract_between(body, "<div class=\"duration\">", "<")
+                .unwrap_or_default()
+                .trim()
+                .to_string();
+            let duration = parse_time_to_seconds(&duration_raw).unwrap_or(0) as u32;
+
+            let views_raw = captures
+                .name("views")
+                .map(|m| m.as_str())
+                .unwrap_or_default()
+                .trim()
+                .to_string();
+            let views = Self::parse_views(&views_raw);
+
+            let rating_raw = captures
+                .name("rating")
+                .map(|m| m.as_str())
+                .unwrap_or_default()
+                .trim()
+                .to_string();
+            let rating = Self::parse_rating(&rating_raw);
+
+            let mut item = VideoItem::new(
+                id,
+                title,
+                video_url,
+                "tokyomotion".to_string(),
+                thumb,
+                duration,
+            );
+            if let Some(views) = views {
+                item = item.views(views);
+            }
+            if let Some(rating) = rating {
+                item = item.rating(rating);
+            }
+            items.push(item);
+        }
+
+        items
+    }
+}
+
+#[async_trait]
+impl Provider for TokyomotionProvider {
+    async fn get_videos(
+        &self,
+        cache: VideoCache,
+        pool: DbPool,
+        sort: String,
+        query: Option<String>,
+        page: String,
+        per_page: String,
+        options: ServerOptions,
+    ) -> Vec<VideoItem> {
+        let _ = pool;
+        let _ = per_page;
+        let page = page.parse::<u32>().unwrap_or(1);
+
+        let videos = match query {
+            Some(query) if !query.trim().is_empty() => {
+                self.query(cache, page, &query, &sort, options).await
+            }
+            _ => self.get(cache, page, &sort, options).await,
+        };
+
+        match videos {
+            Ok(videos) => videos,
+            Err(e) => {
+                report_provider_error(
+                    "tokyomotion",
+                    "get_videos",
+                    &format!("page={page}; error={e}"),
+                )
+                .await;
+                vec![]
+            }
+        }
+    }
+
+    fn get_channel(&self, clientversion: ClientVersion) -> Option<Channel> {
+        Some(self.build_channel(clientversion))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::TokyomotionProvider;
+
+    #[test]
+    fn builds_get_url_with_requested_sort() {
+        let provider = TokyomotionProvider::new();
+        assert_eq!(
+            provider.build_get_url(2, "most-viewed"),
+            "https://www.tokyomotion.net/videos?t=a&o=mv&page=2"
+        );
+        assert_eq!(
+            provider.build_get_url(2, "top-rated"),
+            "https://www.tokyomotion.net/videos?t=a&o=tr&page=2"
+        );
+    }
+
+    #[test]
+    fn builds_query_url_with_requested_sort() {
+        let provider = TokyomotionProvider::new();
+        assert_eq!(
+            provider.build_query_url("cute girl", 2, "most-recent"),
+            "https://www.tokyomotion.net/search?search_query=cute+girl&search_type=videos&o=mr&page=2"
+        );
+        assert_eq!(
+            provider.build_query_url("cute girl", 2, "top-favorites"),
+            "https://www.tokyomotion.net/search?search_query=cute+girl&search_type=videos&o=tf&page=2"
+        );
+    }
+
+    #[test]
+    fn parses_tokyomotion_cards() {
+        let provider = TokyomotionProvider::new();
+        let html = r##"
+            <div class="row">
+                <div class="col-sm-4 col-md-3 col-lg-3">
+                    <div class="well well-sm">
+                        <a href="/video/6225200/いのりちゃん　着エロ　iv-日本美女-cute-japanese-girl" class="thumb-popu">
+                            <div class="thumb-overlay">
+                                <img src="https://cdn.tokyo-motion.net/media/videos/tmb194/6225200/16.jpg" title="いのりちゃん　着エロ　IV    日本美女   Cute Japanese Girl" alt="いのりちゃん　着エロ　IV    日本美女   Cute Japanese Girl" class="img-responsive "/>
+                                <div class="hd-text-icon">HD</div>
+                                <div class="duration">
+                                    01:55:27
+                                </div>
+                            </div>
+                            <span class="video-title title-truncate m-t-5">いのりちゃん　着エロ　IV    日本美女   Cute Japanese Girl</span>
+                        </a>
+                        <div class="video-added">4 days ago</div>
+                        <div class="video-views pull-left">
+                            4000 views
+                        </div>
+                        <div class="video-rating pull-right ">
+                            <i class="fa fa-heart video-rating-heart "></i> <b>57%</b>
+                        </div>
+                        <div class="clearfix"></div>
+                    </div>
+                </div>
+                <div class="col-sm-4 col-md-3 col-lg-3">
+                    <div class="well well-sm">
+                        <a href="/video/6222401/tattooed-trans-tease-jerking-on-cam" class="thumb-popu">
+                            <div class="thumb-overlay">
+                                <img src="https://cdn.tokyo-motion.net/media/videos/tmb194/6222401/1.jpg" title="Tattooed Trans Tease Jerking On Cam" alt="Tattooed Trans Tease Jerking On Cam" class="img-responsive "/>
+                                <div class="hd-text-icon">HD</div>
+                                <div class="duration">
+                                    10:33
+                                </div>
+                            </div>
+                            <span class="video-title title-truncate m-t-5">Tattooed Trans Tease Jerking On Cam</span>
+                        </a>
+                        <div class="video-added">4 days ago</div>
+                        <div class="video-views pull-left">
+                            0 views
+                        </div>
+                        <div class="video-rating pull-right no-rating">
+                            <i class="fa fa-heart video-rating-heart no-rating"></i> <b>-</b>
+                        </div>
+                        <div class="clearfix"></div>
+                    </div>
+                </div>
+            </div>
+        "##;
+
+        let items = provider.get_video_items_from_html(html.to_string());
+        assert_eq!(items.len(), 2);
+        assert_eq!(items[0].id, "6225200");
+        assert_eq!(
+            items[0].url,
+            "https://www.tokyomotion.net/video/6225200/いのりちゃん　着エロ　iv-日本美女-cute-japanese-girl"
+        );
+        assert_eq!(
+            items[0].thumb,
+            "https://cdn.tokyo-motion.net/media/videos/tmb194/6225200/16.jpg"
+        );
+        assert_eq!(items[0].duration, 6927);
+        assert_eq!(items[0].views, Some(4000));
+        assert_eq!(items[0].rating, Some(57.0));
+        assert_eq!(items[1].id, "6222401");
+        assert_eq!(items[1].duration, 633);
+        assert_eq!(items[1].views, Some(0));
+        assert_eq!(items[1].rating, None);
+    }
+}