javtiful done

Cargo.toml

@@ -16,7 +16,7 @@ ntex-files = "2.0.0"
 serde = "1.0.228"
 serde_json = "1.0.145"
 tokio = { version = "1.47.1", features = ["full"] }
-wreq =  { version = "5.3.0", features = ["full", "cookies"] }
+wreq =  { version = "5.3.0", features = ["full", "cookies", "multipart"] }
 wreq-util = "2"
 percent-encoding = "2.3.2"
 capitalize = "0.3.4"

src/providers/javtiful.rs

@@ -0,0 +1,423 @@
+use crate::DbPool;
+use crate::api::ClientVersion;
+use crate::providers::Provider;
+use crate::status::*;
+use crate::util::cache::VideoCache;
+use crate::util::discord::send_discord_error_report;
+use crate::util::requester::Requester;
+use crate::util::time::parse_time_to_seconds;
+use crate::videos::{ServerOptions, VideoFormat, VideoItem};
+
+use async_trait::async_trait;
+use error_chain::error_chain;
+use futures::future::join_all;
+use htmlentity::entity::{decode, ICodedDataTrait};
+use std::sync::{Arc, RwLock};
+use std::{vec};
+use titlecase::Titlecase;
+use wreq::Version;
+
+error_chain! {
+    foreign_links {
+        Io(std::io::Error);
+        HttpRequest(wreq::Error);
+        Json(serde_json::Error);
+    }
+    errors {
+        Parse(msg: String) {
+            description("parse error")
+            display("parse error: {}", msg)
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct JavtifulProvider {
+    url: String,
+    categories: Arc<RwLock<Vec<FilterOption>>>,
+}
+
+impl JavtifulProvider {
+    pub fn new() -> Self {
+        let provider = Self {
+            url: "https://javtiful.com".to_string(),
+            categories: Arc::new(RwLock::new(vec![])),
+        };
+        provider
+    }
+
+    fn build_channel(&self, clientversion: ClientVersion) -> Channel {
+        let _ = clientversion;
+        Channel {
+            id: "pimpbunny".to_string(),
+            name: "Javtiful".to_string(),
+            description: "Watch Porn!".to_string(),
+            premium: false,
+            favicon: "https://www.google.com/s2/favicons?sz=64&domain=javtiful.com".to_string(),
+            status: "active".to_string(),
+            categories: self
+                .categories
+                .read()
+                .unwrap()
+                .iter()
+                .map(|c| c.title.clone())
+                .collect(),
+            options: vec![ChannelOption {
+                id: "sort".to_string(),
+                title: "Sort".to_string(),
+                description: "Sort the Videos".to_string(),
+                systemImage: "list.number".to_string(),
+                colorName: "blue".to_string(),
+                options: vec![
+                    FilterOption {
+                        id: "newest".into(),
+                        title: "Newest".into(),
+                    },
+                    FilterOption {
+                        id: "top rated".into(),
+                        title: "Top Rated".into(),
+                    },
+                    FilterOption {
+                        id: "most viewed".into(),
+                        title: "Most Viewed".into(),
+                    },
+                    FilterOption {
+                        id: "top favorites".into(),
+                        title: "Top Favorites".into(),
+                    },
+                ],
+                multiSelect: false,
+            }],
+            nsfw: true,
+            cacheDuration: None,
+        }
+    }
+
+    fn push_unique(target: &Arc<RwLock<Vec<FilterOption>>>, item: FilterOption) {
+        if let Ok(mut vec) = target.write() {
+            if !vec.iter().any(|x| x.id == item.id) {
+                vec.push(item);
+            }
+        }
+    }
+
+    async fn get(
+        &self,
+        cache: VideoCache,
+        page: u8,
+        sort: &str,
+        options: ServerOptions,
+    ) -> Result<Vec<VideoItem>> {
+        let sort_string = match sort {
+            "top rated" => "/sort=top_rated",
+            "most viewed" => "/sort=most_viewed",
+            _ => "",
+        };
+        let video_url = format!(
+            "{}/videos{}?page={}",
+            self.url, sort_string, page
+        );
+        let old_items = match cache.get(&video_url) {
+            Some((time, items)) => {
+                if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
+                    return Ok(items.clone());
+                } else {
+                    items.clone()
+                }
+            }
+            None => {
+                vec![]
+            }
+        };
+
+        let mut requester = options.requester.clone().unwrap();
+        let text = requester.get(&video_url, Some(Version::HTTP_2)).await.unwrap();
+        if page > 1 && !text.contains(&format!("<li class=\"page-item active\"><span class=\"page-link\">{}</span>", page)) {
+            eprint!("Javtiful query returned no results for page {}", page);
+            return Ok(vec![]);
+        }
+        let video_items: Vec<VideoItem> = self
+            .get_video_items_from_html(text.clone(), &mut requester)
+            .await;
+        if !video_items.is_empty() {
+            cache.remove(&video_url);
+            cache.insert(video_url.clone(), video_items.clone());
+        } else {
+            return Ok(old_items);
+        }
+        Ok(video_items)
+    }
+
+    async fn query(
+        &self,
+        cache: VideoCache,
+        page: u8,
+        query: &str,
+        options: ServerOptions,
+    ) -> Result<Vec<VideoItem>> {
+        let sort_string = match options.sort.as_deref().unwrap_or("") {
+            "top rated" => "/sort=top_rated",
+            "most viewed" => "/sort=most_viewed",
+            _ => "",
+        };
+        let video_url = format!(
+            "{}/videos{}?search_query={}&page={}",
+            self.url, sort_string, query.replace(" ","+"), page
+        );
+        // Check our Video Cache. If the result is younger than 1 hour, we return it.
+        let old_items = match cache.get(&video_url) {
+            Some((time, items)) => {
+                if time.elapsed().unwrap_or_default().as_secs() < 60 * 5 {
+                    return Ok(items.clone());
+                } else {
+                    let _ = cache.check().await;
+                    return Ok(items.clone());
+                }
+            }
+            None => {
+                vec![]
+            }
+        };
+
+        let mut requester = options.requester.clone().unwrap();
+
+        let text = requester.get(&video_url, Some(Version::HTTP_2)).await.unwrap();
+        if page > 1 && !text.contains(&format!("<li class=\"page-item active\"><span class=\"page-link\">{}</span>", page)) {
+            eprint!("Javtiful query returned no results for page {}", page);
+            return Ok(vec![]);
+        }
+        let video_items: Vec<VideoItem> = self
+            .get_video_items_from_html(text.clone(), &mut requester)
+            .await;
+        if !video_items.is_empty() {
+            cache.remove(&video_url);
+            cache.insert(video_url.clone(), video_items.clone());
+        } else {
+            return Ok(old_items);
+        }
+        Ok(video_items)
+    }
+
+    async fn get_video_items_from_html(
+        &self,
+        html: String,
+        requester: &mut Requester,
+    ) -> Vec<VideoItem> {
+        if html.is_empty() || html.contains("404 Not Found") {
+            eprint!("Javtiful returned empty or 404 html");
+            return vec![];
+        }
+
+        let block = match html
+            .split("pagination ")
+            .next()
+            .and_then(|s| s.split("row row-cols-1 row-cols-sm-2 row-cols-lg-3 row-cols-xl-4").nth(1))
+        {
+            Some(b) => b,
+            None => return vec![],
+        };
+
+        let futures = block
+            .split("card ")
+            .skip(1)
+            .map(|el| self.get_video_item(el.to_string(), requester.clone()));
+
+        join_all(futures)
+            .await
+            .into_iter()
+            .inspect(|r| {
+                if let Err(e) = r {
+                    let _ = futures::executor::block_on(send_discord_error_report(
+                        &e,
+                        Some("Javtiful Provider"),
+                        Some("Failed to get video item"),
+                        file!(),
+                        line!(),
+                        module_path!(),
+                    ));
+                }
+            })
+            .filter_map(Result::ok)
+            .collect()
+    }
+
+    async fn get_video_item(
+        &self,
+        seg: String,
+        mut requester: Requester,
+    ) -> Result<VideoItem> {
+        let video_url = seg
+            .split(" href=\"")
+            .nth(1)
+            .and_then(|s| s.split('"').next())
+            .ok_or_else(|| ErrorKind::Parse("video url".into()))?
+            .to_string();
+
+        let mut title = seg
+            .split(" alt=\"")
+            .nth(1)
+            .and_then(|s| s.split('"').next())
+            .ok_or_else(|| ErrorKind::Parse("video title".into()))?
+            .trim()
+            .to_string();
+
+        title = decode(title.as_bytes()).to_string().unwrap_or(title).titlecase();
+        let id = video_url
+            .split('/')
+            .nth(5)
+            .and_then(|s| s.split('.').next())
+            .ok_or_else(|| ErrorKind::Parse("video id".into()))?
+            .to_string();
+        let thumb_block = seg
+            .split("<img ")
+            .nth(1)
+            .ok_or_else(|| ErrorKind::Parse("thumb block".into()))?;
+
+        let thumb = thumb_block
+            .split("data-src=\"")
+            .nth(1)
+            .and_then(|s| s.split('"').next())
+            .unwrap_or("")
+            .to_string();
+        let mut preview = seg
+            .split("data-trailer=\"")
+            .nth(1)
+            .and_then(|s| s.split('"').next())
+            .unwrap_or("")
+            .to_string();
+        let raw_duration = seg
+            .split("label-duration\">")
+            .nth(1)
+            .and_then(|s| s.split('<').next())
+            .unwrap_or("")
+            .to_string();
+        let duration = parse_time_to_seconds(&raw_duration).unwrap_or(0) as u32;
+        let (tags, formats, views) =
+            self.extract_media(&video_url, &mut requester).await?;
+
+        if preview.len() == 0 {
+            preview = format!("https://trailers.jav.si/preview/{id}.mp4");
+        }
+        let video_item = VideoItem::new(
+            id,
+            title,
+            video_url,
+            "javtiful".into(),
+            thumb,
+            duration,
+        )
+        .formats(formats)
+        .tags(tags)
+        .preview(preview)
+        .views(views);
+        Ok(video_item)
+
+    }
+
+    async fn extract_media(
+        &self,
+        url: &str,
+        requester: &mut Requester,
+    ) -> Result<(Vec<String>, Vec<VideoFormat>, u32)> {
+        let text = requester
+            .get(url, Some(Version::HTTP_2))
+            .await
+            .map_err(|e| Error::from(format!("{}", e)))?;
+        let tags = text.split("related-actress").next()
+            .and_then(|s| s.split("video-comments").next())
+            .and_then(|s| s.split(">Tags<").nth(1))
+            .map(|tag_block| {
+                tag_block
+                    .split("<a ")
+                    .skip(1)
+                    .filter_map(|tag_el| {
+                        tag_el
+                            .split('>')
+                            .nth(1)
+                            .and_then(|s| s.split('<').next())
+                            .map(|s| decode(s.as_bytes()).to_string().unwrap_or(s.to_string()).titlecase())
+                    })
+                    .collect()
+            })
+            .unwrap_or_else(|| vec![]);
+        for tag in &tags {
+            Self::push_unique(&self.categories, FilterOption {
+                id: tag.to_ascii_lowercase().replace(" ","+"),
+                title: tag.to_string(),
+            });
+        }
+        let views = text.split(" Views ")
+            .next()
+            .and_then(|s| s.split(" ").last())
+            .and_then(|s| s.replace(".","")
+            .parse::<u32>().ok())
+            .unwrap_or(0);
+
+        let quality="1080p".to_string();
+        let video_id = url
+            .split('/')
+            .nth(4)
+            .ok_or_else(|| ErrorKind::Parse("video id for media".into()))?
+            .to_string();
+
+        let token = text.split("data-csrf-token=\"")
+            .nth(1)
+            .and_then(|s| s.split('"').next())
+            .ok_or_else(|| ErrorKind::Parse("csrf token".into()))?
+            .to_string();
+        
+        let form = wreq::multipart::Form::new()
+            .text("video_id", video_id.clone())
+            .text("pid_c", "".to_string())
+            .text("token", token.clone());
+        let resp = requester
+            .post_multipart(
+                "https://javtiful.com/ajax/get_cdn",
+                form,
+                vec![("Referer".to_string(), url.to_string())],
+                Some(Version::HTTP_11),
+            )
+            .await
+            .map_err(|e| Error::from(format!("{}", e)))?;
+        let text = resp.text().await?;
+        let json: serde_json::Value = serde_json::from_str(&text)?;
+        let video_url = json.get("playlists")
+            .ok_or_else(|| ErrorKind::Parse("video_url in json".into()))?
+            .to_string().replace("\"", "");
+        Ok((
+            tags,
+            vec![VideoFormat::new(video_url, quality, "video/mp4".into())],
+            views,
+        ))
+    }
+}
+
+#[async_trait]
+impl Provider for JavtifulProvider {
+    async fn get_videos(
+        &self,
+        cache: VideoCache,
+        _pool: DbPool,
+        sort: String,
+        query: Option<String>,
+        page: String,
+        _per_page: String,
+        options: ServerOptions,
+    ) -> Vec<VideoItem> {
+        let page = page.parse::<u8>().unwrap_or(1);
+
+        let res = match query {
+            Some(q) => self.to_owned().query(cache, page, &q, options).await,
+            None => self.get(cache, page, &sort, options).await,
+        };
+
+        res.unwrap_or_else(|e| {
+            eprintln!("javtiful error: {e}");
+            vec![]
+        })
+    }
+
+    fn get_channel(&self, v: ClientVersion) -> Channel {
+        self.build_channel(v)
+    }
+}

src/providers/mod.rs

@@ -39,6 +39,7 @@ pub mod xxdbx;
 pub mod hqporner;
 pub mod noodlemagazine;
 pub mod pimpbunny;
+pub mod javtiful;
 
 // convenient alias
 pub type DynProvider = Arc<dyn Provider>;
@@ -55,6 +56,7 @@ pub static ALL_PROVIDERS: Lazy<HashMap<&'static str, DynProvider>> = Lazy::new(|
     m.insert("pmvhaven", Arc::new(pmvhaven::PmvhavenProvider::new()) as DynProvider);
     m.insert("noodlemagazine", Arc::new(noodlemagazine::NoodlemagazineProvider::new()) as DynProvider);
     m.insert("pimpbunny", Arc::new(pimpbunny::PimpbunnyProvider::new()) as DynProvider);
+    m.insert("javtiful", Arc::new(javtiful::JavtifulProvider::new()) as DynProvider);
     // add more here as you migrate them
     m
 });

src/util/requester.rs

@@ -1,4 +1,5 @@
 use serde::Serialize;
+use wreq::multipart::Form;
 use std::env;
 use wreq::Client;
 use wreq::Proxy;
@@ -119,14 +120,37 @@ impl Requester {
     where
         S: Serialize + ?Sized,
     {
-        let client = Client::builder()
+        let mut request = self.client.post(url).version(Version::HTTP_11).json(data);
-            .cert_verification(false)
-            .emulation(Emulation::Firefox136)
-            .cookie_store(true)
-            .build()
-            .expect("Failed to create HTTP client");
 
-        let mut request = client.post(url).version(Version::HTTP_11).json(data);
+        // Set custom headers
+        for (key, value) in headers.iter() {
+            request = request.header(key, value);
+        }
+
+        if self.proxy {
+            if let Ok(proxy_url) = env::var("BURP_URL") {
+                let proxy = Proxy::all(&proxy_url).unwrap();
+                request = request.proxy(proxy);
+            }
+        }
+
+        request.send().await
+    }
+
+    pub async fn post_multipart(
+        &mut self,
+        url: &str,
+        form: Form,
+        headers: Vec<(String, String)>,
+        _http_version: Option<Version>,
+    ) -> Result<Response, wreq::Error>
+    {
+        let http_version = match _http_version {
+            Some(v) => v,
+            None => Version::HTTP_11,
+        };
+
+        let mut request = self.client.post(url).multipart(form).version(http_version);
 
         // Set custom headers
         for (key, value) in headers.iter() {