diff --git a/Cargo.toml b/Cargo.toml index d0aee5b..2846f59 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,8 +16,8 @@ ntex-files = "2.0.0" serde = "1.0.228" serde_json = "1.0.145" tokio = { version = "1.49", features = ["full"] } -wreq = { version = "5.3.0", features = ["full", "cookies", "multipart"] } -wreq-util = "2" +wreq = { version = "6.0.0-rc.26", features = ["cookies", "multipart", "json"] } +wreq-util = "3.0.0-rc.10" percent-encoding = "2.3.2" capitalize = "0.3.4" url = "2.5.7" diff --git a/src/util/requester.rs b/src/util/requester.rs index 183f4f3..2dbe68a 100644 --- a/src/util/requester.rs +++ b/src/util/requester.rs @@ -1,10 +1,13 @@ use serde::Serialize; use std::env; +use std::fmt; +use std::sync::Arc; use wreq::Client; use wreq::Proxy; use wreq::Response; use wreq::Version; -use wreq::header::HeaderValue; +use wreq::cookie::Jar; +use wreq::header::{HeaderMap, HeaderValue, USER_AGENT}; use wreq::multipart::Form; use wreq::redirect::Policy; use wreq_util::Emulation; @@ -16,28 +19,56 @@ use crate::util::proxy; // A Send + Sync error type for all async paths type AnyErr = Box; -#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)] +#[derive(serde::Serialize, serde::Deserialize, Clone)] pub struct Requester { #[serde(skip)] client: Client, + #[serde(skip)] + cookie_jar: Arc, proxy: bool, flaresolverr_session: Option, + user_agent: Option, +} + +impl fmt::Debug for Requester { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Requester") + .field("proxy", &self.proxy) + .field("flaresolverr_session", &self.flaresolverr_session) + .field("user_agent", &self.user_agent) + .finish() + } } impl Requester { - pub fn new() -> Self { - let client = Client::builder() + fn build_client(cookie_jar: Arc, user_agent: Option<&str>) -> Client { + let mut builder = Client::builder() .cert_verification(false) .emulation(Emulation::Firefox136) - .cookie_store(true) - .redirect(Policy::default()) - .build() - .expect("Failed to create HTTP client"); + .cookie_provider(cookie_jar) + .redirect(Policy::default()); + + if let Some(user_agent) = user_agent { + let mut headers = HeaderMap::new(); + if let Ok(value) = HeaderValue::from_str(user_agent) { + headers.insert(USER_AGENT, value); + builder = builder.default_headers(headers); + } + } + + builder.build().expect("Failed to create HTTP client") + } + + pub fn new() -> Self { + let cookie_jar = Arc::new(Jar::default()); + let client = Self::build_client(cookie_jar.clone(), None); let requester = Requester { client, + cookie_jar, proxy: false, flaresolverr_session: None, + user_agent: None, }; proxy::init_all_proxies_background(requester.clone()); @@ -53,12 +84,7 @@ impl Requester { } pub async fn get_raw(&mut self, url: &str) -> Result { - let client = Client::builder() - .cert_verification(false) - .emulation(Emulation::Firefox136) - .cookie_store(true) - .build() - .expect("Failed to create HTTP client"); + let client = Self::build_client(self.cookie_jar.clone(), self.user_agent.as_deref()); let mut request = client.get(url).version(Version::HTTP_11); @@ -77,12 +103,7 @@ impl Requester { url: &str, headers: Vec<(String, String)>, ) -> Result { - let client = Client::builder() - .cert_verification(false) - .emulation(Emulation::Firefox136) - .cookie_store(true) - .build() - .expect("Failed to create HTTP client"); + let client = Self::build_client(self.cookie_jar.clone(), self.user_agent.as_deref()); let mut request = client.get(url).version(Version::HTTP_11); @@ -250,31 +271,18 @@ impl Requester { // Rebuild client and apply UA/cookies from FlareSolverr let cookie_origin = url.split('/').take(3).collect::>().join("/"); - self.client = Client::builder() - .cert_verification(false) - .emulation(Emulation::Firefox136) - .cookie_store(true) - .redirect(Policy::default()) - .build() - .expect("Failed to create HTTP client"); - let useragent = res.solution.userAgent; - self.client - .update() - .headers(|headers| { - headers.insert("User-Agent", HeaderValue::from_str(&useragent).unwrap()); - }) - .apply() - .unwrap(); + self.user_agent = Some(useragent); - if let Ok(origin) = url::Url::parse(&cookie_origin) { + if url::Url::parse(&cookie_origin).is_ok() { for cookie in res.solution.cookies { - let header = - HeaderValue::from_str(&format!("{}={}", cookie.name, cookie.value)).unwrap(); - self.client.set_cookie(&origin, header); + self.cookie_jar + .add_cookie_str(&format!("{}={}", cookie.name, cookie.value), &cookie_origin); } } + self.client = Self::build_client(self.cookie_jar.clone(), self.user_agent.as_deref()); + // Retry the original URL with the updated client & (optional) proxy let mut request = self.client.get(url).version(Version::HTTP_11); for (key, value) in headers.iter() {