upgraded wreq

Cargo.toml

@@ -16,8 +16,8 @@ ntex-files = "2.0.0"
 serde = "1.0.228"
 serde_json = "1.0.145"
 tokio = { version = "1.49", features = ["full"] }
-wreq =  { version = "5.3.0", features = ["full", "cookies", "multipart"] }
+wreq =  { version = "6.0.0-rc.26", features = ["cookies", "multipart", "json"] }
-wreq-util = "2"
+wreq-util = "3.0.0-rc.10"
 percent-encoding = "2.3.2"
 capitalize = "0.3.4"
 url = "2.5.7"

src/util/requester.rs

@@ -1,10 +1,13 @@
 use serde::Serialize;
 use std::env;
+use std::fmt;
+use std::sync::Arc;
 use wreq::Client;
 use wreq::Proxy;
 use wreq::Response;
 use wreq::Version;
-use wreq::header::HeaderValue;
+use wreq::cookie::Jar;
+use wreq::header::{HeaderMap, HeaderValue, USER_AGENT};
 use wreq::multipart::Form;
 use wreq::redirect::Policy;
 use wreq_util::Emulation;
@@ -16,28 +19,56 @@ use crate::util::proxy;
 // A Send + Sync error type for all async paths
 type AnyErr = Box<dyn std::error::Error + Send + Sync + 'static>;
 
-#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)]
+#[derive(serde::Serialize, serde::Deserialize, Clone)]
 pub struct Requester {
     #[serde(skip)]
     client: Client,
+    #[serde(skip)]
+    cookie_jar: Arc<Jar>,
     proxy: bool,
     flaresolverr_session: Option<String>,
+    user_agent: Option<String>,
+}
+
+impl fmt::Debug for Requester {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("Requester")
+            .field("proxy", &self.proxy)
+            .field("flaresolverr_session", &self.flaresolverr_session)
+            .field("user_agent", &self.user_agent)
+            .finish()
+    }
 }
 
 impl Requester {
-    pub fn new() -> Self {
+    fn build_client(cookie_jar: Arc<Jar>, user_agent: Option<&str>) -> Client {
-        let client = Client::builder()
+        let mut builder = Client::builder()
             .cert_verification(false)
             .emulation(Emulation::Firefox136)
-            .cookie_store(true)
+            .cookie_provider(cookie_jar)
-            .redirect(Policy::default())
+            .redirect(Policy::default());
-            .build()
+
-            .expect("Failed to create HTTP client");
+        if let Some(user_agent) = user_agent {
+            let mut headers = HeaderMap::new();
+            if let Ok(value) = HeaderValue::from_str(user_agent) {
+                headers.insert(USER_AGENT, value);
+                builder = builder.default_headers(headers);
+            }
+        }
+
+        builder.build().expect("Failed to create HTTP client")
+    }
+
+    pub fn new() -> Self {
+        let cookie_jar = Arc::new(Jar::default());
+        let client = Self::build_client(cookie_jar.clone(), None);
 
         let requester = Requester {
             client,
+            cookie_jar,
             proxy: false,
             flaresolverr_session: None,
+            user_agent: None,
         };
 
         proxy::init_all_proxies_background(requester.clone());
@@ -53,12 +84,7 @@ impl Requester {
     }
 
     pub async fn get_raw(&mut self, url: &str) -> Result<Response, wreq::Error> {
-        let client = Client::builder()
+        let client = Self::build_client(self.cookie_jar.clone(), self.user_agent.as_deref());
-            .cert_verification(false)
-            .emulation(Emulation::Firefox136)
-            .cookie_store(true)
-            .build()
-            .expect("Failed to create HTTP client");
 
         let mut request = client.get(url).version(Version::HTTP_11);
 
@@ -77,12 +103,7 @@ impl Requester {
         url: &str,
         headers: Vec<(String, String)>,
     ) -> Result<Response, wreq::Error> {
-        let client = Client::builder()
+        let client = Self::build_client(self.cookie_jar.clone(), self.user_agent.as_deref());
-            .cert_verification(false)
-            .emulation(Emulation::Firefox136)
-            .cookie_store(true)
-            .build()
-            .expect("Failed to create HTTP client");
 
         let mut request = client.get(url).version(Version::HTTP_11);
 
@@ -250,31 +271,18 @@ impl Requester {
         // Rebuild client and apply UA/cookies from FlareSolverr
         let cookie_origin = url.split('/').take(3).collect::<Vec<&str>>().join("/");
 
-        self.client = Client::builder()
-            .cert_verification(false)
-            .emulation(Emulation::Firefox136)
-            .cookie_store(true)
-            .redirect(Policy::default())
-            .build()
-            .expect("Failed to create HTTP client");
-
         let useragent = res.solution.userAgent;
-        self.client
+        self.user_agent = Some(useragent);
-            .update()
-            .headers(|headers| {
-                headers.insert("User-Agent", HeaderValue::from_str(&useragent).unwrap());
-            })
-            .apply()
-            .unwrap();
 
-        if let Ok(origin) = url::Url::parse(&cookie_origin) {
+        if url::Url::parse(&cookie_origin).is_ok() {
             for cookie in res.solution.cookies {
-                let header =
+                self.cookie_jar
-                    HeaderValue::from_str(&format!("{}={}", cookie.name, cookie.value)).unwrap();
+                    .add_cookie_str(&format!("{}={}", cookie.name, cookie.value), &cookie_origin);
-                self.client.set_cookie(&origin, header);
             }
         }
 
+        self.client = Self::build_client(self.cookie_jar.clone(), self.user_agent.as_deref());
+
         // Retry the original URL with the updated client & (optional) proxy
         let mut request = self.client.get(url).version(Version::HTTP_11);
         for (key, value) in headers.iter() {