some upgrades

src/util/requester.rs

@@ -8,7 +8,9 @@ use wreq::Version;
 use wreq::header::HeaderValue;
 use wreq::redirect::Policy;
 use wreq_util::Emulation;
+use rand::seq::SliceRandom;
 
+use crate::util::proxy;
 use crate::util::flaresolverr::FlareSolverrRequest;
 use crate::util::flaresolverr::Flaresolverr;
 
@@ -21,6 +23,7 @@ pub struct Requester {
     client: Client,
     proxy: bool,
     flaresolverr_session: Option<String>,
+    use_random_proxy: bool,
 }
 
 impl Requester {
@@ -33,11 +36,16 @@ impl Requester {
             .build()
             .expect("Failed to create HTTP client");
 
-        Requester {
+        let requester = Requester {
             client,
             proxy: false,
             flaresolverr_session: None,
-        }
+            use_random_proxy: false,
+        };
+
+        proxy::init_all_proxies_background(requester.clone());
+
+        requester
     }
 
     pub fn set_proxy(&mut self, proxy: bool) {
@@ -47,6 +55,10 @@ impl Requester {
         self.proxy = proxy;
     }
 
+    pub fn set_random_proxy(&mut self, random: bool) {
+        self.use_random_proxy = random;
+    }
+
     pub async fn get_raw(&mut self, url: &str) -> Result<Response, wreq::Error> {
         let client = Client::builder()
             .cert_verification(false)
@@ -67,6 +79,26 @@ impl Requester {
         request.send().await
     }
 
+    pub async fn get_raw_with_proxy(
+        &mut self,
+        url: &str,
+        proxy: Proxy,
+    ) -> Result<Response, wreq::Error> {
+        let client = Client::builder()
+            .cert_verification(false)
+            .emulation(Emulation::Firefox136)
+            .cookie_store(true)
+            .build()
+            .expect("Failed to create HTTP client");
+
+        client
+            .get(url)
+            .version(Version::HTTP_11)
+            .proxy(proxy)
+            .send()
+            .await
+    }
+
     pub async fn get_raw_with_headers(
         &mut self,
         url: &str,
@@ -185,7 +217,14 @@ pub async fn post_json<S>(
                     let proxy = Proxy::all(&proxy_url).unwrap();
                     request = request.proxy(proxy);
                 }
-            }
+            } 
+            // else if self.use_random_proxy {
+            //     let proxies = proxy::all_proxies_snapshot().await.unwrap_or_default();
+            //     if !proxies.is_empty() {
+            //         let mut random_proxy = proxies.choose_mut(&mut rand::thread_rng()).unwrap().clone();
+            //         request = request.proxy(random_proxy);
+            //     }
+            // }
             let response = request.send().await?;
             if response.status().is_success() || response.status().as_u16() == 404 {
                 return Ok(response.text().await?);