use serde::Serialize; use wreq::multipart::Form; use std::env; use wreq::Client; use wreq::Proxy; use wreq::Response; use wreq::Version; use wreq::header::HeaderValue; use wreq::redirect::Policy; use wreq_util::Emulation; use rand::seq::SliceRandom; use crate::util::proxy; use crate::util::flaresolverr::FlareSolverrRequest; use crate::util::flaresolverr::Flaresolverr; // A Send + Sync error type for all async paths type AnyErr = Box; #[derive(serde::Serialize, serde::Deserialize, Debug, Clone)] pub struct Requester { #[serde(skip)] client: Client, proxy: bool, flaresolverr_session: Option, use_random_proxy: bool, } impl Requester { pub fn new() -> Self { let client = Client::builder() .cert_verification(false) .emulation(Emulation::Firefox136) .cookie_store(true) .redirect(Policy::default()) .build() .expect("Failed to create HTTP client"); let requester = Requester { client, proxy: false, flaresolverr_session: None, use_random_proxy: false, }; proxy::init_all_proxies_background(requester.clone()); requester } pub fn set_proxy(&mut self, proxy: bool) { if proxy { println!("Proxy enabled"); } self.proxy = proxy; } pub fn set_random_proxy(&mut self, random: bool) { self.use_random_proxy = random; } pub async fn get_raw(&mut self, url: &str) -> Result { let client = Client::builder() .cert_verification(false) .emulation(Emulation::Firefox136) .cookie_store(true) .build() .expect("Failed to create HTTP client"); let mut request = client.get(url).version(Version::HTTP_11); if self.proxy { if let Ok(proxy_url) = env::var("BURP_URL") { let proxy = Proxy::all(&proxy_url).unwrap(); request = request.proxy(proxy); } } request.send().await } pub async fn get_raw_with_proxy( &mut self, url: &str, proxy: Proxy, ) -> Result { let client = Client::builder() .cert_verification(false) .emulation(Emulation::Firefox136) .cookie_store(true) .build() .expect("Failed to create HTTP client"); client .get(url) .version(Version::HTTP_11) .proxy(proxy) .send() .await } pub async fn get_raw_with_headers( &mut self, url: &str, headers: Vec<(String, String)>, ) -> Result { let client = Client::builder() .cert_verification(false) .emulation(Emulation::Firefox136) .cookie_store(true) .build() .expect("Failed to create HTTP client"); let mut request = client.get(url).version(Version::HTTP_11); if self.proxy { if let Ok(proxy_url) = env::var("BURP_URL") { let proxy = Proxy::all(&proxy_url).unwrap(); request = request.proxy(proxy); } } // Set custom headers for (key, value) in headers.iter() { request = request.header(key, value); } request.send().await } pub async fn post_json( &mut self, url: &str, data: &S, headers: Vec<(String, String)>, ) -> Result where S: Serialize + ?Sized, { let mut request = self.client.post(url).version(Version::HTTP_11).json(data); // Set custom headers for (key, value) in headers.iter() { request = request.header(key, value); } if self.proxy { if let Ok(proxy_url) = env::var("BURP_URL") { let proxy = Proxy::all(&proxy_url).unwrap(); request = request.proxy(proxy); } } request.send().await } pub async fn post( &mut self, url: &str, data: &str, headers: Vec<(&str, &str)>, ) -> Result { let mut request = self.client.post(url).version(Version::HTTP_11).body(data.to_string()); // Set custom headers for (key, value) in headers.iter() { request = request.header(key.to_string(), value.to_string()); } if self.proxy { if let Ok(proxy_url) = env::var("BURP_URL") { let proxy = Proxy::all(&proxy_url).unwrap(); request = request.proxy(proxy); } } request.send().await } pub async fn post_multipart( &mut self, url: &str, form: Form, headers: Vec<(String, String)>, _http_version: Option, ) -> Result { let http_version = match _http_version { Some(v) => v, None => Version::HTTP_11, }; let mut request = self.client.post(url).multipart(form).version(http_version); // Set custom headers for (key, value) in headers.iter() { request = request.header(key, value); } if self.proxy { if let Ok(proxy_url) = env::var("BURP_URL") { let proxy = Proxy::all(&proxy_url).unwrap(); request = request.proxy(proxy); } } request.send().await } pub async fn get(&mut self, url: &str, _http_version: Option) -> Result { let http_version = match _http_version { Some(v) => v, None => Version::HTTP_11, }; loop { let mut request = self.client.get(url).version(http_version); if self.proxy { if let Ok(proxy_url) = env::var("BURP_URL") { let proxy = Proxy::all(&proxy_url).unwrap(); request = request.proxy(proxy); } } // else if self.use_random_proxy { // let proxies = proxy::all_proxies_snapshot().await.unwrap_or_default(); // if !proxies.is_empty() { // let mut random_proxy = proxies.choose_mut(&mut rand::thread_rng()).unwrap().clone(); // request = request.proxy(random_proxy); // } // } let response = request.send().await?; if response.status().is_success() || response.status().as_u16() == 404 { return Ok(response.text().await?); } if response.status().as_u16() == 429 { tokio::time::sleep(std::time::Duration::from_secs(1)).await; continue; } else { println!( "Direct request to {} failed with status: {}", url, response.status() ); break; } } // If direct request failed, try FlareSolverr. Map its error to a Send+Sync error immediately, // so no non-Send error value lives across later `.await`s. let flare_url = env::var("FLARE_URL").expect("FLARE_URL not set"); let mut flare = Flaresolverr::new(flare_url); if self.proxy && env::var("BURP_URL").is_ok() { flare.set_proxy(true); } let res = flare .solve(FlareSolverrRequest { cmd: "request.get".to_string(), url: url.to_string(), maxTimeout: 60000, }) .await .map_err(|e| -> AnyErr { format!("Failed to solve FlareSolverr: {e}").into() })?; // Rebuild client and apply UA/cookies from FlareSolverr let cookie_origin = url.split('/').take(3).collect::>().join("/"); self.client = Client::builder() .cert_verification(false) .emulation(Emulation::Firefox136) .cookie_store(true) .redirect(Policy::default()) .build() .expect("Failed to create HTTP client"); let useragent = res.solution.userAgent; self.client .update() .headers(|headers| { headers.insert("User-Agent", HeaderValue::from_str(&useragent).unwrap()); }) .apply() .unwrap(); if let Ok(origin) = url::Url::parse(&cookie_origin) { for cookie in res.solution.cookies { let header = HeaderValue::from_str(&format!("{}={}", cookie.name, cookie.value)).unwrap(); self.client.set_cookie(&origin, header); } } // Retry the original URL with the updated client & (optional) proxy let mut request = self.client.get(url).version(Version::HTTP_11); if self.proxy { if let Ok(proxy_url) = env::var("BURP_URL") { let proxy = Proxy::all(&proxy_url).unwrap(); request = request.proxy(proxy); } } let response = request.send().await?; if response.status().is_success() { return Ok(response.text().await?); } // Fall back to FlareSolverr-provided body Ok(res.solution.response) } }