noodlemagazine upgrade

src/util/requester.rs

@@ -136,6 +136,42 @@ impl Requester {
             .unwrap_or_else(|| "none".to_string())
     }
 
+    fn jina_mirror_url(url: &str) -> Option<String> {
+        if url.trim().is_empty() {
+            return None;
+        }
+        if url.starts_with("https://r.jina.ai/") || url.starts_with("http://r.jina.ai/") {
+            return Some(url.to_string());
+        }
+        let stripped = url
+            .strip_prefix("https://")
+            .or_else(|| url.strip_prefix("http://"))
+            .unwrap_or(url);
+        Some(format!("https://r.jina.ai/http://{stripped}"))
+    }
+
+    async fn fetch_jina_mirror_body(
+        cookie_jar: Arc<Jar>,
+        user_agent: Option<String>,
+        proxy_enabled: bool,
+        url: &str,
+    ) -> Result<String, AnyErr> {
+        let mirror_url = Self::jina_mirror_url(url).ok_or("invalid mirror url")?;
+        let client = Self::build_client(cookie_jar, user_agent.as_deref());
+        let mut request = client.get(&mirror_url).version(Version::HTTP_11);
+        if proxy_enabled {
+            if let Ok(proxy_url) = env::var("BURP_URL") {
+                let proxy = Proxy::all(&proxy_url)?;
+                request = request.proxy(proxy);
+            }
+        }
+        let response = request.send().await?;
+        if !response.status().is_success() {
+            return Err(format!("jina mirror returned status {}", response.status()).into());
+        }
+        Ok(response.text().await?)
+    }
+
     #[cfg(any(not(hottub_single_provider), hottub_provider = "hypnotube"))]
     fn debug_cookie_preview_from_borrowed_headers(
         &self,
@@ -453,6 +489,14 @@ impl Requester {
             Some(v) => v,
             None => Version::HTTP_11,
         };
+        let cookie_jar = self.cookie_jar.clone();
+        let user_agent = self.user_agent.clone();
+        let proxy_enabled = self.proxy;
+        let _trace_id = self.debug_trace_id.as_deref().unwrap_or("none").to_string();
+        let url_owned = url.to_string();
+        let jina_handle = tokio::spawn(async move {
+            Self::fetch_jina_mirror_body(cookie_jar, user_agent, proxy_enabled, &url_owned).await
+        });
         loop {
             let mut request = self.client.get(url).version(http_version);
             for (key, value) in headers.iter() {
@@ -464,7 +508,38 @@ impl Requester {
                     request = request.proxy(proxy);
                 }
             }
-            let response = request.send().await?;
+            let response = match request.send().await {
+                Ok(response) => response,
+                Err(error) => {
+                    crate::flow_debug!(
+                        "trace={} requester direct transport failed url={} error={} using_jina_fallback=true",
+                        _trace_id,
+                        crate::util::flow_debug::preview(url, 120),
+                        crate::util::flow_debug::preview(&error.to_string(), 160)
+                    );
+                    match jina_handle.await {
+                        Ok(Ok(body)) => return Ok(body),
+                        Ok(Err(_jina_error)) => {
+                            crate::flow_debug!(
+                                "trace={} requester jina fallback failed after transport error url={} error={}",
+                                _trace_id,
+                                crate::util::flow_debug::preview(url, 120),
+                                crate::util::flow_debug::preview(&_jina_error.to_string(), 160)
+                            );
+                            return Err(error.into());
+                        }
+                        Err(_join_error) => {
+                            crate::flow_debug!(
+                                "trace={} requester jina task join failed after transport error url={} error={}",
+                                _trace_id,
+                                crate::util::flow_debug::preview(url, 120),
+                                crate::util::flow_debug::preview(&_join_error.to_string(), 160)
+                            );
+                            return Err(error.into());
+                        }
+                    }
+                }
+            };
             self.store_response_cookies(url, &response);
             crate::flow_debug!(
                 "trace={} requester direct response url={} status={}",
@@ -473,6 +548,7 @@ impl Requester {
                 response.status()
             );
             if response.status().is_success() || response.status().as_u16() == 404 {
+                jina_handle.abort();
                 return Ok(response.text().await?);
             }
             if response.status().as_u16() == 429 {
@@ -484,11 +560,31 @@ impl Requester {
                 tokio::time::sleep(std::time::Duration::from_secs(1)).await;
                 continue;
             } else {
-                println!(
-                    "Direct request to {} failed with status: {}",
-                    url,
+                crate::flow_debug!(
+                    "trace={} requester direct failed url={} status={} using_jina_fallback=true",
+                    _trace_id,
+                    crate::util::flow_debug::preview(url, 120),
                     response.status()
                 );
+                match jina_handle.await {
+                    Ok(Ok(body)) => return Ok(body),
+                    Ok(Err(_error)) => {
+                        crate::flow_debug!(
+                            "trace={} requester jina fallback failed url={} error={}",
+                            _trace_id,
+                            crate::util::flow_debug::preview(url, 120),
+                            crate::util::flow_debug::preview(&_error.to_string(), 160)
+                        );
+                    }
+                    Err(_error) => {
+                        crate::flow_debug!(
+                            "trace={} requester jina task join failed url={} error={}",
+                            _trace_id,
+                            crate::util::flow_debug::preview(url, 120),
+                            crate::util::flow_debug::preview(&_error.to_string(), 160)
+                        );
+                    }
+                }
                 break;
             }
         }