simon/hottub
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

proxy pool: spread rule34video enrichment over checked public proxies

Browse Source 
Rework the unused ALL_PROXIES list into util/proxy_pool.rs:
- hourly sweep of the public lists, TLS-verified cloudflare trace checks,
  >=2 successes with a median under 1.5s, one proxy per exit IP; proxies
  that show our own IP are dropped, two failures in a row evict
- per-site sets checked against a site URL (rule34video: robots.txt),
  round-robin, 10-min per-site cooldown on 429/challenge pages, live
  request outcomes fed back
- started from main (PROXY_POOL=0 disables; PROXY_POOL_LISTS,
  PROXY_POOL_CONCURRENCY), /api/proxies shows its status without credentials
- wreq "socks" feature enabled: SOCKS proxies silently failed before

rule34video enrichment is back, through the pool: full detail pages
(1080p..360p formats, tags, artist), 8 concurrent, one 4s deadline per
listing; late pages keep running and land in the 2h detail cache, which
cached listings are re-merged with on every serve. No ready proxies means
no enrichment (no direct fallback). Requester::get_via sends a pooled
request with certificate checks, sharing the session's cookie jar.

Also adds an ignored live probe showing /embed/ shares the 429 limit.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01AEp724zDpCWNaSeCrMSvjx
This commit is contained in:
Simon
2026-11-05 21:59:28 +00:00
parent 4f83dae3fa
commit 29d526b5fc
11 changed files with 1204 additions and 211 deletions
Download Patch File Download Diff File Expand all files Collapse all files

181
src/util/proxy.rs
View File

@@ -1,26 +1,9 @@
use std::fmt;
use std::str::FromStr;
use std::sync::Arc;
use serde::Serialize;
use tokio::sync::{OnceCell, RwLock};
use tokio::time::{self, Duration};
use url::Url;
use wreq::Proxy as WreqProxy;
use wreq::{Client, Version};
use crate::util::requester::Requester;
pub static ALL_PROXIES: OnceCell<Arc<RwLock<Vec<Proxy>>>> = OnceCell::const_new();
static ALL_PROXY_KEYS: OnceCell<Arc<RwLock<std::collections::HashSet<String>>>> =
OnceCell::const_new();
const PROXY_LIST: [&str; 3] = [
"https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/socks5.txt",
"https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/http.txt",
"https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/socks4.txt",
];
const IFCONFIG_URL: &str = "https://ifconfig.co";
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct Proxy {
@@ -44,12 +27,6 @@ impl ProxyParseError {
}
}
pub async fn all_proxies_snapshot() -> Option<Vec<Proxy>> {
let handle = ALL_PROXIES.get()?.clone();
let proxies = handle.read().await;
Some(proxies.clone())
}
impl fmt::Display for ProxyParseError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{}", self.message)
@@ -78,6 +55,11 @@ impl fmt::Display for Proxy {
}
impl Proxy {
/// `protocol://host:port`, without credentials: safe to show or log.
pub fn redacted(&self) -> String {
format!("{}://{}:{}", self.protocol, self.host, self.port)
}
pub fn parse(input: &str) -> Result<Self, ProxyParseError> {
input.parse()
}
@@ -99,147 +81,6 @@ impl Proxy {
}
}
pub async fn fetch_proxies(
requester: &mut Requester,
proxy_list_url: &str,
proxy_sink: Arc<RwLock<Vec<Proxy>>>,
) -> Result<usize, ProxyParseError> {
let default_protocol = protocol_from_list_url(proxy_list_url)?;
let body = requester
.get(proxy_list_url, None)
.await
.map_err(|e| ProxyParseError::new(format!("failed to fetch proxy list: {e}")))?;
let mut tasks = tokio::task::JoinSet::new();
for (line_index, line) in body.lines().enumerate() {
let trimmed = line.trim();
if trimmed.is_empty() {
continue;
}
let proxy_input = if trimmed.contains("://") {
trimmed.to_string()
} else {
format!("{default_protocol}://{trimmed}")
};
let proxy = Proxy::parse(&proxy_input).map_err(|e| {
ProxyParseError::new(format!("invalid proxy on line {}: {e}", line_index + 1))
})?;
tasks.spawn(async move { verify_proxy(proxy).await });
}
let mut added = 0usize;
while let Some(result) = tasks.join_next().await {
match result {
Ok(Ok(proxy)) => {
if let Some(proxy_keys) = ALL_PROXY_KEYS.get() {
let key = proxy_key(&proxy);
let mut keys = proxy_keys.write().await;
if !keys.insert(key) {
continue;
}
let mut proxies = proxy_sink.write().await;
proxies.push(proxy);
added += 1;
}
}
Ok(Err(_err)) => {
// eprintln!("Proxy verification failed: {err}");
}
Err(err) => {
eprintln!("Proxy verification task failed: {err}");
}
}
}
Ok(added)
}
fn protocol_from_list_url(list_url: &str) -> Result<&'static str, ProxyParseError> {
if list_url.contains("http.txt") {
Ok("http")
} else if list_url.contains("socks4.txt") {
Ok("socks4")
} else if list_url.contains("socks5.txt") {
Ok("socks5")
} else {
Err(ProxyParseError::new(format!(
"unknown proxy list protocol for url: {list_url}"
)))
}
}
async fn verify_proxy(proxy: Proxy) -> Result<Proxy, ProxyParseError> {
let wreq_proxy = proxy.to_wreq_proxy()?;
let client = Client::builder()
.tls_cert_verification(false)
.build()
.map_err(|e| ProxyParseError::new(format!("failed to build http client: {e}")))?;
let response = client
.get(IFCONFIG_URL)
.version(Version::HTTP_11)
.proxy(wreq_proxy)
.timeout(Duration::from_secs(5))
.send()
.await
.map_err(|e| ProxyParseError::new(format!("proxy request failed: {e}")))?;
if response.status().is_success() {
Ok(proxy)
} else {
Err(ProxyParseError::new(format!(
"proxy returned status {}",
response.status()
)))
}
}
pub fn init_all_proxies_background(requester: Requester) {
if ALL_PROXIES.get().is_some() {
return;
}
if tokio::runtime::Handle::try_current().is_err() {
eprintln!("Skipping proxy list init: no Tokio runtime available");
return;
}
let proxy_cache = Arc::new(RwLock::new(Vec::new()));
let proxy_keys = Arc::new(RwLock::new(std::collections::HashSet::new()));
if ALL_PROXIES.set(proxy_cache.clone()).is_err() {
return;
}
let _ = ALL_PROXY_KEYS.set(proxy_keys);
tokio::spawn(async move {
let mut interval = time::interval(Duration::from_secs(60 * 60));
loop {
let mut tasks = tokio::task::JoinSet::new();
for list in PROXY_LIST {
let proxy_cache = proxy_cache.clone();
let mut requester = requester.clone();
tasks.spawn(async move { fetch_proxies(&mut requester, list, proxy_cache).await });
}
while let Some(result) = tasks.join_next().await {
match result {
Ok(Ok(_added)) => {}
Ok(Err(err)) => {
eprintln!("Failed to fetch proxy list: {err}");
}
Err(err) => {
eprintln!("Proxy list task failed: {err}");
}
}
}
interval.tick().await;
}
});
}
impl FromStr for Proxy {
type Err = ProxyParseError;
@@ -261,7 +102,6 @@ impl FromStr for Proxy {
|| url.query().is_some()
|| url.fragment().is_some()
{
println!("Parsed proxy URL: {:?}", url);
return Err(ProxyParseError::new(format!(
"proxy url must not include path, query, or fragment: {:?}",
input
@@ -287,14 +127,3 @@ impl FromStr for Proxy {
})
}
}
fn proxy_key(proxy: &Proxy) -> String {
format!(
"{}://{}:{}@{}:{}",
proxy.protocol,
proxy.username.as_deref().unwrap_or(""),
proxy.password.as_deref().unwrap_or(""),
proxy.host,
proxy.port
)
}