spankbang fix
This commit is contained in:
Simon
@@ -5,11 +5,13 @@ use crate::status::*;
|
||||
use crate::util::cache::VideoCache;
|
||||
use crate::util::parse_abbreviated_number;
|
||||
use crate::util::time::parse_time_to_seconds;
|
||||
use crate::videos::{ServerOptions, VideoItem};
|
||||
use crate::videos::{ServerOptions, VideoFormat, VideoItem};
|
||||
use async_trait::async_trait;
|
||||
use error_chain::error_chain;
|
||||
use htmlentity::entity::{ICodedDataTrait, decode};
|
||||
use scraper::{ElementRef, Html, Selector};
|
||||
use std::process::Command;
|
||||
use std::time::Duration;
|
||||
use url::form_urlencoded::byte_serialize;
|
||||
|
||||
pub const CHANNEL_METADATA: crate::providers::ProviderChannelMetadata =
|
||||
@@ -144,6 +146,177 @@ impl SpankbangProvider {
|
||||
vec![("Referer".to_string(), format!("{}/", self.url))]
|
||||
}
|
||||
|
||||
fn is_cloudflare_block(text: &str) -> bool {
|
||||
let lowercase = text.to_ascii_lowercase();
|
||||
lowercase.contains("attention required")
|
||||
|| lowercase.contains("you have been blocked")
|
||||
|| lowercase.contains("cloudflare ray id")
|
||||
}
|
||||
|
||||
fn fallback_items_from_ytdlp(&self, page_url: &str, limit: usize) -> Vec<VideoItem> {
|
||||
let output = match Command::new("yt-dlp")
|
||||
.arg("-J")
|
||||
.arg("--flat-playlist")
|
||||
.arg("--extractor-args")
|
||||
.arg("generic:impersonate=chrome")
|
||||
.arg(page_url)
|
||||
.output()
|
||||
{
|
||||
Ok(output) if output.status.success() => output,
|
||||
_ => return vec![],
|
||||
};
|
||||
|
||||
let payload: serde_json::Value = match serde_json::from_slice(&output.stdout) {
|
||||
Ok(payload) => payload,
|
||||
Err(_) => return vec![],
|
||||
};
|
||||
|
||||
let entries = match payload.get("entries").and_then(|value| value.as_array()) {
|
||||
Some(entries) => entries,
|
||||
None => return vec![],
|
||||
};
|
||||
|
||||
let mut items = Vec::new();
|
||||
for (index, entry) in entries.iter().take(limit).enumerate() {
|
||||
let Some(url) = entry.get("url").and_then(|value| value.as_str()) else {
|
||||
continue;
|
||||
};
|
||||
if !(url.starts_with("https://") || url.starts_with("http://")) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let id = entry
|
||||
.get("id")
|
||||
.and_then(|value| value.as_str())
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(ToOwned::to_owned)
|
||||
.unwrap_or_else(|| format!("spankbang-fallback-{}", index + 1));
|
||||
let title = entry
|
||||
.get("title")
|
||||
.and_then(|value| value.as_str())
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(Self::decode_html)
|
||||
.unwrap_or_else(|| format!("SpankBang Video {}", index + 1));
|
||||
let thumb = entry
|
||||
.get("thumbnail")
|
||||
.and_then(|value| value.as_str())
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
let duration = entry
|
||||
.get("duration")
|
||||
.and_then(|value| value.as_u64())
|
||||
.and_then(|value| u32::try_from(value).ok())
|
||||
.unwrap_or(0);
|
||||
|
||||
let format_kind = if url.contains(".m3u8") {
|
||||
"m3u8"
|
||||
} else {
|
||||
"video/mp4"
|
||||
};
|
||||
let mut format = VideoFormat::new(url.to_string(), "auto".to_string(), format_kind.to_string());
|
||||
if let Some(headers) = entry.get("http_headers").and_then(|value| value.as_object()) {
|
||||
for (key, value) in headers {
|
||||
if let Some(value) = value.as_str() {
|
||||
format.add_http_header(key.to_string(), value.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
if entry
|
||||
.get("http_headers")
|
||||
.and_then(|value| value.as_object())
|
||||
.is_none()
|
||||
{
|
||||
format.add_http_header("Referer".to_string(), format!("{}/", self.url));
|
||||
}
|
||||
|
||||
let mut item = VideoItem::new(
|
||||
id,
|
||||
title,
|
||||
url.to_string(),
|
||||
"spankbang".to_string(),
|
||||
thumb,
|
||||
duration,
|
||||
)
|
||||
.formats(vec![format]);
|
||||
|
||||
if let Some(views) = entry
|
||||
.get("view_count")
|
||||
.and_then(|value| value.as_u64())
|
||||
.and_then(|value| u32::try_from(value).ok())
|
||||
{
|
||||
item = item.views(views);
|
||||
}
|
||||
if let Some(uploader) = entry
|
||||
.get("uploader")
|
||||
.and_then(|value| value.as_str())
|
||||
.filter(|value| !value.is_empty())
|
||||
{
|
||||
item = item.uploader(uploader.to_string());
|
||||
}
|
||||
items.push(item);
|
||||
}
|
||||
|
||||
items
|
||||
}
|
||||
|
||||
async fn fallback_items_with_working_media(
|
||||
&self,
|
||||
page_url: &str,
|
||||
options: &ServerOptions,
|
||||
) -> Vec<VideoItem> {
|
||||
let fallback_items = self.fallback_items_from_ytdlp(page_url, 72);
|
||||
if fallback_items.is_empty() {
|
||||
return vec![];
|
||||
}
|
||||
|
||||
let mut requester = requester_or_default(
|
||||
options,
|
||||
"spankbang",
|
||||
"spankbang.fallback_items_with_working_media.missing_requester",
|
||||
);
|
||||
let mut working_items = Vec::new();
|
||||
|
||||
for item in fallback_items {
|
||||
let format_headers = item
|
||||
.formats
|
||||
.as_ref()
|
||||
.and_then(|formats| formats.first())
|
||||
.map(|format| format.http_headers_pairs())
|
||||
.unwrap_or_default();
|
||||
let media_url = item
|
||||
.formats
|
||||
.as_ref()
|
||||
.and_then(|formats| formats.first())
|
||||
.map(|format| format.url.clone())
|
||||
.unwrap_or_else(|| item.url.clone());
|
||||
if media_url.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut headers = format_headers;
|
||||
if !headers
|
||||
.iter()
|
||||
.any(|(key, _)| key.eq_ignore_ascii_case("range"))
|
||||
{
|
||||
headers.push(("Range".to_string(), "bytes=0-2047".to_string()));
|
||||
}
|
||||
|
||||
let is_working = match requester
|
||||
.get_raw_with_headers_timeout(&media_url, headers, Some(Duration::from_secs(20)))
|
||||
.await
|
||||
{
|
||||
Ok(response) => response.status().is_success(),
|
||||
Err(_) => false,
|
||||
};
|
||||
|
||||
if is_working {
|
||||
working_items.push(item);
|
||||
}
|
||||
}
|
||||
|
||||
working_items
|
||||
}
|
||||
|
||||
fn build_query_url(&self, query: &str, page: u32, sort: &str) -> String {
|
||||
let encoded_query = Self::encode_search_query(query);
|
||||
let mut url = if page > 1 {
|
||||
@@ -432,6 +605,14 @@ impl SpankbangProvider {
|
||||
&format!("url={video_url}; error={e}"),
|
||||
)
|
||||
.await;
|
||||
let fallback_items = self
|
||||
.fallback_items_with_working_media(&video_url, &options)
|
||||
.await;
|
||||
if !fallback_items.is_empty() {
|
||||
cache.remove(&video_url);
|
||||
cache.insert(video_url.clone(), fallback_items.clone());
|
||||
return Ok(fallback_items);
|
||||
}
|
||||
return Ok(old_items);
|
||||
}
|
||||
};
|
||||
@@ -443,6 +624,32 @@ impl SpankbangProvider {
|
||||
&format!("url={video_url}"),
|
||||
)
|
||||
.await;
|
||||
let fallback_items = self
|
||||
.fallback_items_with_working_media(&video_url, &options)
|
||||
.await;
|
||||
if !fallback_items.is_empty() {
|
||||
cache.remove(&video_url);
|
||||
cache.insert(video_url.clone(), fallback_items.clone());
|
||||
return Ok(fallback_items);
|
||||
}
|
||||
return Ok(old_items);
|
||||
}
|
||||
|
||||
if Self::is_cloudflare_block(&text) {
|
||||
report_provider_error(
|
||||
"spankbang",
|
||||
"get.cloudflare_block",
|
||||
&format!("url={video_url}"),
|
||||
)
|
||||
.await;
|
||||
let fallback_items = self
|
||||
.fallback_items_with_working_media(&video_url, &options)
|
||||
.await;
|
||||
if !fallback_items.is_empty() {
|
||||
cache.remove(&video_url);
|
||||
cache.insert(video_url.clone(), fallback_items.clone());
|
||||
return Ok(fallback_items);
|
||||
}
|
||||
return Ok(old_items);
|
||||
}
|
||||
|
||||
@@ -490,6 +697,14 @@ impl SpankbangProvider {
|
||||
&format!("url={video_url}; error={e}"),
|
||||
)
|
||||
.await;
|
||||
let fallback_items = self
|
||||
.fallback_items_with_working_media(&video_url, &options)
|
||||
.await;
|
||||
if !fallback_items.is_empty() {
|
||||
cache.remove(&video_url);
|
||||
cache.insert(video_url.clone(), fallback_items.clone());
|
||||
return Ok(fallback_items);
|
||||
}
|
||||
return Ok(old_items);
|
||||
}
|
||||
};
|
||||
@@ -501,6 +716,32 @@ impl SpankbangProvider {
|
||||
&format!("url={video_url}"),
|
||||
)
|
||||
.await;
|
||||
let fallback_items = self
|
||||
.fallback_items_with_working_media(&video_url, &options)
|
||||
.await;
|
||||
if !fallback_items.is_empty() {
|
||||
cache.remove(&video_url);
|
||||
cache.insert(video_url.clone(), fallback_items.clone());
|
||||
return Ok(fallback_items);
|
||||
}
|
||||
return Ok(old_items);
|
||||
}
|
||||
|
||||
if Self::is_cloudflare_block(&text) {
|
||||
report_provider_error(
|
||||
"spankbang",
|
||||
"query.cloudflare_block",
|
||||
&format!("url={video_url}"),
|
||||
)
|
||||
.await;
|
||||
let fallback_items = self
|
||||
.fallback_items_with_working_media(&video_url, &options)
|
||||
.await;
|
||||
if !fallback_items.is_empty() {
|
||||
cache.remove(&video_url);
|
||||
cache.insert(video_url.clone(), fallback_items.clone());
|
||||
return Ok(fallback_items);
|
||||
}
|
||||
return Ok(old_items);
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user