youtube: instrument the extractor with diagnostic logging
Some checks failed
gitleaks / scan (push) Failing after 4s
Some checks failed
gitleaks / scan (push) Failing after 4s
The crate had zero logging — every failure, fallback, cache reset, and empty-parse was silent by construction, which made the 2026 bot-wall outage nearly undiagnosable. Add `log = "0.4"` and 52 statements across the extractor following a one-INFO-per-outcome / WARN-on-every-failure / DEBUG-for-internals policy, so a future break shows up in a log grep instead of a debugger. Highlights: nsig->throttled-URL fallback (aggregated one WARN per extraction, never per-format, via a threaded UrlProcessStats counter); the nsig identity-output trap now returns DeobfError::NsigIdentity and is NOT cached (previously it cached and permanently throttled); player.js build/eval/install/ StillBad lifecycle; the ANDROID->VISIONOS cascade outcome; visitorData decline + reset; HTTP 429 bot-flag; channel/search layout-change empties; per-request DEBUG (query stripped). Plus a `BotDetected` error variant (Display byte- identical to the old string) so the wall is greppable. No secrets: only videoId/channel ids, error Displays (already URL/token-scrubbed at the exceptions.rs choke points), query-stripped endpoints/player.js URLs, counts, and lengths are logged — never token/poToken/visitorData/signature values or response bodies.
This commit is contained in:
parent
dd964407cc
commit
6df61c1f52
14 changed files with 559 additions and 70 deletions
1
Cargo.lock
generated
1
Cargo.lock
generated
|
|
@ -988,6 +988,7 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596"
|
||||||
name = "strawcore-core"
|
name = "strawcore-core"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
|
"log",
|
||||||
"once_cell",
|
"once_cell",
|
||||||
"parking_lot",
|
"parking_lot",
|
||||||
"regex",
|
"regex",
|
||||||
|
|
|
||||||
|
|
@ -19,6 +19,7 @@ reqwest = { version = "0.12", default-features = false, features = ["rustls-tls-
|
||||||
serde = { version = "1", features = ["derive"] }
|
serde = { version = "1", features = ["derive"] }
|
||||||
serde_json = "1"
|
serde_json = "1"
|
||||||
thiserror = "1"
|
thiserror = "1"
|
||||||
|
log = "0.4"
|
||||||
parking_lot = "0.12"
|
parking_lot = "0.12"
|
||||||
url = "2"
|
url = "2"
|
||||||
once_cell = "1"
|
once_cell = "1"
|
||||||
|
|
|
||||||
|
|
@ -9,7 +9,7 @@
|
||||||
// as Ok(Response)
|
// as Ok(Response)
|
||||||
|
|
||||||
use std::io::Read;
|
use std::io::Read;
|
||||||
use std::time::Duration;
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
use reqwest::blocking::Client;
|
use reqwest::blocking::Client;
|
||||||
use reqwest::redirect::Policy;
|
use reqwest::redirect::Policy;
|
||||||
|
|
@ -52,6 +52,11 @@ impl ReqwestDownloader {
|
||||||
|
|
||||||
impl Downloader for ReqwestDownloader {
|
impl Downloader for ReqwestDownloader {
|
||||||
fn execute(&self, request: Request) -> Result<Response, NetworkError> {
|
fn execute(&self, request: Request) -> Result<Response, NetworkError> {
|
||||||
|
let started = Instant::now();
|
||||||
|
let method_str = match request.method() {
|
||||||
|
Method::Get => "GET",
|
||||||
|
Method::Post => "POST",
|
||||||
|
};
|
||||||
let method = match request.method() {
|
let method = match request.method() {
|
||||||
Method::Get => reqwest::Method::GET,
|
Method::Get => reqwest::Method::GET,
|
||||||
Method::Post => reqwest::Method::POST,
|
Method::Post => reqwest::Method::POST,
|
||||||
|
|
@ -79,6 +84,12 @@ impl Downloader for ReqwestDownloader {
|
||||||
|
|
||||||
let status = resp.status();
|
let status = resp.status();
|
||||||
let url_after_redirects = resp.url().to_string();
|
let url_after_redirects = resp.url().to_string();
|
||||||
|
// Query/fragment carry n/sig/pot/cpn/id — strip for every log line.
|
||||||
|
let endpoint = url_after_redirects
|
||||||
|
.split(['?', '#'])
|
||||||
|
.next()
|
||||||
|
.unwrap_or(&url_after_redirects)
|
||||||
|
.to_string();
|
||||||
|
|
||||||
if status.as_u16() == 429 {
|
if status.as_u16() == 429 {
|
||||||
// Privacy: this URL ends up in error strings (and from there in
|
// Privacy: this URL ends up in error strings (and from there in
|
||||||
|
|
@ -91,6 +102,8 @@ impl Downloader for ReqwestDownloader {
|
||||||
stripped.set_query(None);
|
stripped.set_query(None);
|
||||||
stripped.set_fragment(None);
|
stripped.set_fragment(None);
|
||||||
stripped.set_path("/");
|
stripped.set_path("/");
|
||||||
|
// Densest bot-flag signal — host only (path/query already stripped).
|
||||||
|
log::warn!("HTTP 429 reCAPTCHA / bot-wall from {stripped} — YouTube is rate-limiting");
|
||||||
return Err(NetworkError::Recaptcha { url: stripped.to_string() });
|
return Err(NetworkError::Recaptcha { url: stripped.to_string() });
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -107,6 +120,9 @@ impl Downloader for ReqwestDownloader {
|
||||||
// Fail fast when a known Content-Length already exceeds the cap.
|
// Fail fast when a known Content-Length already exceeds the cap.
|
||||||
if let Some(len) = resp.content_length() {
|
if let Some(len) = resp.content_length() {
|
||||||
if len > MAX_BODY_BYTES {
|
if len > MAX_BODY_BYTES {
|
||||||
|
log::warn!(
|
||||||
|
"{endpoint}: Content-Length {len}B exceeds cap {MAX_BODY_BYTES}B; aborting"
|
||||||
|
);
|
||||||
return Err(NetworkError::Transport(format!(
|
return Err(NetworkError::Transport(format!(
|
||||||
"response body {len} bytes exceeds cap {MAX_BODY_BYTES}"
|
"response body {len} bytes exceeds cap {MAX_BODY_BYTES}"
|
||||||
)));
|
)));
|
||||||
|
|
@ -123,6 +139,7 @@ impl Downloader for ReqwestDownloader {
|
||||||
.read_to_end(&mut buf)
|
.read_to_end(&mut buf)
|
||||||
.map_err(|e| NetworkError::Transport(format!("body read: {e}")))?;
|
.map_err(|e| NetworkError::Transport(format!("body read: {e}")))?;
|
||||||
if buf.len() as u64 > MAX_BODY_BYTES {
|
if buf.len() as u64 > MAX_BODY_BYTES {
|
||||||
|
log::warn!("{endpoint}: streamed body exceeded cap {MAX_BODY_BYTES}B; aborting");
|
||||||
return Err(NetworkError::Transport(format!(
|
return Err(NetworkError::Transport(format!(
|
||||||
"response body exceeded cap {MAX_BODY_BYTES}"
|
"response body exceeded cap {MAX_BODY_BYTES}"
|
||||||
)));
|
)));
|
||||||
|
|
@ -131,11 +148,19 @@ impl Downloader for ReqwestDownloader {
|
||||||
// common case): String::from_utf8 reinterprets the Vec in place,
|
// common case): String::from_utf8 reinterprets the Vec in place,
|
||||||
// whereas from_utf8_lossy always allocates + copies. Fall back to
|
// whereas from_utf8_lossy always allocates + copies. Fall back to
|
||||||
// lossy only on genuinely invalid bytes, preserving U+FFFD behavior.
|
// lossy only on genuinely invalid bytes, preserving U+FFFD behavior.
|
||||||
|
let body_len = buf.len();
|
||||||
let body = match String::from_utf8(buf) {
|
let body = match String::from_utf8(buf) {
|
||||||
Ok(s) => s,
|
Ok(s) => s,
|
||||||
Err(e) => String::from_utf8_lossy(e.as_bytes()).into_owned(),
|
Err(e) => String::from_utf8_lossy(e.as_bytes()).into_owned(),
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// One DEBUG per request (dogfood ring). Query stripped; body length +
|
||||||
|
// status only — never the body. Gated behind the Debug-level bump.
|
||||||
|
log::debug!(
|
||||||
|
"{method_str} {endpoint} → {code} {body_len}B {}ms",
|
||||||
|
started.elapsed().as_millis()
|
||||||
|
);
|
||||||
|
|
||||||
Ok(Response::new(code, message, headers, body, url_after_redirects))
|
Ok(Response::new(code, message, headers, body, url_after_redirects))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -59,6 +59,13 @@ pub enum ContentUnavailable {
|
||||||
SoundCloudGoPlus,
|
SoundCloudGoPlus,
|
||||||
#[error("account terminated")]
|
#[error("account terminated")]
|
||||||
AccountTerminated,
|
AccountTerminated,
|
||||||
|
/// The 2026 poToken/BotGuard "sign in to confirm you're not a bot" wall.
|
||||||
|
/// Its Display is kept byte-identical to the old
|
||||||
|
/// `Other("sign in to confirm you're not a bot")` so string-matching
|
||||||
|
/// callers keep working; the dedicated variant makes the wall greppable in
|
||||||
|
/// logs and distinctly classifiable in app state.
|
||||||
|
#[error("sign in to confirm you're not a bot")]
|
||||||
|
BotDetected,
|
||||||
#[error("unavailable: {0}")]
|
#[error("unavailable: {0}")]
|
||||||
Other(String),
|
Other(String),
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -46,7 +46,14 @@ pub fn channel_info(identifier: ChannelIdentifier) -> Result<ChannelInfo, Extrac
|
||||||
ChannelIdentifier::Custom(c) => resolve_handle_to_channel_id(&format!("c/{c}"))?,
|
ChannelIdentifier::Custom(c) => resolve_handle_to_channel_id(&format!("c/{c}"))?,
|
||||||
ChannelIdentifier::LegacyUser(u) => resolve_handle_to_channel_id(&format!("user/{u}"))?,
|
ChannelIdentifier::LegacyUser(u) => resolve_handle_to_channel_id(&format!("user/{u}"))?,
|
||||||
};
|
};
|
||||||
fetch_channel_browse(&resolved)
|
let info = fetch_channel_browse(&resolved)?;
|
||||||
|
log::info!(
|
||||||
|
"[channel {}] channel_info ok: videos={} continuation={}",
|
||||||
|
info.channel_id,
|
||||||
|
info.recent_videos.len(),
|
||||||
|
info.videos_continuation.is_some()
|
||||||
|
);
|
||||||
|
Ok(info)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn resolve_handle_to_channel_id(url_fragment: &str) -> Result<String, ExtractionError> {
|
pub fn resolve_handle_to_channel_id(url_fragment: &str) -> Result<String, ExtractionError> {
|
||||||
|
|
@ -113,10 +120,21 @@ pub fn fetch_channel_browse(channel_id: &str) -> Result<ChannelInfo, ExtractionE
|
||||||
|
|
||||||
// Videos tab is best-effort: a fetch error OR a panicked worker thread
|
// Videos tab is best-effort: a fetch error OR a panicked worker thread
|
||||||
// just leaves recent_videos empty (header still populated above).
|
// just leaves recent_videos empty (header still populated above).
|
||||||
if let Ok(Ok(videos_response)) = videos_result {
|
match videos_result {
|
||||||
info.recent_videos = parse_videos_tab(&videos_response);
|
Ok(Ok(videos_response)) => {
|
||||||
if let Some(token) = parse_videos_continuation(&videos_response) {
|
info.recent_videos = parse_videos_tab(&videos_response, channel_id);
|
||||||
info.videos_continuation = Some(token);
|
if let Some(token) = parse_videos_continuation(&videos_response) {
|
||||||
|
info.videos_continuation = Some(token);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(Err(e)) => {
|
||||||
|
log::warn!("[channel {channel_id}] Videos-tab fetch failed ({e}); recent_videos empty (header still populated)");
|
||||||
|
}
|
||||||
|
Err(_) => {
|
||||||
|
// The worker closure can't panic by construction → invariant breach.
|
||||||
|
log::error!(
|
||||||
|
"[channel {channel_id}] Videos-tab worker thread panicked; recent_videos empty"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Ok(info)
|
Ok(info)
|
||||||
|
|
@ -129,7 +147,13 @@ pub fn fetch_channel_browse(channel_id: &str) -> Result<ChannelInfo, ExtractionE
|
||||||
/// exhausted).
|
/// exhausted).
|
||||||
pub fn channel_videos_continuation(token: &str) -> Result<ContinuationPage, ExtractionError> {
|
pub fn channel_videos_continuation(token: &str) -> Result<ContinuationPage, ExtractionError> {
|
||||||
let body = fetch_continuation_browse(token)?;
|
let body = fetch_continuation_browse(token)?;
|
||||||
Ok(parse_channel_continuation(&body))
|
let page = parse_channel_continuation(&body);
|
||||||
|
log::info!(
|
||||||
|
"channel_videos_continuation ok: items={} more={}",
|
||||||
|
page.items.len(),
|
||||||
|
page.continuation.is_some()
|
||||||
|
);
|
||||||
|
Ok(page)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// POST `browse` with `{"continuation": token}` (no browseId/params).
|
/// POST `browse` with `{"continuation": token}` (no browseId/params).
|
||||||
|
|
@ -170,12 +194,24 @@ fn fetch_continuation_browse(token: &str) -> Result<Value, ExtractionError> {
|
||||||
/// `continuationItems` array.
|
/// `continuationItems` array.
|
||||||
pub fn parse_channel_continuation(body: &Value) -> ContinuationPage {
|
pub fn parse_channel_continuation(body: &Value) -> ContinuationPage {
|
||||||
let Some(items) = continuation_items(body, "onResponseReceivedActions") else {
|
let Some(items) = continuation_items(body, "onResponseReceivedActions") else {
|
||||||
|
// A non-empty body with no continuationItems = the pagination shape
|
||||||
|
// moved; the page returns empty and pagination silently ends.
|
||||||
|
if body.as_object().map(|m| !m.is_empty()).unwrap_or(false) {
|
||||||
|
log::warn!("channel continuation: no continuationItems found (pagination shape changed?); ending pagination");
|
||||||
|
}
|
||||||
return ContinuationPage::default();
|
return ContinuationPage::default();
|
||||||
};
|
};
|
||||||
let videos = items.iter().filter_map(parse_rich_grid_item).collect();
|
let videos: Vec<StreamInfoItem> = items.iter().filter_map(parse_rich_grid_item).collect();
|
||||||
|
// Aggregate renderer-migration signal: cells present but none parsed.
|
||||||
|
if !items.is_empty() && videos.is_empty() {
|
||||||
|
log::warn!(
|
||||||
|
"channel continuation: {} cell(s) but 0 parsed (renderer migration?)",
|
||||||
|
items.len()
|
||||||
|
);
|
||||||
|
}
|
||||||
ContinuationPage {
|
ContinuationPage {
|
||||||
items: videos,
|
|
||||||
continuation: token_from_items(items),
|
continuation: token_from_items(items),
|
||||||
|
items: videos,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -289,11 +325,23 @@ pub fn parse_channel_browse(channel_id: &str, body: &Value) -> ChannelInfo {
|
||||||
/// Handles BOTH old-style `videoRenderer` items and new-style
|
/// Handles BOTH old-style `videoRenderer` items and new-style
|
||||||
/// `lockupViewModel` items (YT migrated channel-videos UI to
|
/// `lockupViewModel` items (YT migrated channel-videos UI to
|
||||||
/// lockupViewModel around 2024).
|
/// lockupViewModel around 2024).
|
||||||
fn parse_videos_tab(body: &Value) -> Vec<StreamInfoItem> {
|
fn parse_videos_tab(body: &Value, channel_id: &str) -> Vec<StreamInfoItem> {
|
||||||
match selected_tab_grid_contents(body) {
|
let Some(items) = selected_tab_grid_contents(body) else {
|
||||||
Some(items) => items.iter().filter_map(parse_rich_grid_item).collect(),
|
// A real browse response (has `contents`) but no grid = layout change.
|
||||||
None => Vec::new(),
|
if body.get("contents").is_some() {
|
||||||
|
log::warn!("[channel {channel_id}] Videos-tab grid not found (layout change?); recent_videos empty");
|
||||||
|
}
|
||||||
|
return Vec::new();
|
||||||
|
};
|
||||||
|
let parsed: Vec<StreamInfoItem> = items.iter().filter_map(parse_rich_grid_item).collect();
|
||||||
|
// Aggregate renderer-migration signal: cells present but none parsed.
|
||||||
|
if !items.is_empty() && parsed.is_empty() {
|
||||||
|
log::warn!(
|
||||||
|
"[channel {channel_id}] Videos-tab grid had {} cell(s) but 0 parsed (renderer migration?)",
|
||||||
|
items.len()
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
parsed
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The `contents[]` array of the selected channel tab's
|
/// The `contents[]` array of the selected channel tab's
|
||||||
|
|
|
||||||
|
|
@ -60,6 +60,8 @@ pub enum DeobfError {
|
||||||
JsRuntimeFailed(String),
|
JsRuntimeFailed(String),
|
||||||
#[error("nsig output was empty (function neutered?)")]
|
#[error("nsig output was empty (function neutered?)")]
|
||||||
NsigEmpty,
|
NsigEmpty,
|
||||||
|
#[error("nsig output equalled its input (identity — deobfuscator neutered?)")]
|
||||||
|
NsigIdentity,
|
||||||
#[error("downloader not initialized")]
|
#[error("downloader not initialized")]
|
||||||
DownloaderMissing,
|
DownloaderMissing,
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -177,7 +177,10 @@ fn deobfuscation_function_body(
|
||||||
let function_base = format!("{function_name}=function");
|
let function_base = format!("{function_name}=function");
|
||||||
match match_to_closing_brace(player_code, &function_base) {
|
match match_to_closing_brace(player_code, &function_base) {
|
||||||
Ok(body) => Ok(format!("{function_base}{body};")),
|
Ok(body) => Ok(format!("{function_base}{body};")),
|
||||||
Err(_) => deobfuscation_function_body_regex(player_code, function_name),
|
Err(_) => {
|
||||||
|
log::debug!("nsig body: lexer brace-match failed, falling back to regex extraction");
|
||||||
|
deobfuscation_function_body_regex(player_code, function_name)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -220,6 +223,11 @@ pub fn fixup_function(function: &str) -> Result<String, DeobfError> {
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
|
|
||||||
if first_arg.is_empty() {
|
if first_arg.is_empty() {
|
||||||
|
// The args regex didn't capture a first argument, so the
|
||||||
|
// `if(typeof X==="undefined")return <arg>;` guard can't be stripped —
|
||||||
|
// the deobfuscator will return its input unchanged (throttled URLs).
|
||||||
|
// Fires ~once per player.js generation (build is cached).
|
||||||
|
log::warn!("nsig fixup: could not extract the function's first argument; the early-return guard cannot be stripped → deobfuscator may return its input unchanged");
|
||||||
return Ok(function.to_string());
|
return Ok(function.to_string());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -108,6 +108,22 @@ impl Derived {
|
||||||
Derived::Nsig => &mut state.nsig_fail,
|
Derived::Nsig => &mut state.nsig_fail,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Short log label for the derived artifact.
|
||||||
|
fn label(&self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
Derived::SigTimestamp => "signature-timestamp",
|
||||||
|
Derived::Sig => "sig",
|
||||||
|
Derived::Nsig => "nsig",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Strip the query/fragment off a player.js URL before logging. Player URLs are
|
||||||
|
/// static + user-independent (safe to log per the audit), but this keeps the
|
||||||
|
/// no-secrets discipline uniform with the rest of the crate.
|
||||||
|
fn strip_query(url: &str) -> &str {
|
||||||
|
url.split(['?', '#']).next().unwrap_or(url)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Verdict of a memo check, decided under the state lock.
|
/// Verdict of a memo check, decided under the state lock.
|
||||||
|
|
@ -282,6 +298,9 @@ impl PlayerManager {
|
||||||
) {
|
) {
|
||||||
Ok(result) => {
|
Ok(result) => {
|
||||||
if result == "null" {
|
if result == "null" {
|
||||||
|
log::warn!(
|
||||||
|
"sig deobf returned \"null\" → empty signature (affected formats may 403)"
|
||||||
|
);
|
||||||
Ok(String::new()) // NPE: Objects.requireNonNullElse(..., "")
|
Ok(String::new()) // NPE: Objects.requireNonNullElse(..., "")
|
||||||
} else {
|
} else {
|
||||||
Ok(result)
|
Ok(result)
|
||||||
|
|
@ -340,6 +359,21 @@ impl PlayerManager {
|
||||||
self.on_eval_failure(Derived::Nsig, fresh, &e);
|
self.on_eval_failure(Derived::Nsig, fresh, &e);
|
||||||
return Err(e);
|
return Err(e);
|
||||||
}
|
}
|
||||||
|
if deobf == obf {
|
||||||
|
// Identity output: the deobfuscator ran but returned its input
|
||||||
|
// UNCHANGED — almost always a surviving `if(typeof X==="undefined")
|
||||||
|
// return p;` guard that `fixup_function` failed to strip. Caching it
|
||||||
|
// would permanently serve THROTTLED URLs for the whole session, so
|
||||||
|
// treat it like a neutered function: DON'T cache, invalidate +
|
||||||
|
// (fresh) memoize so the rest of the open replays cheaply, and let
|
||||||
|
// the throttled formats be counted by the aggregate nsig-fallback
|
||||||
|
// WARN in populate_streams. The memo/invalidate machinery bounds
|
||||||
|
// this WARN to ~once per player.js generation per open.
|
||||||
|
let e = DeobfError::NsigIdentity;
|
||||||
|
log::warn!("nsig deobf returned its input unchanged (identity) → NOT caching (would permanently throttle); player.js likely has an unstripped guard");
|
||||||
|
self.on_eval_failure(Derived::Nsig, fresh, &e);
|
||||||
|
return Err(e);
|
||||||
|
}
|
||||||
|
|
||||||
{
|
{
|
||||||
let mut state = self.state.lock();
|
let mut state = self.state.lock();
|
||||||
|
|
@ -408,6 +442,10 @@ impl PlayerManager {
|
||||||
// The post-cooldown probe itself failed (network).
|
// The post-cooldown probe itself failed (network).
|
||||||
// Re-arm the cooldown so we don't probe on every
|
// Re-arm the cooldown so we don't probe on every
|
||||||
// subsequent call while the network is down.
|
// subsequent call while the network is down.
|
||||||
|
log::warn!(
|
||||||
|
"player.js {} post-cooldown probe failed ({fetch_err}); re-arming cooldown",
|
||||||
|
which.label()
|
||||||
|
);
|
||||||
let mut state = self.state.lock();
|
let mut state = self.state.lock();
|
||||||
if let Some(m) = which.memo_mut(&mut state).as_mut() {
|
if let Some(m) = which.memo_mut(&mut state).as_mut() {
|
||||||
m.at = Instant::now();
|
m.at = Instant::now();
|
||||||
|
|
@ -425,6 +463,11 @@ impl PlayerManager {
|
||||||
let mut state = self.state.lock();
|
let mut state = self.state.lock();
|
||||||
match which.memo_mut(&mut state).as_mut() {
|
match which.memo_mut(&mut state).as_mut() {
|
||||||
Some(m) => {
|
Some(m) => {
|
||||||
|
log::warn!(
|
||||||
|
"player.js {} still broken: YT is still serving the known-bad player.js after cooldown (player={})",
|
||||||
|
which.label(),
|
||||||
|
strip_query(&m.player_url)
|
||||||
|
);
|
||||||
m.at = Instant::now();
|
m.at = Instant::now();
|
||||||
return Err(m.error.clone());
|
return Err(m.error.clone());
|
||||||
}
|
}
|
||||||
|
|
@ -458,6 +501,12 @@ impl PlayerManager {
|
||||||
// the memo so every further call within the cooldown
|
// the memo so every further call within the cooldown
|
||||||
// replays instead of re-fetching ~1.7 MB per format.
|
// replays instead of re-fetching ~1.7 MB per format.
|
||||||
let player_url = state.player_url.clone().unwrap_or_default();
|
let player_url = state.player_url.clone().unwrap_or_default();
|
||||||
|
log::warn!(
|
||||||
|
"player.js {} extraction failed ({e}); memoizing for {}s (player={})",
|
||||||
|
which.label(),
|
||||||
|
FAILURE_COOLDOWN.as_secs(),
|
||||||
|
strip_query(&player_url)
|
||||||
|
);
|
||||||
*which.memo_mut(&mut state) = Some(FailMemo {
|
*which.memo_mut(&mut state) = Some(FailMemo {
|
||||||
player_url,
|
player_url,
|
||||||
error: e.clone(),
|
error: e.clone(),
|
||||||
|
|
@ -467,6 +516,12 @@ impl PlayerManager {
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
// Invariant breach: the 2-pass retry loop is bounded and should always
|
||||||
|
// resolve; exhausting it means an unexpected concurrent-invalidation storm.
|
||||||
|
log::error!(
|
||||||
|
"player.js {} unavailable after retry (concurrent cache-invalidation loop exhausted)",
|
||||||
|
which.label()
|
||||||
|
);
|
||||||
Err(DeobfError::FetchPlayerCode(
|
Err(DeobfError::FetchPlayerCode(
|
||||||
"player.js unavailable after retry (concurrent cache invalidation)".into(),
|
"player.js unavailable after retry (concurrent cache invalidation)".into(),
|
||||||
))
|
))
|
||||||
|
|
@ -482,6 +537,16 @@ impl PlayerManager {
|
||||||
fn on_eval_failure(&self, which: Derived, fresh: bool, error: &DeobfError) {
|
fn on_eval_failure(&self, which: Derived, fresh: bool, error: &DeobfError) {
|
||||||
let mut state = self.state.lock();
|
let mut state = self.state.lock();
|
||||||
let player_url = state.player_url.clone().unwrap_or_default();
|
let player_url = state.player_url.clone().unwrap_or_default();
|
||||||
|
log::warn!(
|
||||||
|
"player.js {} eval failed ({error}); invalidating{} (player={})",
|
||||||
|
which.label(),
|
||||||
|
if fresh {
|
||||||
|
" + memoizing (fresh download)"
|
||||||
|
} else {
|
||||||
|
" for refetch"
|
||||||
|
},
|
||||||
|
strip_query(&player_url)
|
||||||
|
);
|
||||||
state.invalidate();
|
state.invalidate();
|
||||||
if fresh {
|
if fresh {
|
||||||
*which.memo_mut(&mut state) = Some(FailMemo {
|
*which.memo_mut(&mut state) = Some(FailMemo {
|
||||||
|
|
@ -550,6 +615,13 @@ impl PlayerManager {
|
||||||
*memo = None;
|
*memo = None;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// The one INFO that makes every sig/nsig incident diagnosable: which
|
||||||
|
// player.js generation is installed + its size. Lifecycle, ~1/rotation.
|
||||||
|
log::info!(
|
||||||
|
"player.js installed: {} ({}KB, fresh)",
|
||||||
|
strip_query(&url),
|
||||||
|
code.len() / 1024
|
||||||
|
);
|
||||||
state.player_url = Some(url);
|
state.player_url = Some(url);
|
||||||
state.player_code = Some(code);
|
state.player_code = Some(code);
|
||||||
Ok(Ensured::Ready { fresh: true })
|
Ok(Ensured::Ready { fresh: true })
|
||||||
|
|
@ -719,6 +791,12 @@ mod tests {
|
||||||
// (no outer-global references), so eval succeeds for any input.
|
// (no outer-global references), so eval succeeds for any input.
|
||||||
const PLAYER_GOOD: &str = r#"m85=function(p){var a=p.split("");a.reverse();return a.join("");};dummy=function(q){return Q[1];};var foo={signatureTimestamp:20244};"#;
|
const PLAYER_GOOD: &str = r#"m85=function(p){var a=p.split("");a.reverse();return a.join("");};dummy=function(q){return Q[1];};var foo={signatureTimestamp:20244};"#;
|
||||||
|
|
||||||
|
// Parses + evals fine, but the nsig function returns its INPUT unchanged
|
||||||
|
// (identity) — the neutered-deobfuscator shape (e.g. an unstripped guard).
|
||||||
|
// Name matches regex 0 via the trailing `return Q[1]`; the extracted body
|
||||||
|
// is just `m85=function(p){return p;}`.
|
||||||
|
const PLAYER_NSIG_IDENTITY: &str = r#"m85=function(p){return p;};dummy=function(q){return Q[1];};var foo={signatureTimestamp:20244};"#;
|
||||||
|
|
||||||
// Parses fine but throws for EVERY input at eval time.
|
// Parses fine but throws for EVERY input at eval time.
|
||||||
const PLAYER_EVAL_THROWS: &str = r#"m85=function(p){throw Error("boom");return Q[1];};"#;
|
const PLAYER_EVAL_THROWS: &str = r#"m85=function(p){throw Error("boom");return Q[1];};"#;
|
||||||
|
|
||||||
|
|
@ -753,6 +831,29 @@ mod tests {
|
||||||
assert_eq!(stub.count(), 2, "memo must suppress refetching");
|
assert_eq!(stub.count(), 2, "memo must suppress refetching");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn nsig_identity_output_is_rejected_and_not_cached() {
|
||||||
|
let _g = GLOBAL_DOWNLOADER_LOCK.lock();
|
||||||
|
let stub = Arc::new(StubDownloader::new("aaaaaaaa", PLAYER_NSIG_IDENTITY));
|
||||||
|
install(&stub);
|
||||||
|
let mgr = PlayerManager::new();
|
||||||
|
|
||||||
|
// The deobfuscator returns its input unchanged → must surface as
|
||||||
|
// NsigIdentity, NOT be treated as a successful deobfuscation.
|
||||||
|
let err = mgr
|
||||||
|
.url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=abc")
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(matches!(err, DeobfError::NsigIdentity), "got {err:?}");
|
||||||
|
|
||||||
|
// Critically, the identity result must NOT be cached — caching it would
|
||||||
|
// serve THROTTLED URLs for the whole session (the permanent-throttle trap).
|
||||||
|
assert_eq!(
|
||||||
|
mgr.throttling_parameter_cache_size(),
|
||||||
|
0,
|
||||||
|
"identity output must never be cached"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn artifact_memos_are_independent() {
|
fn artifact_memos_are_independent() {
|
||||||
let _g = GLOBAL_DOWNLOADER_LOCK.lock();
|
let _g = GLOBAL_DOWNLOADER_LOCK.lock();
|
||||||
|
|
|
||||||
|
|
@ -64,10 +64,11 @@ pub fn compile_or_throw(snippet: &str) -> Result<(), DeobfError> {
|
||||||
/// with one string argument, returns the toString of the result.
|
/// with one string argument, returns the toString of the result.
|
||||||
/// Mirrors NPE `JavaScript.run(snippet, functionName, parameters)`.
|
/// Mirrors NPE `JavaScript.run(snippet, functionName, parameters)`.
|
||||||
pub fn run(snippet: &str, function_name: &str, parameter: &str) -> Result<String, DeobfError> {
|
pub fn run(snippet: &str, function_name: &str, parameter: &str) -> Result<String, DeobfError> {
|
||||||
|
let started = Instant::now();
|
||||||
let runtime = guarded_runtime().map_err(|e| DeobfError::JsRuntimeFailed(e.to_string()))?;
|
let runtime = guarded_runtime().map_err(|e| DeobfError::JsRuntimeFailed(e.to_string()))?;
|
||||||
let context =
|
let context =
|
||||||
Context::full(&runtime).map_err(|e| DeobfError::JsRuntimeFailed(e.to_string()))?;
|
Context::full(&runtime).map_err(|e| DeobfError::JsRuntimeFailed(e.to_string()))?;
|
||||||
context.with(|ctx| -> Result<String, DeobfError> {
|
let out = context.with(|ctx| -> Result<String, DeobfError> {
|
||||||
ctx.eval::<(), _>(snippet)
|
ctx.eval::<(), _>(snippet)
|
||||||
.map_err(|e| DeobfError::JsRuntimeFailed(format!("eval: {e}")))?;
|
.map_err(|e| DeobfError::JsRuntimeFailed(format!("eval: {e}")))?;
|
||||||
let func: Function = ctx
|
let func: Function = ctx
|
||||||
|
|
@ -80,7 +81,18 @@ pub fn run(snippet: &str, function_name: &str, parameter: &str) -> Result<String
|
||||||
.call((parameter,))
|
.call((parameter,))
|
||||||
.map_err(|e| DeobfError::JsRuntimeFailed(format!("call {function_name}: {e}")))?;
|
.map_err(|e| DeobfError::JsRuntimeFailed(format!("call {function_name}: {e}")))?;
|
||||||
Ok(result)
|
Ok(result)
|
||||||
})
|
});
|
||||||
|
// Distinguish the interrupt-handler deadline abort (a pathological /
|
||||||
|
// looping player.js) from an ordinary eval error — both surface as a
|
||||||
|
// generic QuickJS error at the eval site. `function_name` is a deobf
|
||||||
|
// function id (e.g. "deobfuscate"), not user data.
|
||||||
|
if out.is_err() && started.elapsed() >= JS_DEADLINE {
|
||||||
|
log::warn!(
|
||||||
|
"JS deadline ({}s) hit during `{function_name}` — execution aborted (pathological player.js?)",
|
||||||
|
JS_DEADLINE.as_secs()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
out
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|
|
||||||
|
|
@ -101,7 +101,10 @@ pub fn deobfuscate_function_body(
|
||||||
let function_base = format!("{function_name}=function");
|
let function_base = format!("{function_name}=function");
|
||||||
match match_to_closing_brace(player_code, &function_base) {
|
match match_to_closing_brace(player_code, &function_base) {
|
||||||
Ok(body) => Ok(format!("{function_base}{body}")),
|
Ok(body) => Ok(format!("{function_base}{body}")),
|
||||||
Err(_) => deobfuscate_with_regex(player_code, function_name),
|
Err(_) => {
|
||||||
|
log::debug!("sig body: lexer brace-match failed, falling back to regex extraction");
|
||||||
|
deobfuscate_with_regex(player_code, function_name)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -113,10 +113,12 @@ static REGISTERED_PROVIDER: Lazy<RwLock<Option<Arc<dyn PoTokenProvider>>>> =
|
||||||
Lazy::new(|| RwLock::new(None));
|
Lazy::new(|| RwLock::new(None));
|
||||||
|
|
||||||
pub fn set_po_token_provider(provider: Arc<dyn PoTokenProvider>) {
|
pub fn set_po_token_provider(provider: Arc<dyn PoTokenProvider>) {
|
||||||
|
log::info!("poToken provider registered");
|
||||||
*REGISTERED_PROVIDER.write() = Some(provider);
|
*REGISTERED_PROVIDER.write() = Some(provider);
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn clear_po_token_provider() {
|
pub fn clear_po_token_provider() {
|
||||||
|
log::info!("poToken provider cleared");
|
||||||
*REGISTERED_PROVIDER.write() = None;
|
*REGISTERED_PROVIDER.write() = None;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -69,7 +69,15 @@ pub fn search(query: &str, filter: SearchFilter) -> Result<SearchInfo, Extractio
|
||||||
}
|
}
|
||||||
let parsed: Value = serde_json::from_str(resp.response_body())
|
let parsed: Value = serde_json::from_str(resp.response_body())
|
||||||
.map_err(|e| ExtractionError::Parsing(ParsingError::JsonShape(e.to_string())))?;
|
.map_err(|e| ExtractionError::Parsing(ParsingError::JsonShape(e.to_string())))?;
|
||||||
Ok(parse_search_response(query, &parsed))
|
let info = parse_search_response(query, &parsed);
|
||||||
|
log::info!(
|
||||||
|
"search ok: query_len={} items={} corrected={} continuation={}",
|
||||||
|
query.len(),
|
||||||
|
info.videos.len(),
|
||||||
|
info.corrected_query.is_some(),
|
||||||
|
info.continuation_token.is_some()
|
||||||
|
);
|
||||||
|
Ok(info)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Fetch the NEXT page of search results via a continuation token.
|
/// Fetch the NEXT page of search results via a continuation token.
|
||||||
|
|
@ -105,7 +113,13 @@ pub fn search_continuation(token: &str) -> Result<ContinuationPage, ExtractionEr
|
||||||
}
|
}
|
||||||
let parsed: Value = serde_json::from_str(resp.response_body())
|
let parsed: Value = serde_json::from_str(resp.response_body())
|
||||||
.map_err(|e| ExtractionError::Parsing(ParsingError::JsonShape(e.to_string())))?;
|
.map_err(|e| ExtractionError::Parsing(ParsingError::JsonShape(e.to_string())))?;
|
||||||
Ok(parse_search_continuation(&parsed))
|
let page = parse_search_continuation(&parsed);
|
||||||
|
log::info!(
|
||||||
|
"search_continuation ok: items={} more={}",
|
||||||
|
page.items.len(),
|
||||||
|
page.continuation.is_some()
|
||||||
|
);
|
||||||
|
Ok(page)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Parse a search continuation response. Items arrive under
|
/// Parse a search continuation response. Items arrive under
|
||||||
|
|
@ -116,6 +130,9 @@ pub fn search_continuation(token: &str) -> Result<ContinuationPage, ExtractionEr
|
||||||
/// continuationItemRenderer carrying the next token.
|
/// continuationItemRenderer carrying the next token.
|
||||||
pub fn parse_search_continuation(body: &Value) -> ContinuationPage {
|
pub fn parse_search_continuation(body: &Value) -> ContinuationPage {
|
||||||
let Some(items) = continuation_items(body, "onResponseReceivedCommands") else {
|
let Some(items) = continuation_items(body, "onResponseReceivedCommands") else {
|
||||||
|
if body.as_object().map(|m| !m.is_empty()).unwrap_or(false) {
|
||||||
|
log::warn!("search continuation: no continuationItems found (shape changed?); ending pagination");
|
||||||
|
}
|
||||||
return ContinuationPage::default();
|
return ContinuationPage::default();
|
||||||
};
|
};
|
||||||
let mut info = SearchInfo::default();
|
let mut info = SearchInfo::default();
|
||||||
|
|
@ -155,6 +172,16 @@ pub fn parse_search_response(query: &str, body: &Value) -> SearchInfo {
|
||||||
.and_then(|c| c.get("sectionListRenderer"))
|
.and_then(|c| c.get("sectionListRenderer"))
|
||||||
.and_then(|c| c.get("contents"));
|
.and_then(|c| c.get("contents"));
|
||||||
|
|
||||||
|
// A non-empty response body whose primary results path is gone = a layout
|
||||||
|
// change; we'd otherwise silently return zero results. Log query_len, never
|
||||||
|
// the query.
|
||||||
|
if primary.is_none() && body.as_object().map(|m| !m.is_empty()).unwrap_or(false) {
|
||||||
|
log::warn!(
|
||||||
|
"search: primary results path missing (layout change?); 0 results for query_len={}",
|
||||||
|
query.len()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
if let Some(sections) = primary.and_then(|v| v.as_array()) {
|
if let Some(sections) = primary.and_then(|v| v.as_array()) {
|
||||||
for section in sections {
|
for section in sections {
|
||||||
if let Some(items) = section
|
if let Some(items) = section
|
||||||
|
|
|
||||||
|
|
@ -35,7 +35,7 @@ use crate::youtube::itag::{lookup as itag_lookup, ItagType};
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
use crate::youtube::itag::MediaFormat;
|
use crate::youtube::itag::MediaFormat;
|
||||||
use crate::youtube::js::PlayerManager;
|
use crate::youtube::js::PlayerManager;
|
||||||
use crate::youtube::potoken::{po_token_provider, PoTokenResult};
|
use crate::youtube::potoken::{po_token_provider, PoTokenError, PoTokenResult};
|
||||||
use crate::youtube::stream_helper::{self, generate_content_playback_nonce};
|
use crate::youtube::stream_helper::{self, generate_content_playback_nonce};
|
||||||
use crate::youtube::xtags;
|
use crate::youtube::xtags;
|
||||||
|
|
||||||
|
|
@ -128,9 +128,9 @@ pub fn stream_info_with(
|
||||||
options.android_streaming_pot.as_deref(),
|
options.android_streaming_pot.as_deref(),
|
||||||
options.android_visitor_data.as_deref(),
|
options.android_visitor_data.as_deref(),
|
||||||
|| {
|
|| {
|
||||||
provider
|
provider.as_ref().and_then(|p| {
|
||||||
.as_ref()
|
provider_token(p.get_android_client_po_token(video_id), "ANDROID", video_id)
|
||||||
.and_then(|p| p.get_android_client_po_token(video_id).ok().flatten())
|
})
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|
@ -157,11 +157,18 @@ pub fn stream_info_with(
|
||||||
// the win is the warm/success path. Worst case is a rare, bounded cold-fail
|
// the win is the warm/success path. Worst case is a rare, bounded cold-fail
|
||||||
// delay — never a change in what's extracted.
|
// delay — never a change in what's extracted.
|
||||||
let (android_result, signature_timestamp) = std::thread::scope(|s| {
|
let (android_result, signature_timestamp) = std::thread::scope(|s| {
|
||||||
let sig_ts = s.spawn(|| {
|
let sig_ts = s.spawn(
|
||||||
PlayerManager::instance()
|
|| match PlayerManager::instance().signature_timestamp(video_id) {
|
||||||
.signature_timestamp(video_id)
|
Ok(ts) => ts,
|
||||||
.unwrap_or(0)
|
Err(e) => {
|
||||||
});
|
// WARN, not fatal: sig-required (WEB-family) formats may 403,
|
||||||
|
// but the android-primary formats carry direct URLs. The
|
||||||
|
// timestamp also feeds the WEB metadata playbackContext.
|
||||||
|
log::warn!("[{video_id}] signature_timestamp unavailable ({e}); using 0");
|
||||||
|
0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
);
|
||||||
let android = fetch_android(
|
let android = fetch_android(
|
||||||
video_id,
|
video_id,
|
||||||
&localization,
|
&localization,
|
||||||
|
|
@ -172,8 +179,16 @@ pub fn stream_info_with(
|
||||||
.map(|t| t.player_request_po_token.as_str()),
|
.map(|t| t.player_request_po_token.as_str()),
|
||||||
android_token.as_ref().map(|t| t.visitor_data.as_str()),
|
android_token.as_ref().map(|t| t.visitor_data.as_str()),
|
||||||
);
|
);
|
||||||
// A panicked worker joins as Err → 0, same as an extraction failure.
|
// A panicked worker is an invariant breach (the spawned closure never
|
||||||
(android, sig_ts.join().unwrap_or(0))
|
// panics by construction) → ERROR; still degrade to 0 like a failure.
|
||||||
|
let ts = match sig_ts.join() {
|
||||||
|
Ok(ts) => ts,
|
||||||
|
Err(_) => {
|
||||||
|
log::error!("[{video_id}] signature_timestamp worker thread panicked; using 0");
|
||||||
|
0
|
||||||
|
}
|
||||||
|
};
|
||||||
|
(android, ts)
|
||||||
});
|
});
|
||||||
// ── Primary player-response selection: ANDROID → VISIONOS cascade ────────
|
// ── Primary player-response selection: ANDROID → VISIONOS cascade ────────
|
||||||
// ANDROID was the REQUIRED primary, but the 2026 poToken/BotGuard bot-wall
|
// ANDROID was the REQUIRED primary, but the 2026 poToken/BotGuard bot-wall
|
||||||
|
|
@ -228,9 +243,9 @@ pub fn stream_info_with(
|
||||||
options.ios_streaming_pot.as_deref(),
|
options.ios_streaming_pot.as_deref(),
|
||||||
options.ios_visitor_data.as_deref(),
|
options.ios_visitor_data.as_deref(),
|
||||||
|| {
|
|| {
|
||||||
provider
|
provider.as_ref().and_then(|p| {
|
||||||
.as_ref()
|
provider_token(p.get_ios_client_po_token(video_id), "IOS", video_id)
|
||||||
.and_then(|p| p.get_ios_client_po_token(video_id).ok().flatten())
|
})
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
} else {
|
} else {
|
||||||
|
|
@ -250,7 +265,16 @@ pub fn stream_info_with(
|
||||||
ios_token.as_ref().map(|t| t.visitor_data.as_str()),
|
ios_token.as_ref().map(|t| t.visitor_data.as_str()),
|
||||||
) {
|
) {
|
||||||
Ok(r) if !is_player_response_not_valid(&r, video_id) => (Some(r), Some(ios_cpn)),
|
Ok(r) if !is_player_response_not_valid(&r, video_id) => (Some(r), Some(ios_cpn)),
|
||||||
_ => (None, None),
|
Ok(_) => {
|
||||||
|
log::warn!(
|
||||||
|
"[{video_id}] iOS add-on response was a decoy (videoId mismatch); dropped"
|
||||||
|
);
|
||||||
|
(None, None)
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
log::warn!("[{video_id}] iOS add-on fetch failed ({e}); dropped");
|
||||||
|
(None, None)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
(None, None)
|
(None, None)
|
||||||
|
|
@ -277,7 +301,16 @@ pub fn stream_info_with(
|
||||||
&cpn,
|
&cpn,
|
||||||
) {
|
) {
|
||||||
Ok(r) if !is_player_response_not_valid(&r, video_id) => (Some(r), Some(cpn)),
|
Ok(r) if !is_player_response_not_valid(&r, video_id) => (Some(r), Some(cpn)),
|
||||||
_ => (None, None),
|
Ok(_) => {
|
||||||
|
log::warn!(
|
||||||
|
"[{video_id}] visionOS add-on response was a decoy (videoId mismatch); dropped"
|
||||||
|
);
|
||||||
|
(None, None)
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
log::warn!("[{video_id}] visionOS add-on fetch failed ({e}); dropped");
|
||||||
|
(None, None)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
(None, None)
|
(None, None)
|
||||||
|
|
@ -362,6 +395,17 @@ pub fn stream_info_with(
|
||||||
);
|
);
|
||||||
populate_captions(&mut info, &player_response);
|
populate_captions(&mut info, &player_response);
|
||||||
|
|
||||||
|
log::info!(
|
||||||
|
"[{video_id}] extraction ok: primary={} audio={} video={} video_only={}",
|
||||||
|
if primary_is_visionos {
|
||||||
|
"VISIONOS"
|
||||||
|
} else {
|
||||||
|
"ANDROID"
|
||||||
|
},
|
||||||
|
info.audio_streams.len(),
|
||||||
|
info.video_streams.len(),
|
||||||
|
info.video_only_streams.len(),
|
||||||
|
);
|
||||||
Ok(info)
|
Ok(info)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -392,9 +436,9 @@ pub fn stream_metadata(video_id: &str) -> Result<StreamInfo, ExtractionError> {
|
||||||
// registered (the default) this resolves to None → anonymous reel.
|
// registered (the default) this resolves to None → anonymous reel.
|
||||||
let provider = po_token_provider();
|
let provider = po_token_provider();
|
||||||
let android_token: Option<PoTokenResult> = options_or_provider(None, None, None, || {
|
let android_token: Option<PoTokenResult> = options_or_provider(None, None, None, || {
|
||||||
provider
|
provider.as_ref().and_then(|p| {
|
||||||
.as_ref()
|
provider_token(p.get_android_client_po_token(video_id), "ANDROID", video_id)
|
||||||
.and_then(|p| p.get_android_client_po_token(video_id).ok().flatten())
|
})
|
||||||
});
|
});
|
||||||
|
|
||||||
let android_cpn = generate_content_playback_nonce();
|
let android_cpn = generate_content_playback_nonce();
|
||||||
|
|
@ -432,9 +476,36 @@ pub fn stream_metadata(video_id: &str) -> Result<StreamInfo, ExtractionError> {
|
||||||
..StreamInfo::default()
|
..StreamInfo::default()
|
||||||
};
|
};
|
||||||
populate_video_details(&mut info, &player_response);
|
populate_video_details(&mut info, &player_response);
|
||||||
|
log::info!(
|
||||||
|
"[{video_id}] stream_metadata ok: title_len={} views={} duration={}s",
|
||||||
|
info.name.len(),
|
||||||
|
info.view_count,
|
||||||
|
info.duration_seconds,
|
||||||
|
);
|
||||||
Ok(info)
|
Ok(info)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Unwrap a PoTokenProvider result, logging a WARN on the Err arm the old
|
||||||
|
/// `.ok().flatten()` swallowed silently. `Ok(None)` = provider declined (go
|
||||||
|
/// anonymous, no log); `Err` = provider tried and failed (still go anonymous,
|
||||||
|
/// but it's a degradation worth a breadcrumb). Never logs the token — only the
|
||||||
|
/// error kind + client + videoId.
|
||||||
|
fn provider_token(
|
||||||
|
result: Result<Option<PoTokenResult>, PoTokenError>,
|
||||||
|
client: &str,
|
||||||
|
video_id: &str,
|
||||||
|
) -> Option<PoTokenResult> {
|
||||||
|
match result {
|
||||||
|
Ok(opt) => opt,
|
||||||
|
Err(e) => {
|
||||||
|
log::warn!(
|
||||||
|
"[{video_id}] {client} poToken provider errored ({e}); continuing anonymous"
|
||||||
|
);
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn options_or_provider(
|
fn options_or_provider(
|
||||||
opt_player_token: Option<&str>,
|
opt_player_token: Option<&str>,
|
||||||
opt_streaming_token: Option<&str>,
|
opt_streaming_token: Option<&str>,
|
||||||
|
|
@ -561,13 +632,21 @@ fn fetch_web_metadata(
|
||||||
content_country: &ContentCountry,
|
content_country: &ContentCountry,
|
||||||
signature_timestamp: i32,
|
signature_timestamp: i32,
|
||||||
) -> Value {
|
) -> Value {
|
||||||
stream_helper::get_web_metadata_player_response(
|
match stream_helper::get_web_metadata_player_response(
|
||||||
video_id,
|
video_id,
|
||||||
localization,
|
localization,
|
||||||
content_country,
|
content_country,
|
||||||
signature_timestamp,
|
signature_timestamp,
|
||||||
)
|
) {
|
||||||
.unwrap_or(Value::Null)
|
Ok(v) => v,
|
||||||
|
Err(e) => {
|
||||||
|
// WEB metadata is best-effort; on failure we fall back to the
|
||||||
|
// primary response's videoDetails thumbnails and lose upload
|
||||||
|
// date / category / hi-res thumbnails. Also masks a WEB-client wall.
|
||||||
|
log::warn!("[{video_id}] WEB metadata fetch failed ({e}); upload date/category/hi-res thumbnails unavailable");
|
||||||
|
Value::Null
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn check_playability_status(player_response: &Value) -> Result<(), ExtractionError> {
|
fn check_playability_status(player_response: &Value) -> Result<(), ExtractionError> {
|
||||||
|
|
@ -583,7 +662,7 @@ fn check_playability_status(player_response: &Value) -> Result<(), ExtractionErr
|
||||||
let mapped = match status_code {
|
let mapped = match status_code {
|
||||||
"LOGIN_REQUIRED" => {
|
"LOGIN_REQUIRED" => {
|
||||||
if reason_lc.contains("a bot") {
|
if reason_lc.contains("a bot") {
|
||||||
ContentUnavailable::Other("sign in to confirm you're not a bot".into())
|
ContentUnavailable::BotDetected
|
||||||
} else if reason_lc.contains("inappropriate") {
|
} else if reason_lc.contains("inappropriate") {
|
||||||
ContentUnavailable::AgeRestricted
|
ContentUnavailable::AgeRestricted
|
||||||
} else if reason_lc.contains("private") {
|
} else if reason_lc.contains("private") {
|
||||||
|
|
@ -657,15 +736,28 @@ fn select_primary(
|
||||||
// ANDROID unusable → drop the (likely poisoned) visitorData, then try
|
// ANDROID unusable → drop the (likely poisoned) visitorData, then try
|
||||||
// VISIONOS as the primary. On any VISIONOS failure, surface the ORIGINAL
|
// VISIONOS as the primary. On any VISIONOS failure, surface the ORIGINAL
|
||||||
// ANDROID error rather than the visionOS one.
|
// ANDROID error rather than the visionOS one.
|
||||||
|
// `android_error`'s Display carries the playabilityStatus reason (the
|
||||||
|
// choke point in exceptions.rs strips any URL, so no stream URL leaks).
|
||||||
|
log::warn!("[{video_id}] ANDROID primary unusable ({android_error}); trying VISIONOS");
|
||||||
on_fallback();
|
on_fallback();
|
||||||
match fetch_visionos() {
|
match fetch_visionos() {
|
||||||
Ok(resp) if primary_response_usable(&resp, video_id).is_ok() => Ok((resp, true)),
|
Ok(resp) if primary_response_usable(&resp, video_id).is_ok() => {
|
||||||
_ => Err(android_error),
|
log::info!("[{video_id}] primary=VISIONOS (ANDROID fallback succeeded)");
|
||||||
|
Ok((resp, true))
|
||||||
|
}
|
||||||
|
_ => {
|
||||||
|
log::warn!("[{video_id}] extraction failed: ANDROID and VISIONOS both unusable ({android_error})");
|
||||||
|
Err(android_error)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn populate_video_details(info: &mut StreamInfo, player_response: &Value) {
|
fn populate_video_details(info: &mut StreamInfo, player_response: &Value) {
|
||||||
let Some(vd) = player_response.get("videoDetails") else {
|
let Some(vd) = player_response.get("videoDetails") else {
|
||||||
|
log::warn!(
|
||||||
|
"[{}] player response had no videoDetails; title/uploader/duration left empty",
|
||||||
|
info.video_id
|
||||||
|
);
|
||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
if let Some(s) = vd.get("title").and_then(|v| v.as_str()) {
|
if let Some(s) = vd.get("title").and_then(|v| v.as_str()) {
|
||||||
|
|
@ -759,6 +851,22 @@ fn populate_microformat(info: &mut StreamInfo, web_metadata: &Value) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Per-EXTRACTION accumulator for per-format URL-processing outcomes, so the
|
||||||
|
/// degradations that happen inside the per-format loop are logged ONCE (in
|
||||||
|
/// `populate_streams`) as aggregate counters instead of spamming a line per
|
||||||
|
/// format. Threaded through `process_url` / `build_*`; never logged from inside
|
||||||
|
/// the loop.
|
||||||
|
#[derive(Default)]
|
||||||
|
struct UrlProcessStats {
|
||||||
|
/// Formats that shipped the THROTTLED original URL because nsig deobf was
|
||||||
|
/// unavailable (`url_with_throttling_parameter_deobfuscated` Err).
|
||||||
|
nsig_fallback: u32,
|
||||||
|
/// Formats that took the (android-primary-rare) signatureCipher path.
|
||||||
|
cipher_path: u32,
|
||||||
|
/// signatureCipher-path formats dropped because sig deobf failed.
|
||||||
|
cipher_sig_failed: u32,
|
||||||
|
}
|
||||||
|
|
||||||
/// One client's streamingData plus the tags every format from it carries.
|
/// One client's streamingData plus the tags every format from it carries.
|
||||||
/// Holds borrowed `&Value`s into the (function-lived) player responses.
|
/// Holds borrowed `&Value`s into the (function-lived) player responses.
|
||||||
struct FormatSource<'a> {
|
struct FormatSource<'a> {
|
||||||
|
|
@ -794,9 +902,13 @@ fn populate_streams(
|
||||||
sources: &[FormatSource],
|
sources: &[FormatSource],
|
||||||
video_id: &str,
|
video_id: &str,
|
||||||
) -> Result<(), ExtractionError> {
|
) -> Result<(), ExtractionError> {
|
||||||
|
// Per-call accumulator — every per-format degradation is counted here and
|
||||||
|
// logged ONCE below, never per iteration.
|
||||||
|
let mut stats = UrlProcessStats::default();
|
||||||
|
|
||||||
// Progressive: streamingData.formats[]
|
// Progressive: streamingData.formats[]
|
||||||
for (fmt, _client, cpn, pot) in merge_formats(sources, "formats") {
|
for (fmt, _client, cpn, pot) in merge_formats(sources, "formats") {
|
||||||
if let Some(stream) = build_video_progressive(fmt, video_id, cpn, pot)? {
|
if let Some(stream) = build_video_progressive(fmt, video_id, cpn, pot, &mut stats)? {
|
||||||
push_video_dedup(&mut info.video_streams, stream);
|
push_video_dedup(&mut info.video_streams, stream);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -808,15 +920,58 @@ fn populate_streams(
|
||||||
.and_then(|v| v.as_str())
|
.and_then(|v| v.as_str())
|
||||||
.unwrap_or("");
|
.unwrap_or("");
|
||||||
if mime.starts_with("audio/") {
|
if mime.starts_with("audio/") {
|
||||||
if let Some(audio) = build_audio(fmt, video_id, cpn, pot)? {
|
if let Some(audio) = build_audio(fmt, video_id, cpn, pot, &mut stats)? {
|
||||||
push_audio_dedup(&mut info.audio_streams, audio);
|
push_audio_dedup(&mut info.audio_streams, audio);
|
||||||
}
|
}
|
||||||
} else if mime.starts_with("video/") {
|
} else if mime.starts_with("video/") {
|
||||||
if let Some(video) = build_video_only(fmt, video_id, cpn, pot)? {
|
if let Some(video) = build_video_only(fmt, video_id, cpn, pot, &mut stats)? {
|
||||||
push_video_dedup(&mut info.video_only_streams, video);
|
push_video_dedup(&mut info.video_only_streams, video);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── Aggregate degradation WARNs (one line each per extraction) ──────────
|
||||||
|
if stats.nsig_fallback > 0 {
|
||||||
|
log::warn!(
|
||||||
|
"[{video_id}] {} format(s) shipped THROTTLED URLs (nsig deobf unavailable)",
|
||||||
|
stats.nsig_fallback
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if stats.cipher_path > 0 {
|
||||||
|
log::warn!(
|
||||||
|
"[{video_id}] {} format(s) took the signatureCipher path ({} sig-deobf failure(s))",
|
||||||
|
stats.cipher_path,
|
||||||
|
stats.cipher_sig_failed
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let total = info.video_streams.len() + info.audio_streams.len() + info.video_only_streams.len();
|
||||||
|
if total == 0 {
|
||||||
|
// Playability passed but nothing built — report the raw per-source
|
||||||
|
// format counts YT actually returned so a "why no streams" incident is
|
||||||
|
// diagnosable (itag-bank miss vs empty streamingData vs all-cipher).
|
||||||
|
let counts: Vec<String> = sources
|
||||||
|
.iter()
|
||||||
|
.map(|s| {
|
||||||
|
let f = s
|
||||||
|
.streaming_data
|
||||||
|
.get("formats")
|
||||||
|
.and_then(|v| v.as_array())
|
||||||
|
.map(|a| a.len())
|
||||||
|
.unwrap_or(0);
|
||||||
|
let af = s
|
||||||
|
.streaming_data
|
||||||
|
.get("adaptiveFormats")
|
||||||
|
.and_then(|v| v.as_array())
|
||||||
|
.map(|a| a.len())
|
||||||
|
.unwrap_or(0);
|
||||||
|
format!("{}={}+{}", s.client, f, af)
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
log::warn!(
|
||||||
|
"[{video_id}] playability OK but ZERO usable streams built (per-source formats+adaptive: {})",
|
||||||
|
counts.join(" ")
|
||||||
|
);
|
||||||
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -914,12 +1069,15 @@ fn process_url(
|
||||||
video_id: &str,
|
video_id: &str,
|
||||||
cpn: &str,
|
cpn: &str,
|
||||||
pot: Option<&str>,
|
pot: Option<&str>,
|
||||||
|
stats: &mut UrlProcessStats,
|
||||||
) -> Result<Option<String>, ExtractionError> {
|
) -> Result<Option<String>, ExtractionError> {
|
||||||
let mut url = if let Some(u) = raw_format.get("url").and_then(|v| v.as_str()) {
|
let mut url = if let Some(u) = raw_format.get("url").and_then(|v| v.as_str()) {
|
||||||
u.to_string()
|
u.to_string()
|
||||||
} else {
|
} else {
|
||||||
// signatureCipher path — WEB-family only; not exercised in the
|
// signatureCipher path — WEB-family only; not exercised in the
|
||||||
// Android-primary flow but mirror NPE's behavior for completeness.
|
// Android-primary flow but mirror NPE's behavior for completeness.
|
||||||
|
// Counted per-extraction (aggregate WARN in populate_streams).
|
||||||
|
stats.cipher_path += 1;
|
||||||
let cipher_str = raw_format
|
let cipher_str = raw_format
|
||||||
.get("signatureCipher")
|
.get("signatureCipher")
|
||||||
.or_else(|| raw_format.get("cipher"))
|
.or_else(|| raw_format.get("cipher"))
|
||||||
|
|
@ -940,7 +1098,10 @@ fn process_url(
|
||||||
// the Android-primary flow.)
|
// the Android-primary flow.)
|
||||||
let deobf = match PlayerManager::instance().deobfuscate_signature(video_id, s) {
|
let deobf = match PlayerManager::instance().deobfuscate_signature(video_id, s) {
|
||||||
Ok(d) => d,
|
Ok(d) => d,
|
||||||
Err(_) => return Ok(None),
|
Err(_) => {
|
||||||
|
stats.cipher_sig_failed += 1;
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
};
|
};
|
||||||
format!("{base}&{sp}={deobf}")
|
format!("{base}&{sp}={deobf}")
|
||||||
};
|
};
|
||||||
|
|
@ -956,9 +1117,19 @@ fn process_url(
|
||||||
// zero network instead of re-downloading ~1.7 MB per format, and retries
|
// zero network instead of re-downloading ~1.7 MB per format, and retries
|
||||||
// after a cooldown / as soon as a new player.js ships. (NPE parity; straw
|
// after a cooldown / as soon as a new player.js ships. (NPE parity; straw
|
||||||
// audits 2026-07-04 + 2026-07-28 H2.)
|
// audits 2026-07-04 + 2026-07-28 H2.)
|
||||||
url = PlayerManager::instance()
|
url = match PlayerManager::instance()
|
||||||
.url_with_throttling_parameter_deobfuscated(video_id, &url)
|
.url_with_throttling_parameter_deobfuscated(video_id, &url)
|
||||||
.unwrap_or(url);
|
{
|
||||||
|
Ok(u) => u,
|
||||||
|
Err(_e) => {
|
||||||
|
// nsig deobf unavailable (fetch/build/eval/identity failure) → ship
|
||||||
|
// the THROTTLED original URL. YouTube still serves it, rate-limited.
|
||||||
|
// Counted per-extraction; populate_streams emits ONE aggregate WARN.
|
||||||
|
// Deliberately NOT logged here (per-format = spam).
|
||||||
|
stats.nsig_fallback += 1;
|
||||||
|
url
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
let sep_cpn = if url.contains('?') { '&' } else { '?' };
|
let sep_cpn = if url.contains('?') { '&' } else { '?' };
|
||||||
url = format!("{url}{sep_cpn}cpn={cpn}");
|
url = format!("{url}{sep_cpn}cpn={cpn}");
|
||||||
|
|
@ -987,12 +1158,13 @@ fn build_video_progressive(
|
||||||
video_id: &str,
|
video_id: &str,
|
||||||
cpn: &str,
|
cpn: &str,
|
||||||
pot: Option<&str>,
|
pot: Option<&str>,
|
||||||
|
stats: &mut UrlProcessStats,
|
||||||
) -> Result<Option<VideoStream>, ExtractionError> {
|
) -> Result<Option<VideoStream>, ExtractionError> {
|
||||||
let itag_id = fmt.get("itag").and_then(|v| v.as_u64()).unwrap_or(0) as u32;
|
let itag_id = fmt.get("itag").and_then(|v| v.as_u64()).unwrap_or(0) as u32;
|
||||||
let Some(itag) = itag_lookup(itag_id) else {
|
let Some(itag) = itag_lookup(itag_id) else {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
};
|
};
|
||||||
let Some(url) = process_url(fmt, video_id, cpn, pot)? else {
|
let Some(url) = process_url(fmt, video_id, cpn, pot, stats)? else {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
};
|
};
|
||||||
Ok(Some(VideoStream {
|
Ok(Some(VideoStream {
|
||||||
|
|
@ -1019,6 +1191,7 @@ fn build_video_only(
|
||||||
video_id: &str,
|
video_id: &str,
|
||||||
cpn: &str,
|
cpn: &str,
|
||||||
pot: Option<&str>,
|
pot: Option<&str>,
|
||||||
|
stats: &mut UrlProcessStats,
|
||||||
) -> Result<Option<VideoStream>, ExtractionError> {
|
) -> Result<Option<VideoStream>, ExtractionError> {
|
||||||
let itag_id = fmt.get("itag").and_then(|v| v.as_u64()).unwrap_or(0) as u32;
|
let itag_id = fmt.get("itag").and_then(|v| v.as_u64()).unwrap_or(0) as u32;
|
||||||
let Some(itag) = itag_lookup(itag_id) else {
|
let Some(itag) = itag_lookup(itag_id) else {
|
||||||
|
|
@ -1027,7 +1200,7 @@ fn build_video_only(
|
||||||
if itag.item_type != ItagType::VideoOnly {
|
if itag.item_type != ItagType::VideoOnly {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
}
|
}
|
||||||
let Some(url) = process_url(fmt, video_id, cpn, pot)? else {
|
let Some(url) = process_url(fmt, video_id, cpn, pot, stats)? else {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
};
|
};
|
||||||
Ok(Some(VideoStream {
|
Ok(Some(VideoStream {
|
||||||
|
|
@ -1054,6 +1227,7 @@ fn build_audio(
|
||||||
video_id: &str,
|
video_id: &str,
|
||||||
cpn: &str,
|
cpn: &str,
|
||||||
pot: Option<&str>,
|
pot: Option<&str>,
|
||||||
|
stats: &mut UrlProcessStats,
|
||||||
) -> Result<Option<AudioStream>, ExtractionError> {
|
) -> Result<Option<AudioStream>, ExtractionError> {
|
||||||
let itag_id = fmt.get("itag").and_then(|v| v.as_u64()).unwrap_or(0) as u32;
|
let itag_id = fmt.get("itag").and_then(|v| v.as_u64()).unwrap_or(0) as u32;
|
||||||
let Some(itag) = itag_lookup(itag_id) else {
|
let Some(itag) = itag_lookup(itag_id) else {
|
||||||
|
|
@ -1062,7 +1236,7 @@ fn build_audio(
|
||||||
if itag.item_type != ItagType::Audio {
|
if itag.item_type != ItagType::Audio {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
}
|
}
|
||||||
let Some(url) = process_url(fmt, video_id, cpn, pot)? else {
|
let Some(url) = process_url(fmt, video_id, cpn, pot, stats)? else {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
};
|
};
|
||||||
let audio_track = fmt.get("audioTrack");
|
let audio_track = fmt.get("audioTrack");
|
||||||
|
|
@ -1214,6 +1388,26 @@ mod tests {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn playability_bot_wall_maps_to_bot_detected() {
|
||||||
|
// The 2026 poToken/BotGuard wall on the pot-less client. It must map to
|
||||||
|
// the dedicated BotDetected variant (greppable / classifiable), and its
|
||||||
|
// Display must stay byte-identical to the old Other(...) string so
|
||||||
|
// string-matching callers keep working.
|
||||||
|
let resp = json!({
|
||||||
|
"playabilityStatus": {
|
||||||
|
"status": "LOGIN_REQUIRED",
|
||||||
|
"reason": "Sign in to confirm you’re not a bot. This helps protect our community."
|
||||||
|
}
|
||||||
|
});
|
||||||
|
match check_playability_status(&resp).unwrap_err() {
|
||||||
|
ExtractionError::ContentUnavailable(ref cu @ ContentUnavailable::BotDetected) => {
|
||||||
|
assert_eq!(cu.to_string(), "sign in to confirm you're not a bot");
|
||||||
|
}
|
||||||
|
other => panic!("expected BotDetected, got {other:?}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn decoy_detected() {
|
fn decoy_detected() {
|
||||||
let resp = json!({"videoDetails": {"videoId": "DIFFERENT_ID"}});
|
let resp = json!({"videoDetails": {"videoId": "DIFFERENT_ID"}});
|
||||||
|
|
@ -1560,7 +1754,9 @@ mod tests {
|
||||||
#[test]
|
#[test]
|
||||||
fn xtags_descriptive_sets_track_type_and_is_descriptive() {
|
fn xtags_descriptive_sets_track_type_and_is_descriptive() {
|
||||||
let fmt = audio_fmt(Some("ChQKBWFjb250EgtkZXNjcmlwdGl2ZQ"), true);
|
let fmt = audio_fmt(Some("ChQKBWFjb250EgtkZXNjcmlwdGl2ZQ"), true);
|
||||||
let a = build_audio(&fmt, "vid", "cpn", None).unwrap().unwrap();
|
let a = build_audio(&fmt, "vid", "cpn", None, &mut UrlProcessStats::default())
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
assert_eq!(a.track_type, Some(AudioTrackType::Descriptive));
|
assert_eq!(a.track_type, Some(AudioTrackType::Descriptive));
|
||||||
assert!(a.is_descriptive);
|
assert!(a.is_descriptive);
|
||||||
}
|
}
|
||||||
|
|
@ -1570,7 +1766,9 @@ mod tests {
|
||||||
// acont=dubbed → Dubbed + is_descriptive false, even though
|
// acont=dubbed → Dubbed + is_descriptive false, even though
|
||||||
// audioIsDefault=false would make the legacy heuristic say "descriptive".
|
// audioIsDefault=false would make the legacy heuristic say "descriptive".
|
||||||
let fmt = audio_fmt(Some("Cg8KBWFjb250EgZkdWJiZWQ"), false);
|
let fmt = audio_fmt(Some("Cg8KBWFjb250EgZkdWJiZWQ"), false);
|
||||||
let a = build_audio(&fmt, "vid", "cpn", None).unwrap().unwrap();
|
let a = build_audio(&fmt, "vid", "cpn", None, &mut UrlProcessStats::default())
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
assert_eq!(a.track_type, Some(AudioTrackType::Dubbed));
|
assert_eq!(a.track_type, Some(AudioTrackType::Dubbed));
|
||||||
assert!(!a.is_descriptive);
|
assert!(!a.is_descriptive);
|
||||||
}
|
}
|
||||||
|
|
@ -1578,7 +1776,9 @@ mod tests {
|
||||||
#[test]
|
#[test]
|
||||||
fn xtags_original_maps_to_original() {
|
fn xtags_original_maps_to_original() {
|
||||||
let fmt = audio_fmt(Some("ChEKBWFjb250EghvcmlnaW5hbA"), true);
|
let fmt = audio_fmt(Some("ChEKBWFjb250EghvcmlnaW5hbA"), true);
|
||||||
let a = build_audio(&fmt, "vid", "cpn", None).unwrap().unwrap();
|
let a = build_audio(&fmt, "vid", "cpn", None, &mut UrlProcessStats::default())
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
assert_eq!(a.track_type, Some(AudioTrackType::Original));
|
assert_eq!(a.track_type, Some(AudioTrackType::Original));
|
||||||
assert!(!a.is_descriptive);
|
assert!(!a.is_descriptive);
|
||||||
}
|
}
|
||||||
|
|
@ -1586,15 +1786,27 @@ mod tests {
|
||||||
#[test]
|
#[test]
|
||||||
fn absent_xtags_falls_back_to_audio_is_default_heuristic() {
|
fn absent_xtags_falls_back_to_audio_is_default_heuristic() {
|
||||||
// No xtags → track_type None → is_descriptive == !audioIsDefault.
|
// No xtags → track_type None → is_descriptive == !audioIsDefault.
|
||||||
let a = build_audio(&audio_fmt(None, false), "vid", "cpn", None)
|
let a = build_audio(
|
||||||
.unwrap()
|
&audio_fmt(None, false),
|
||||||
.unwrap();
|
"vid",
|
||||||
|
"cpn",
|
||||||
|
None,
|
||||||
|
&mut UrlProcessStats::default(),
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
assert_eq!(a.track_type, None);
|
assert_eq!(a.track_type, None);
|
||||||
assert!(a.is_descriptive); // !false
|
assert!(a.is_descriptive); // !false
|
||||||
|
|
||||||
let a = build_audio(&audio_fmt(None, true), "vid", "cpn", None)
|
let a = build_audio(
|
||||||
.unwrap()
|
&audio_fmt(None, true),
|
||||||
.unwrap();
|
"vid",
|
||||||
|
"cpn",
|
||||||
|
None,
|
||||||
|
&mut UrlProcessStats::default(),
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
assert_eq!(a.track_type, None);
|
assert_eq!(a.track_type, None);
|
||||||
assert!(!a.is_descriptive); // !true
|
assert!(!a.is_descriptive); // !true
|
||||||
}
|
}
|
||||||
|
|
@ -1602,9 +1814,15 @@ mod tests {
|
||||||
#[test]
|
#[test]
|
||||||
fn unparseable_xtags_falls_back_to_heuristic() {
|
fn unparseable_xtags_falls_back_to_heuristic() {
|
||||||
// Garbage xtags → extract_audio_track_type None → heuristic used.
|
// Garbage xtags → extract_audio_track_type None → heuristic used.
|
||||||
let a = build_audio(&audio_fmt(Some("!!!not-b64!!!"), false), "vid", "cpn", None)
|
let a = build_audio(
|
||||||
.unwrap()
|
&audio_fmt(Some("!!!not-b64!!!"), false),
|
||||||
.unwrap();
|
"vid",
|
||||||
|
"cpn",
|
||||||
|
None,
|
||||||
|
&mut UrlProcessStats::default(),
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
assert_eq!(a.track_type, None);
|
assert_eq!(a.track_type, None);
|
||||||
assert!(a.is_descriptive);
|
assert!(a.is_descriptive);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -142,8 +142,21 @@ fn get_visitor_data(
|
||||||
// best-effort None (the caller then proceeds without visitorData). An
|
// best-effort None (the caller then proceeds without visitorData). An
|
||||||
// expired-but-present cache entry is deliberately NOT served on a failed
|
// expired-but-present cache entry is deliberately NOT served on a failed
|
||||||
// refetch: a stale visitorData is the thing we're rotating away from.
|
// refetch: a stale visitorData is the thing we're rotating away from.
|
||||||
let parsed = post_youtube(&url, &body, headers).ok()?;
|
// WARN because losing visitorData weakens attestation and kneecaps the
|
||||||
let visitor = visitor_data_from_response(&parsed)?;
|
// visionOS bot-wall survivor. post_youtube already DEBUG-logged the
|
||||||
|
// HTTP-level detail; this is the semantic degradation. `domain` is a
|
||||||
|
// static endpoint constant (no secrets).
|
||||||
|
let parsed = match post_youtube(&url, &body, headers) {
|
||||||
|
Ok(p) => p,
|
||||||
|
Err(_e) => {
|
||||||
|
log::warn!("visitorData unavailable ({domain}visitor_id fetch failed); proceeding without it (attestation/visionOS weakened)");
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let Some(visitor) = visitor_data_from_response(&parsed) else {
|
||||||
|
log::warn!("visitorData unavailable ({domain}visitor_id response had no visitorData); proceeding without it");
|
||||||
|
return None;
|
||||||
|
};
|
||||||
*VISITOR_DATA_CACHE.write() = Some(CachedVisitorData {
|
*VISITOR_DATA_CACHE.write() = Some(CachedVisitorData {
|
||||||
value: visitor.clone(),
|
value: visitor.clone(),
|
||||||
at: std::time::Instant::now(),
|
at: std::time::Instant::now(),
|
||||||
|
|
@ -157,6 +170,9 @@ fn get_visitor_data(
|
||||||
/// the primary path, and since that failure aborts extraction the value would
|
/// the primary path, and since that failure aborts extraction the value would
|
||||||
/// otherwise never self-heal.
|
/// otherwise never self-heal.
|
||||||
pub(crate) fn reset_visitor_data_cache() {
|
pub(crate) fn reset_visitor_data_cache() {
|
||||||
|
// Lifecycle INFO: the self-heal trigger. Covers both the primary-cascade
|
||||||
|
// path and the stream_metadata twin path.
|
||||||
|
log::info!("visitorData cache reset (primary response rejected; next fetch mints fresh)");
|
||||||
*VISITOR_DATA_CACHE.write() = None;
|
*VISITOR_DATA_CACHE.write() = None;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -459,14 +475,32 @@ fn post_youtube(
|
||||||
// put it in an error string (they reach Kotlin exception messages
|
// put it in an error string (they reach Kotlin exception messages
|
||||||
// and logs). Scheme+host+path identifies the endpoint just fine.
|
// and logs). Scheme+host+path identifies the endpoint just fine.
|
||||||
let endpoint = url.split(['?', '#']).next().unwrap_or(url);
|
let endpoint = url.split(['?', '#']).next().unwrap_or(url);
|
||||||
|
// body_len is the bot-wall tell (an HTML challenge page has a
|
||||||
|
// distinctive size vs a JSON error). DEBUG — per-request breadcrumb;
|
||||||
|
// the semantic WARN is emitted by the caller.
|
||||||
|
log::debug!(
|
||||||
|
"post_youtube {endpoint} → HTTP {} ({}B body)",
|
||||||
|
resp.response_code(),
|
||||||
|
resp.response_body().len()
|
||||||
|
);
|
||||||
return Err(ExtractionError::Network(NetworkError::Transport(format!(
|
return Err(ExtractionError::Network(NetworkError::Transport(format!(
|
||||||
"HTTP {} from {endpoint}",
|
"HTTP {} from {endpoint}",
|
||||||
resp.response_code()
|
resp.response_code()
|
||||||
))));
|
))));
|
||||||
}
|
}
|
||||||
let parsed: Value = serde_json::from_str(resp.response_body())
|
match serde_json::from_str(resp.response_body()) {
|
||||||
.map_err(|e| ExtractionError::Parsing(ParsingError::JsonShape(e.to_string())))?;
|
Ok(parsed) => Ok(parsed),
|
||||||
Ok(parsed)
|
Err(e) => {
|
||||||
|
let endpoint = url.split(['?', '#']).next().unwrap_or(url);
|
||||||
|
log::debug!(
|
||||||
|
"post_youtube {endpoint} → 200 but body was not JSON ({}B): {e}",
|
||||||
|
resp.response_body().len()
|
||||||
|
);
|
||||||
|
Err(ExtractionError::Parsing(ParsingError::JsonShape(
|
||||||
|
e.to_string(),
|
||||||
|
)))
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue