diff --git a/Cargo.lock b/Cargo.lock index 82fb84a..c4af244 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -121,7 +121,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -711,9 +711,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.13.1" +version = "1.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" dependencies = [ "aho-corasick", "memchr", @@ -723,9 +723,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.16" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" dependencies = [ "aho-corasick", "memchr", @@ -891,9 +891,9 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", "serde_derive", @@ -901,29 +901,29 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.3", ] [[package]] name = "serde_json" -version = "1.0.150" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ "itoa", "memchr", @@ -1017,6 +1017,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "sync_wrapper" version = "1.0.2" @@ -1034,7 +1045,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1063,7 +1074,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1074,7 +1085,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1316,7 +1327,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wasm-bindgen-shared", ] @@ -1551,7 +1562,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", "synstructure", ] @@ -1572,7 +1583,7 @@ checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1592,7 +1603,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", "synstructure", ] @@ -1632,7 +1643,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] diff --git a/src/downloader/default_impl.rs b/src/downloader/default_impl.rs index 9f3f5ba..9dcefea 100644 --- a/src/downloader/default_impl.rs +++ b/src/downloader/default_impl.rs @@ -81,7 +81,17 @@ impl Downloader for ReqwestDownloader { let url_after_redirects = resp.url().to_string(); if status.as_u16() == 429 { - return Err(NetworkError::Recaptcha { url: url_after_redirects }); + // Privacy: this URL ends up in error strings (and from there in + // Kotlin exception messages / logs, and the deobf failure memo). + // YouTube URLs carry the video id in the query (`?v=` / `&id=`) + // AND in some paths (`/embed/`, `/shorts/`), so keep only + // scheme+host. The Response's latest_url below stays full — it's + // data consumed by redirect-tracking logic, not a message. + let mut stripped = resp.url().clone(); + stripped.set_query(None); + stripped.set_fragment(None); + stripped.set_path("/"); + return Err(NetworkError::Recaptcha { url: stripped.to_string() }); } let code = status.as_u16(); diff --git a/src/downloader/response.rs b/src/downloader/response.rs index cd34d32..4702277 100644 --- a/src/downloader/response.rs +++ b/src/downloader/response.rs @@ -54,6 +54,15 @@ impl Response { &self.response_body } + /// Consumes the response, handing back the owned body `String` with no + /// copy. Use at the last read of a body when the rest of the `Response` + /// is no longer needed — e.g. the ~1.5 MB player.js download, where + /// `response_body().to_string()` would clone the whole buffer only to + /// drop the original (S8 speed fix). + pub fn into_body(self) -> String { + self.response_body + } + pub fn latest_url(&self) -> &str { &self.latest_url } diff --git a/src/exceptions.rs b/src/exceptions.rs index acdb096..cd19946 100644 --- a/src/exceptions.rs +++ b/src/exceptions.rs @@ -83,7 +83,12 @@ pub enum ExtractionError { impl From for NetworkError { fn from(e: reqwest::Error) -> Self { - NetworkError::Transport(e.to_string()) + // Privacy: reqwest's Display appends " for url (…)" whenever the + // error carries a URL — for player requests that URL contains + // `id=`, i.e. what the user was watching. Error strings + // become Kotlin exception messages and land in logcat / exported + // logs, so strip the URL at this single choke point. + NetworkError::Transport(e.without_url().to_string()) } } diff --git a/src/newpipe.rs b/src/newpipe.rs index 3fcb31c..0b9ead4 100644 --- a/src/newpipe.rs +++ b/src/newpipe.rs @@ -66,3 +66,10 @@ impl NewPipe { *Self::instance().preferred_content_country.write() = content_country; } } + +/// Serializes every test that installs/uses a Downloader in the process-global +/// `NewPipe` singleton, so they don't clobber each other's registered +/// downloader (or corrupt request counts) when cargo runs tests in parallel. +/// Shared by the `player_manager` and `stream_helper` test modules. +#[cfg(test)] +pub(crate) static DOWNLOADER_TEST_LOCK: parking_lot::Mutex<()> = parking_lot::Mutex::new(()); diff --git a/src/stream/audio.rs b/src/stream/audio.rs index 079e806..d40e1c0 100644 --- a/src/stream/audio.rs +++ b/src/stream/audio.rs @@ -3,6 +3,21 @@ use crate::stream::DeliveryMethod; use crate::youtube::itag::MediaFormat; +/// The track type of an [`AudioStream`]. Mirrors NPE +/// `stream/AudioTrackType.java`; derived from the format's `xtags` protobuf +/// (`acont` key) — see `youtube::xtags`. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] +pub enum AudioTrackType { + /// The original audio track of a video. + Original, + /// The original voices replaced, typically in a different language. + Dubbed, + /// A descriptive (audio-description) track for accessibility. + Descriptive, + /// A secondary track (e.g. an alternate/commentary track). + Secondary, +} + #[derive(Clone, Debug)] pub struct AudioStream { pub itag: u32, @@ -15,6 +30,12 @@ pub struct AudioStream { pub audio_track_id: Option, pub audio_track_name: Option, pub audio_locale: Option, + /// True iff this is a descriptive (audio-description) track. Preferentially + /// derived from `xtags` (`acont == descriptive`); falls back to the legacy + /// `audioTrack.audioIsDefault` heuristic when `xtags` is absent/unparseable. pub is_descriptive: bool, + /// The authoritative audio track type from the format's `xtags` blob, or + /// `None` for single-track audio / when `xtags` is absent or unparseable. + pub track_type: Option, pub itag_url_format: Option, } diff --git a/src/stream/mod.rs b/src/stream/mod.rs index 9ee9b28..c848cc7 100644 --- a/src/stream/mod.rs +++ b/src/stream/mod.rs @@ -10,7 +10,7 @@ pub mod delivery; pub mod subtitles; pub mod video; -pub use audio::AudioStream; +pub use audio::{AudioStream, AudioTrackType}; pub use delivery::DeliveryMethod; pub use subtitles::SubtitlesStream; pub use video::VideoStream; diff --git a/src/youtube/client_request.rs b/src/youtube/client_request.rs index 55577f4..15058bf 100644 --- a/src/youtube/client_request.rs +++ b/src/youtube/client_request.rs @@ -111,6 +111,29 @@ impl InnertubeClientRequestInfo { }, } } + + /// visionOS client (client 101). Mirrors NPE + /// `InnertubeClientRequestInfo.ofVisionOsClient()`. Same MOBILE/Apple shape + /// as iOS but with the RealityDevice model + visionOS os fields. + pub fn of_visionos_client() -> Self { + Self { + client_info: ClientInfo { + client_name: VISIONOS_CLIENT_NAME.into(), + client_version: VISIONOS_CLIENT_VERSION.into(), + client_id: VISIONOS_CLIENT_ID.into(), + client_screen: Some(WATCH_CLIENT_SCREEN.into()), + visitor_data: None, + }, + device_info: DeviceInfo { + platform: Some(MOBILE_CLIENT_PLATFORM.into()), + device_make: Some("Apple".into()), + device_model: Some(VISIONOS_DEVICE_MODEL.into()), + os_name: Some("visionOS".into()), + os_version: Some(VISIONOS_OS_VERSION.into()), + android_sdk_version: -1, + }, + } + } } /// Builds the InnerTube request envelope mirroring NPE prepareJsonBuilder. @@ -275,6 +298,30 @@ mod tests { assert!(client.get("androidSdkVersion").is_none()); } + #[test] + fn visionos_client_envelope_shape() { + let info = InnertubeClientRequestInfo::of_visionos_client(); + let env = build_envelope( + &info, + &Localization::default(), + &ContentCountry::default(), + None, + ); + let client = &env["context"]["client"]; + assert_eq!(client["clientName"], "VISIONOS"); + assert_eq!(client["clientVersion"], "1.02"); + assert_eq!(client["platform"], "MOBILE"); + assert_eq!(client["deviceMake"], "Apple"); + assert_eq!(client["deviceModel"], "RealityDevice14,1"); + assert_eq!(client["osName"], "visionOS"); + assert_eq!(client["osVersion"], "25.6.0.23O471"); + assert_eq!(client["clientScreen"], "WATCH"); + // no androidSdkVersion for an Apple client + assert!(client.get("androidSdkVersion").is_none()); + // best-effort visitorData not set at envelope-build time + assert!(client.get("visitorData").is_none()); + } + #[test] fn embed_url_lands_in_third_party_block() { let info = InnertubeClientRequestInfo::of_web_embedded_player_client(); diff --git a/src/youtube/constants.rs b/src/youtube/constants.rs index b5e2ac4..cbf4d46 100644 --- a/src/youtube/constants.rs +++ b/src/youtube/constants.rs @@ -36,6 +36,18 @@ pub const ANDROID_CLIENT_VERSION: &str = "21.03.36"; pub const ANDROID_SDK_VERSION: u32 = 36; pub const ANDROID_OS_VERSION: &str = "16"; +// visionOS client (client 101). A best-effort 4th /player fetch that recovers +// >360p streams for SABR-only / "made for kids" videos where the ANDROID/IOS +// clients cap at a single 360p muxed stream. Values from NPE +// ClientsConstants.java (commit 82b7e410, "Workaround again SABR-only +// responses"). Carries NO poToken. +pub const VISIONOS_CLIENT_ID: &str = "101"; +pub const VISIONOS_CLIENT_NAME: &str = "VISIONOS"; +pub const VISIONOS_CLIENT_VERSION: &str = "1.02"; +pub const VISIONOS_DEVICE_MODEL: &str = "RealityDevice14,1"; +pub const VISIONOS_OS_VERSION: &str = "25.6.0.23O471"; +pub const VISIONOS_USER_AGENT_VERSION: &str = "25_6_0"; + // Base URLs (NPE YoutubeParsingHelper.java:91,96). pub const YOUTUBEI_V1_URL: &str = "https://www.youtube.com/youtubei/v1/"; pub const YOUTUBEI_V1_GAPIS_URL: &str = "https://youtubei.googleapis.com/youtubei/v1/"; diff --git a/src/youtube/js/extractor.rs b/src/youtube/js/extractor.rs index 3ec0883..27515aa 100644 --- a/src/youtube/js/extractor.rs +++ b/src/youtube/js/extractor.rs @@ -40,15 +40,28 @@ static SCRIPT_TAG: Lazy = /// Extracts the player.js URL + body. Tries iframe_api first, falls back /// to the embed page on any failure (matches NPE's try/catch flow). pub fn extract_javascript_player_code(video_id: &str) -> Result<(String, String), DeobfError> { - let downloader = NewPipe::downloader().ok_or(DeobfError::DownloaderMissing)?; + let url = extract_javascript_player_url(video_id)?; + let body = download_player_code(&url)?; + Ok((url, body)) +} +/// Discovers the CURRENT player.js URL only (a few-KB iframe_api fetch — +/// cheap next to the ~1.7 MB body). Split out so PlayerManager's failure +/// memo can probe "did YouTube rotate player.js?" without paying for a +/// body download that would deterministically re-fail extraction. +pub fn extract_javascript_player_url(video_id: &str) -> Result { + let downloader = NewPipe::downloader().ok_or(DeobfError::DownloaderMissing)?; let url = match extract_from_iframe(&*downloader) { Ok(u) => u, Err(_iframe_err) => extract_from_embed(&*downloader, video_id)?, }; - let cleaned = clean_javascript_url(&url)?; - let body = download_javascript_code(&*downloader, &cleaned)?; - Ok((cleaned, body)) + clean_javascript_url(&url) +} + +/// Downloads the player.js body for an already-discovered URL. +pub fn download_player_code(url: &str) -> Result { + let downloader = NewPipe::downloader().ok_or(DeobfError::DownloaderMissing)?; + download_javascript_code(&*downloader, url) } fn extract_from_iframe(downloader: &dyn Downloader) -> Result { @@ -123,7 +136,9 @@ fn download_javascript_code(downloader: &dyn Downloader, url: &str) -> Result Result // NPE's `groupCount()` excludes group 0, so: // len() == 2 → 1 capture → direct name // len() == 3 → 2 captures → array indirection + // + // BUT: `caps.len()` is pattern-static — it counts DECLARED groups, + // not the ones that PARTICIPATED in the match. Regexes 6/7 declare + // the array-access group inside an optional `(?:@ARRAY@)?`, so a + // direct-call player.js shape matches with group 2 absent. Route on + // participation: group 2 unmatched → the group-1 name IS the answer. + // (NPE's Java would NPE on `Integer.parseInt(matcher.group(2))` here + // — a latent upstream bug; the optional group's evident intent, and + // yt-dlp's guarded equivalent, is the direct name.) match caps.len() { 2 => { if let Some(m) = caps.get(1) { @@ -119,8 +128,14 @@ fn deobfuscation_function_name(player_code: &str) -> Result } } 3 => { + let Some(index_m) = caps.get(2) else { + if let Some(m) = caps.get(1) { + return Ok(m.as_str().to_string()); + } + continue; + }; let array_name = caps.get(1).map(|m| m.as_str()).unwrap_or_default(); - let index_str = caps.get(2).map(|m| m.as_str()).unwrap_or_default(); + let index_str = index_m.as_str(); let index: usize = index_str.parse().map_err(|_| { DeobfError::NsigArrayLookupFailed(format!("bad index: {index_str}")) })?; @@ -185,8 +200,18 @@ fn deobfuscation_function_body_regex( /// Strips `if(typeof X==="undefined")return ;` so the function /// actually runs standalone. NPE adds this 2024-12-29 (`56595bd9d`). +/// +/// Must handle BOTH body shapes this crate produces: +/// * lexer path: `name=function(a){…};` +/// * regex-fallback path: `function name(a){…}` +/// +/// NPE's FUNCTION_ARGUMENTS_REGEX requires the `=` (its regex-fallback +/// shape would throw from matchGroup1); ours accepts both so the fallback +/// path doesn't silently no-op — a surviving guard makes the function +/// return its input unchanged, which then gets CACHED as the deobfuscated +/// n-param → permanent silent throttling. pub fn fixup_function(function: &str) -> Result { - let args_re = Regex::new(r"=\s*function\s*\(\s*([^)]*)\s*\)") + let args_re = Regex::new(r"(?:=\s*)?function\s*[a-zA-Z0-9$_]*\s*\(\s*([^)]*)\s*\)") .map_err(|e| DeobfError::NsigBodyParseFailed(e.to_string()))?; let first_arg = args_re .captures(function) @@ -203,13 +228,18 @@ pub fn fixup_function(function: &str) -> Result { // Substitute with an alternation of fully-quoted `"undefined"` / // `'undefined'` forms. Loosens slightly (allows `"undefined'`) but // real player.js always uses balanced quotes; harmless. + // + // NPE's EARLY_RETURN_REGEX anchors on a preceding `;` only. When the + // guard is the FIRST statement of the body (regex-fallback shape: + // `function name(a){if(typeof X==="undefined")return a;…}`) the + // preceding token is `{` — accept both and re-emit whichever matched. let early_return_re_src = format!( - r#"(?s);\s*if\s*\(\s*typeof\s+[a-zA-Z0-9$_]+\s*===?\s*(?:"undefined"|'undefined')\s*\)\s*return\s+{};"#, + r#"(?s)([;{{])\s*if\s*\(\s*typeof\s+[a-zA-Z0-9$_]+\s*===?\s*(?:"undefined"|'undefined')\s*\)\s*return\s+{};"#, regex::escape(&first_arg) ); let er_re = Regex::new(&early_return_re_src) .map_err(|e| DeobfError::NsigBodyParseFailed(e.to_string()))?; - Ok(er_re.replace(function, ";").to_string()) + Ok(er_re.replace(function, "$1").to_string()) } #[cfg(test)] @@ -286,4 +316,63 @@ mod tests { Err(e) => panic!("expected name match, got {e:?}"), } } + + #[test] + fn regex_6_direct_call_without_array_access() { + // Regex 6 (String.fromCharCode(110)) with a DIRECT call — the + // optional `(?:\[(\d+)])?` group does not participate. Must route + // to the group-1 name, not the array-indirection branch (which + // would hard-fail with NsigArrayLookupFailed on the empty index). + let src = r#"WL=function(a){a.j=1};(b=String.fromCharCode(110),c=a.get(b))&&(c=mfn(c),a.set(b,c))"#; + match deobfuscation_function_name(src) { + Ok(n) => assert_eq!(n, "mfn"), + Err(e) => panic!("direct-call shape must yield the direct name, got {e:?}"), + } + } + + #[test] + fn regex_7_direct_call_without_array_access() { + // Regex 7 (.get("n")) with a direct call — same non-participating + // optional group. + let src = r#"(c=d.get("n"))&&(e=Nfn(e),d.set("n",e))"#; + match deobfuscation_function_name(src) { + Ok(n) => assert_eq!(n, "Nfn"), + Err(e) => panic!("direct-call shape must yield the direct name, got {e:?}"), + } + } + + #[test] + fn fixup_strips_guard_on_function_declaration_shape() { + // The regex-fallback body shape: `function name(a){…}` — no `=` + // before `function`, and the guard sits right after `{` with no + // preceding `;`. The old fixup no-op'd on both counts, letting the + // early return survive → nsig returns its input unchanged → the + // identity result gets cached → permanent silent throttling. + let body = r#"function m85(p){if(typeof RUQ==="undefined")return p;var a=p.split("");a.reverse();return a.join("");}"#; + let fixed = fixup_function(body).unwrap(); + assert!( + !fixed.contains("typeof RUQ"), + "guard must be stripped on the function-declaration shape, got: {fixed}" + ); + assert!(fixed.contains(r#"var a=p.split("");"#)); + } + + #[test] + fn fixup_function_declaration_shape_runs_non_identity() { + // End-to-end: the fixed fallback-shape body must actually transform + // its input when run (i.e. the guard is gone, not just renamed). + let body = r#"function m85(p){if(typeof RUQ==="undefined")return p;var a=p.split("");a.reverse();return a.join("");}"#; + let fixed = fixup_function(body).unwrap(); + let out = crate::youtube::js::runtime::run(&fixed, "m85", "abc123").unwrap(); + assert_eq!(out, "321cba"); + } + + #[test] + fn fixup_lexer_shape_guard_at_start_of_body() { + // Lexer shape (`name=function(...)`) whose guard is the FIRST + // statement — preceded by `{`, not `;`. Also must strip. + let body = r#"m85=function(p){if(typeof RUQ==="undefined")return p;var a=p.split("");return a.join("");}"#; + let fixed = fixup_function(body).unwrap(); + assert!(!fixed.contains("typeof RUQ")); + } } diff --git a/src/youtube/js/player_manager.rs b/src/youtube/js/player_manager.rs index 9c233dd..9dc65e9 100644 --- a/src/youtube/js/player_manager.rs +++ b/src/youtube/js/player_manager.rs @@ -3,27 +3,60 @@ // sole public class in the JS subsystem). // // Cache layout: -// * cached_player_code — process-lifetime, until invalidate() +// * cached_player_code — the raw ~1.5 MB player.js. Retained only +// until the hot-path snippets (signature timestamp + nsig) are built, +// then DROPPED to reclaim native heap (S7); `player_url` remains as the +// installed-generation sentinel. Re-fetched on demand if a later, +// un-built artifact (the dead-in-android sig path) ever needs the bytes. // * cached_signature_timestamp // * cached_sig_snippet — assembled JS, ready for runtime::run // * cached_nsig_name + snippet // * cached_throttling_params — obfuscated → deobfuscated cache (per-session) // -// Failure handling: on ANY deobf/extraction error, invalidate() drops the -// fetched player.js and everything derived from it, so the NEXT call re-fetches -// a fresh player.js and retries. YouTube rotates player.js every few hours, so -// a failure is far more often "our cached player.js went stale" than "this -// player.js is unparseable" — invalidating lets the next attempt self-heal. -// (This replaces the previous sticky-error flags, which were set on failure and -// never cleared by any code path, permanently wedging ALL extraction for the -// process lifetime after a single routine rotation. straw audit, 2026-07-04.) +// Failure handling — three cases, each tuned so a single player.js rotation +// self-heals while a persistently-unparseable player.js can't trigger a +// per-format refetch storm (~20-25 × ~1.7 MB per video open — the H2 finding +// of the 2026-07-28 audit; the previous invalidate-on-any-error design had no +// memory of "I just refetched and it STILL failed"): // -// NPE uses static fields and is not thread-safe; callers serialize. We give the -// same shape via a `Mutex` — call sites can still hammer it from +// * Artifact extraction fails (regex bank miss / body parse): arm a +// per-artifact failure memo keyed by the player_url that failed — +// mirrors NPE's cached ParsingException fields (sigTimestampExtractionEx +// et al.), except NPE's are sticky until clearAllCaches() while ours +// expire after FAILURE_COOLDOWN. The installed player.js is KEPT so +// sibling artifacts (e.g. a working signature timestamp) stay usable. +// While the memo is live the original error is replayed with zero +// network. After the cooldown, discovery re-runs (a few-KB iframe_api +// probe): a rotated player.js is fetched fresh and retried; an +// unchanged one just re-arms the memo — extraction is deterministic, +// so re-downloading ~1.7 MB to re-fail is pure waste. +// +// * Eval (QuickJS run) fails: invalidate() everything so the next call +// re-fetches — the stale-rotation self-heal (a cached player.js whose +// function chokes on a new-generation n-param is fixed by fetching the +// current player.js). If the snippet came from a player.js downloaded +// by THIS call, a refetch would rebuild the identical snippet, so the +// memo is armed instead — bounding the storm to one refetch per open. +// +// * Fetch fails (network): plain error, no memo — transient by nature. +// +// Locking: `state` (Mutex) is NEVER held across the network. +// `fetch_gate` serializes player.js downloads so concurrent extractions +// don't each re-download (~1.7 MB); waiters re-check `state` after taking +// the gate and find the code already installed. Lock order is always +// fetch_gate → state; state is released before any fetch. (The previous +// design held the state mutex across ensure_player_code's downloads, +// serializing every concurrent extraction behind up-to-30 s requests — +// the S6 finding.) +// +// NPE uses static fields and is not thread-safe; callers serialize. We give +// the same shape via the state mutex — call sites can still hammer it from // multiple threads safely. use parking_lot::Mutex; use std::collections::HashMap; +use std::sync::Arc; +use std::time::{Duration, Instant}; use crate::youtube::js::extractor; use crate::youtube::js::nsig; @@ -31,23 +64,159 @@ use crate::youtube::js::runtime; use crate::youtube::js::signature; use crate::youtube::js::DeobfError; +/// How long a failure memo suppresses re-fetch/re-parse attempts for the +/// player.js that produced it. YouTube rotates player.js every few hours, +/// so five minutes bounds the retry storm without meaningfully delaying +/// recovery once a genuinely new player.js ships (recovery is also +/// immediate whenever a different player_url is discovered). +const FAILURE_COOLDOWN: Duration = Duration::from_secs(5 * 60); + +/// One derived artifact's failure record. `player_url` pins the player.js +/// generation the failure belongs to: a different discovered URL retires +/// the memo instantly (rotation self-heal), an identical one replays +/// `error` without re-downloading (extraction is deterministic). +struct FailMemo { + player_url: String, + error: DeobfError, + at: Instant, +} + +/// The three derived artifacts, each with its own cache + failure memo — +/// mirrors NPE's three cached exception fields. Kept independent so e.g. +/// a signature-timestamp regex miss doesn't block nsig extraction that +/// would succeed on the same player.js. +#[derive(Clone, Copy)] +enum Derived { + SigTimestamp, + Sig, + Nsig, +} + +impl Derived { + fn memo<'a>(&self, state: &'a ManagerState) -> &'a Option { + match self { + Derived::SigTimestamp => &state.ts_fail, + Derived::Sig => &state.sig_fail, + Derived::Nsig => &state.nsig_fail, + } + } + + fn memo_mut<'a>(&self, state: &'a mut ManagerState) -> &'a mut Option { + match self { + Derived::SigTimestamp => &mut state.ts_fail, + Derived::Sig => &mut state.sig_fail, + Derived::Nsig => &mut state.nsig_fail, + } + } +} + +/// Verdict of a memo check, decided under the state lock. +enum MemoVerdict { + /// No live memo applies — proceed normally. + Proceed, + /// Memo cooldown expired — proceed, but treat this URL as known-bad: + /// discover first and skip the ~1.7 MB body download if it's unchanged. + Probe(String), + /// Memo is live — replay the memoized error, zero work. + Replay(DeobfError), +} + +fn memo_verdict(state: &ManagerState, memo: &Option) -> MemoVerdict { + let Some(m) = memo else { + return MemoVerdict::Proceed; + }; + let live = m.at.elapsed() < FAILURE_COOLDOWN; + // Installed-generation sentinel: `player_url` marks WHICH player.js is + // installed, set/cleared in lockstep with install/invalidate and keyed the + // same way the memos are. It is DECOUPLED from `player_code` (the raw bytes), + // which S7 drops once the hot-path snippets are built — so a byte-released + // player.js is still "installed" for memo purposes. (ensure_player_code keeps + // keying off `player_code` because IT needs the bytes to build from.) + let installed_same = state.player_url.as_deref() == Some(m.player_url.as_str()); + if installed_same { + if live { + MemoVerdict::Replay(m.error.clone()) + } else { + MemoVerdict::Probe(m.player_url.clone()) + } + } else if state.player_url.is_some() { + // A different player.js has been installed since the failure — + // retry this artifact against it. + MemoVerdict::Proceed + } else if live { + MemoVerdict::Replay(m.error.clone()) + } else { + MemoVerdict::Probe(m.player_url.clone()) + } +} + +/// S7: reclaim the raw ~1.5 MB player.js once the hot-path artifacts are built. +/// +/// The bytes exist only to build the derived snippets. The android-primary flow +/// needs exactly two — the signature timestamp and the nsig deobfuscator — and +/// both are always built per open (timestamp by `stream_info_with`, nsig by the +/// first throttled format). Once both are present, every later call hits the +/// snippet/timestamp cache and never touches the bytes again, so we drop them +/// and keep `player_url` as the installed-generation sentinel. +/// +/// Deliberately keyed on {timestamp, nsig} and NOT on the sig snippet: the sig +/// path is dead in the android-primary flow (formats carry direct URLs), so +/// requiring it would pin the bytes forever. If that dead path IS ever hit after +/// a release, `ensure_player_code` (which keys off `player_code`) simply +/// re-fetches — the audit-sanctioned worst case. Called ONLY on a clean build +/// success; a build FAILURE keeps the bytes so sibling artifacts can still +/// extract from them. +fn release_player_code_if_built(state: &mut ManagerState) { + if state.signature_timestamp.is_some() && state.nsig_snippet.is_some() { + state.player_code = None; + } +} + +/// Outcome of ensure_player_code. +enum Ensured { + /// player_code is installed; `fresh` = this call downloaded it. + Ready { fresh: bool }, + /// The probe found YouTube still serving the caller's known-bad URL — + /// no body was downloaded. + StillBad, +} + #[derive(Default)] struct ManagerState { + // `player_url` is the installed-GENERATION sentinel — Some iff a player.js + // has been discovered+installed and its derived snippets are the current + // generation. `player_code` is the raw ~1.5 MB bytes, dropped by + // `release_player_code_if_built` once the hot-path snippets exist (S7), so + // it can be None while `player_url` is Some. memo/rotation logic keys off + // `player_url`; only `ensure_player_code` (which needs to build) keys off + // `player_code`, re-fetching when the bytes are gone. player_url: Option, player_code: Option, signature_timestamp: Option, - sig_snippet: Option, - nsig_name: Option, - nsig_snippet: Option, + // Snippets held as `Arc` so the per-eval hand-off out of the state + // lock is a pointer bump instead of a ~30-120 KB `String` copy (S10). Only + // the cloned payload type changes — the lock/memo/fetch_gate discipline is + // exactly as Loop-1 left it. + sig_snippet: Option>, + nsig_name: Option>, + nsig_snippet: Option>, throttling_param_cache: HashMap, + + // Failure memos — deliberately NOT cleared by invalidate(): they must + // survive the invalidate that accompanies the failure they record. + // Retired by: a different player_url being installed, their artifact + // building successfully, clear_all_caches(), or cooldown expiry. + ts_fail: Option, + sig_fail: Option, + nsig_fail: Option, } impl ManagerState { /// Drop the fetched player.js and everything derived from it, so the next - /// call re-fetches a fresh player.js and re-extracts. Called on any - /// deobf/extraction failure so a stale-player.js error self-heals on retry - /// instead of wedging extraction for the process lifetime. + /// call re-fetches a fresh player.js and re-extracts. Called on eval + /// failures (stale-player.js self-heal) and before installing a newly + /// downloaded player.js. Failure memos are kept — see the field comment. fn invalidate(&mut self) { self.player_url = None; self.player_code = None; @@ -62,13 +231,17 @@ impl ManagerState { } pub struct PlayerManager { - inner: Mutex, + state: Mutex, + /// Serializes player.js downloads WITHOUT blocking `state` readers. + /// Held only while discovering/downloading; never while `state` is held. + fetch_gate: Mutex<()>, } impl PlayerManager { pub fn new() -> Self { Self { - inner: Mutex::new(ManagerState::default()), + state: Mutex::new(ManagerState::default()), + fetch_gate: Mutex::new(()), } } @@ -79,23 +252,14 @@ impl PlayerManager { } pub fn signature_timestamp(&self, video_id: &str) -> Result { - let r = self.signature_timestamp_inner(video_id); - if r.is_err() { - self.inner.lock().invalidate(); - } - r - } - - fn signature_timestamp_inner(&self, video_id: &str) -> Result { - let mut state = self.inner.lock(); - if let Some(ts) = state.signature_timestamp { - return Ok(ts); - } - Self::ensure_player_code(&mut state, video_id)?; - let code = state.player_code.as_deref().unwrap(); - let ts = signature::signature_timestamp(code)?; - state.signature_timestamp = Some(ts); - Ok(ts) + self.derived_artifact( + video_id, + Derived::SigTimestamp, + |state| state.signature_timestamp, + signature::signature_timestamp, + |state, ts| state.signature_timestamp = Some(*ts), + ) + .map(|(ts, _)| ts) } pub fn deobfuscate_signature( @@ -103,56 +267,37 @@ impl PlayerManager { video_id: &str, obfuscated_signature: &str, ) -> Result { - let r = self.deobfuscate_signature_inner(video_id, obfuscated_signature); - if r.is_err() { - self.inner.lock().invalidate(); - } - r - } + let (snippet, fresh) = self.derived_artifact( + video_id, + Derived::Sig, + |state| state.sig_snippet.clone(), + |code| signature::build_deobfuscator(code).map(Arc::::from), + |state, snippet| state.sig_snippet = Some(snippet.clone()), + )?; - fn deobfuscate_signature_inner( - &self, - video_id: &str, - obfuscated_signature: &str, - ) -> Result { - let snippet = { - let mut state = self.inner.lock(); - if state.sig_snippet.is_none() { - Self::ensure_player_code(&mut state, video_id)?; - let code = state.player_code.as_deref().unwrap(); - let s = signature::build_deobfuscator(code)?; - state.sig_snippet = Some(s); - } - state.sig_snippet.clone().unwrap() - }; - - let result = runtime::run( + match runtime::run( &snippet, signature::DEOBFUSCATION_FUNCTION_NAME, obfuscated_signature, - )?; - if result == "null" { - return Ok(String::new()); // NPE: Objects.requireNonNullElse(..., "") + ) { + Ok(result) => { + if result == "null" { + Ok(String::new()) // NPE: Objects.requireNonNullElse(..., "") + } else { + Ok(result) + } + } + Err(e) => { + self.on_eval_failure(Derived::Sig, fresh, &e); + Err(e) + } } - Ok(result) } pub fn url_with_throttling_parameter_deobfuscated( &self, video_id: &str, streaming_url: &str, - ) -> Result { - let r = self.url_with_throttling_parameter_deobfuscated_inner(video_id, streaming_url); - if r.is_err() { - self.inner.lock().invalidate(); - } - r - } - - fn url_with_throttling_parameter_deobfuscated_inner( - &self, - video_id: &str, - streaming_url: &str, ) -> Result { let obf = match nsig::throttling_parameter_from_url(streaming_url) { Some(s) => s, @@ -160,34 +305,44 @@ impl PlayerManager { }; { - let state = self.inner.lock(); + let state = self.state.lock(); if let Some(cached) = state.throttling_param_cache.get(&obf) { return Ok(streaming_url.replace(&obf, cached)); } } - let (name, snippet) = { - let mut state = self.inner.lock(); - if state.nsig_snippet.is_none() { - Self::ensure_player_code(&mut state, video_id)?; - let code = state.player_code.as_deref().unwrap(); - let (n, s) = nsig::build_deobfuscator(code)?; - state.nsig_name = Some(n); - state.nsig_snippet = Some(s); - } - ( - state.nsig_name.clone().unwrap(), - state.nsig_snippet.clone().unwrap(), - ) - }; + let ((name, snippet), fresh) = self.derived_artifact( + video_id, + Derived::Nsig, + |state| match (&state.nsig_name, &state.nsig_snippet) { + (Some(n), Some(s)) => Some((n.clone(), s.clone())), + _ => None, + }, + |code| { + nsig::build_deobfuscator(code) + .map(|(n, s)| (Arc::::from(n), Arc::::from(s))) + }, + |state, (n, s)| { + state.nsig_name = Some(n.clone()); + state.nsig_snippet = Some(s.clone()); + }, + )?; - let deobf = runtime::run(&snippet, &name, &obf)?; + let deobf = match runtime::run(&snippet, &name, &obf) { + Ok(d) => d, + Err(e) => { + self.on_eval_failure(Derived::Nsig, fresh, &e); + return Err(e); + } + }; if deobf.is_empty() { - return Err(DeobfError::NsigEmpty); + let e = DeobfError::NsigEmpty; + self.on_eval_failure(Derived::Nsig, fresh, &e); + return Err(e); } { - let mut state = self.inner.lock(); + let mut state = self.state.lock(); state .throttling_param_cache .insert(obf.clone(), deobf.clone()); @@ -197,29 +352,240 @@ impl PlayerManager { } pub fn throttling_parameter_cache_size(&self) -> usize { - self.inner.lock().throttling_param_cache.len() + self.state.lock().throttling_param_cache.len() } pub fn clear_all_caches(&self) { - self.inner.lock().invalidate(); + let mut state = self.state.lock(); + state.invalidate(); + state.ts_fail = None; + state.sig_fail = None; + state.nsig_fail = None; } pub fn clear_throttling_parameters_cache(&self) { - self.inner.lock().throttling_param_cache.clear(); + self.state.lock().throttling_param_cache.clear(); } pub fn player_url(&self) -> Option { - self.inner.lock().player_url.clone() + self.state.lock().player_url.clone() } - fn ensure_player_code(state: &mut ManagerState, video_id: &str) -> Result<(), DeobfError> { - if state.player_code.is_some() { - return Ok(()); + /// Shared spine for the three derived artifacts: cache fast path → + /// failure-memo verdict → make player code available (fetching outside + /// the state lock) → build + store under the lock, arming the memo on + /// extraction failure. Returns the artifact plus whether the player.js + /// it was built from was downloaded by THIS call (`fresh` — feeds the + /// eval-failure policy). + fn derived_artifact( + &self, + video_id: &str, + which: Derived, + cached: impl Fn(&ManagerState) -> Option, + build: impl Fn(&str) -> Result, + store: impl Fn(&mut ManagerState, &T), + ) -> Result<(T, bool), DeobfError> { + // Two passes: the second absorbs the narrow races (concurrent + // invalidate between ensure and the build lock; memo retired by + // clear_all_caches between verdict and probe). + for _ in 0..2 { + let known_bad: Option = { + let state = self.state.lock(); + if let Some(v) = cached(&state) { + return Ok((v, false)); + } + match memo_verdict(&state, which.memo(&state)) { + MemoVerdict::Replay(e) => return Err(e), + MemoVerdict::Probe(url) => Some(url), + MemoVerdict::Proceed => None, + } + }; + + let ensured = match self.ensure_player_code(video_id, known_bad.as_deref()) { + Ok(e) => e, + Err(fetch_err) => { + if known_bad.is_some() { + // The post-cooldown probe itself failed (network). + // Re-arm the cooldown so we don't probe on every + // subsequent call while the network is down. + let mut state = self.state.lock(); + if let Some(m) = which.memo_mut(&mut state).as_mut() { + m.at = Instant::now(); + } + } + return Err(fetch_err); + } + }; + let fresh = match ensured { + Ensured::Ready { fresh } => fresh, + Ensured::StillBad => { + // YouTube still serves the player.js this artifact + // failed on — replay the original error and restart + // the cooldown. + let mut state = self.state.lock(); + match which.memo_mut(&mut state).as_mut() { + Some(m) => { + m.at = Instant::now(); + return Err(m.error.clone()); + } + // Memo retired concurrently (clear_all_caches) — + // loop and take the fresh-slate path. + None => continue, + } + } + }; + + let mut state = self.state.lock(); + let built = match state.player_code.as_deref() { + Some(code) => build(code), + // Invalidated between ensure and here — loop to re-fetch. + None => continue, + }; + return match built { + Ok(v) => { + store(&mut state, &v); + // A successful build proves any lingering memo for this + // artifact records a superseded player.js. + *which.memo_mut(&mut state) = None; + // S7: once the hot-path snippets exist, the raw player.js + // bytes are dead weight — release them (keeping player_url). + release_player_code_if_built(&mut state); + Ok((v, fresh)) + } + Err(e) => { + // Extraction failed. Keep the installed player.js — + // sibling artifacts may extract fine from it — and arm + // the memo so every further call within the cooldown + // replays instead of re-fetching ~1.7 MB per format. + let player_url = state.player_url.clone().unwrap_or_default(); + *which.memo_mut(&mut state) = Some(FailMemo { + player_url, + error: e.clone(), + at: Instant::now(), + }); + Err(e) + } + }; + } + Err(DeobfError::FetchPlayerCode( + "player.js unavailable after retry (concurrent cache invalidation)".into(), + )) + } + + /// Eval (QuickJS) failures: invalidate so the next call re-fetches — + /// the stale-rotation self-heal (a cached player.js whose function + /// chokes on a new-generation n-param is fixed by fetching the current + /// player.js). When the failing snippet came from a player.js this very + /// call downloaded (`fresh`), a refetch would rebuild the identical + /// snippet — arm the memo instead so the remaining ~20-25 formats of + /// the open replay the error instead of each re-downloading. + fn on_eval_failure(&self, which: Derived, fresh: bool, error: &DeobfError) { + let mut state = self.state.lock(); + let player_url = state.player_url.clone().unwrap_or_default(); + state.invalidate(); + if fresh { + *which.memo_mut(&mut state) = Some(FailMemo { + player_url, + error: error.clone(), + at: Instant::now(), + }); + } + } + + /// Makes `state.player_code` available. The network work happens with + /// only `fetch_gate` held — never the state lock — so cache-hit readers + /// stay unblocked and concurrent fetchers collapse into one download. + /// + /// `known_bad_url`: the caller's memoized failing player URL. Discovery + /// still runs, but if YouTube is serving that exact URL the ~1.7 MB body + /// download is skipped (`StillBad`) — extraction is deterministic, so it + /// could only re-fail. + fn ensure_player_code( + &self, + video_id: &str, + known_bad_url: Option<&str>, + ) -> Result { + // Fast path — usable code already installed. + { + let state = self.state.lock(); + if state.player_code.is_some() + && (known_bad_url.is_none() || state.player_url.as_deref() != known_bad_url) + { + return Ok(Ensured::Ready { fresh: false }); + } + } + + // One fetcher at a time. Waiters block HERE (not on `state`), then + // usually find the winner's code installed. + let _gate = self.fetch_gate.lock(); + { + let state = self.state.lock(); + if state.player_code.is_some() + && (known_bad_url.is_none() || state.player_url.as_deref() != known_bad_url) + { + return Ok(Ensured::Ready { fresh: false }); + } + } + + // Discovery (iframe_api / embed fallback) — a few KB. + let url = extractor::extract_javascript_player_url(video_id)?; + if Some(url.as_str()) == known_bad_url { + return Ok(Ensured::StillBad); + } + // Body download — ~1.7 MB. + let code = extractor::download_player_code(&url)?; + + let mut state = self.state.lock(); + let state: &mut ManagerState = &mut state; + // Everything derived from any previously installed player.js dies + // with it; memos recorded against OTHER player URLs are retired so + // a genuinely new player.js gets a clean slate for every artifact. + state.invalidate(); + for memo in [ + &mut state.ts_fail, + &mut state.sig_fail, + &mut state.nsig_fail, + ] { + if memo.as_ref().is_some_and(|m| m.player_url != url) { + *memo = None; + } } - let (url, code) = extractor::extract_javascript_player_code(video_id)?; state.player_url = Some(url); state.player_code = Some(code); - Ok(()) + Ok(Ensured::Ready { fresh: true }) + } + + /// Test hook: age every armed memo past FAILURE_COOLDOWN so the probe + /// path can be exercised without a five-minute sleep. + #[cfg(test)] + fn test_expire_memos(&self) { + let mut state = self.state.lock(); + let state: &mut ManagerState = &mut state; + for memo in [ + &mut state.ts_fail, + &mut state.sig_fail, + &mut state.nsig_fail, + ] { + if let Some(m) = memo.as_mut() { + if let Some(aged) = Instant::now().checked_sub(FAILURE_COOLDOWN) { + m.at = aged; + } + } + } + } + + /// Test hook: is the state mutex currently free? Used to prove the + /// player.js fetch runs without holding it. + #[cfg(test)] + fn state_lock_is_free(&self) -> bool { + self.state.try_lock().is_some() + } + + /// Test hook: are the raw player.js bytes still retained? (S7 drops them + /// once the hot-path snippets are built, keeping `player_url`.) + #[cfg(test)] + fn player_code_is_retained(&self) -> bool { + self.state.lock().player_code.is_some() } } @@ -232,6 +598,13 @@ impl Default for PlayerManager { #[cfg(test)] mod tests { use super::*; + use crate::downloader::request::Request; + use crate::downloader::response::Response; + use crate::downloader::Downloader; + use crate::exceptions::NetworkError; + use crate::newpipe::NewPipe; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::{mpsc, Arc}; #[test] fn assembled_sig_snippet_runs_against_synthetic_player() { @@ -264,4 +637,359 @@ mod tests { mgr.clear_all_caches(); assert!(mgr.player_url().is_none()); } + + // ---- failure-memo / refetch-storm tests ------------------------------- + // + // These register a stub Downloader in the process-global NewPipe + // singleton, so they serialize on GLOBAL_DOWNLOADER_LOCK (no other lib + // test touches the downloader). Each uses its own PlayerManager + // instance, so manager state never crosses tests. + + // Shared with the stream_helper test module so downloader-touching tests + // across both modules serialize (they all mutate the same NewPipe global). + use crate::newpipe::DOWNLOADER_TEST_LOCK as GLOBAL_DOWNLOADER_LOCK; + + /// Both channel endpoints wrapped in Mutex for the Downloader Sync bound. + type FetchBlocker = (Mutex>, Mutex>); + + /// Serves a synthetic iframe_api (announcing `hash`) and a synthetic + /// player.js body; counts every request. + struct StubDownloader { + hash: Mutex, + player_js: Mutex, + requests: AtomicUsize, + /// When set, the player.js body request signals `entered` then + /// blocks until `release` fires — used by the lock-freedom test. + block_player_fetch: Option, + } + + impl StubDownloader { + fn new(hash: &str, player_js: &str) -> Self { + Self { + hash: Mutex::new(hash.to_string()), + player_js: Mutex::new(player_js.to_string()), + requests: AtomicUsize::new(0), + block_player_fetch: None, + } + } + + fn rotate(&self, hash: &str, player_js: &str) { + *self.hash.lock() = hash.to_string(); + *self.player_js.lock() = player_js.to_string(); + } + + fn count(&self) -> usize { + self.requests.load(Ordering::SeqCst) + } + } + + impl Downloader for StubDownloader { + fn execute(&self, request: Request) -> Result { + self.requests.fetch_add(1, Ordering::SeqCst); + let url = request.url().to_string(); + if url.contains("iframe_api") { + let h = self.hash.lock().clone(); + // Shape matched by IFRAME_RES_JS_BASE_PLAYER_HASH (escaped + // `\/` form, 8-char [a-z0-9] hash). + let body = format!( + r#"src:"https://www.youtube.com/s/player\/{h}\/player_ias.vflset/en_US/www-embed.js""# + ); + Ok(Response::new(200, "OK", Default::default(), body, url)) + } else if url.contains("/s/player/") { + if let Some((entered, release)) = &self.block_player_fetch { + let _ = entered.lock().send(()); + let _ = release + .lock() + .recv_timeout(std::time::Duration::from_secs(10)); + } + let body = self.player_js.lock().clone(); + Ok(Response::new(200, "OK", Default::default(), body, url)) + } else { + Err(NetworkError::Transport("stub: unexpected request".into())) + } + } + } + + // Parseable by ts + NOTHING nsig-shaped → nsig build fails while the + // signature timestamp extracts fine. + const PLAYER_NSIG_BROKEN: &str = r#"var foo={signatureTimestamp:19999};var unrelated=1;"#; + + // nsig name via regex 0 (m85 … the greedy `.*` reaches dummy's + // `return Q[1]` on the same line); body via lexer; self-contained + // (no outer-global references), so eval succeeds for any input. + const PLAYER_GOOD: &str = r#"m85=function(p){var a=p.split("");a.reverse();return a.join("");};dummy=function(q){return Q[1];};var foo={signatureTimestamp:20244};"#; + + // Parses fine but throws for EVERY input at eval time. + const PLAYER_EVAL_THROWS: &str = r#"m85=function(p){throw Error("boom");return Q[1];};"#; + + // Parses fine; throws only for the input "evil". + const PLAYER_EVAL_SELECTIVE: &str = r#"m85=function(p){if(p==="evil")throw Error("bad");var a=p.split("");a.reverse();return a.join("");};dummy=function(q){return Q[1];};"#; + + fn install(stub: &Arc) { + NewPipe::init(stub.clone() as Arc); + } + + #[test] + fn nsig_build_failure_memoizes_instead_of_refetching_per_format() { + let _g = GLOBAL_DOWNLOADER_LOCK.lock(); + let stub = Arc::new(StubDownloader::new("aaaaaaaa", PLAYER_NSIG_BROKEN)); + install(&stub); + let mgr = PlayerManager::new(); + + // First format: discovery + body download (2 requests), build fails. + let err = mgr + .url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=AAA") + .unwrap_err(); + assert!(matches!(err, DeobfError::NsigFuncNotFound), "got {err:?}"); + assert_eq!(stub.count(), 2); + + // Remaining formats of the same open: replayed error, ZERO network. + for _ in 0..5 { + let err = mgr + .url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=BBB") + .unwrap_err(); + assert!(matches!(err, DeobfError::NsigFuncNotFound)); + } + assert_eq!(stub.count(), 2, "memo must suppress refetching"); + } + + #[test] + fn artifact_memos_are_independent() { + let _g = GLOBAL_DOWNLOADER_LOCK.lock(); + let stub = Arc::new(StubDownloader::new("aaaaaaaa", PLAYER_NSIG_BROKEN)); + install(&stub); + let mgr = PlayerManager::new(); + + // nsig fails and memoizes… + assert!(mgr + .url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=AAA") + .is_err()); + assert_eq!(stub.count(), 2); + // …but the player.js stays installed, so the signature timestamp + // extracts from it without any further network. + assert_eq!(mgr.signature_timestamp("vid").unwrap(), 19999); + assert_eq!(stub.count(), 2); + } + + #[test] + fn rotation_after_cooldown_fetches_fresh_and_recovers() { + let _g = GLOBAL_DOWNLOADER_LOCK.lock(); + let stub = Arc::new(StubDownloader::new("aaaaaaaa", PLAYER_NSIG_BROKEN)); + install(&stub); + let mgr = PlayerManager::new(); + + assert!(mgr + .url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=AAA") + .is_err()); + assert_eq!(stub.count(), 2); + + // YouTube rotates to a player.js our bank parses. + stub.rotate("bbbbbbbb", PLAYER_GOOD); + + // Within the cooldown the memo still replays — no network. + assert!(mgr + .url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=AAA") + .is_err()); + assert_eq!(stub.count(), 2); + + // After the cooldown: probe discovers the NEW url → full fetch → + // extraction + eval succeed. + mgr.test_expire_memos(); + let out = mgr + .url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=abc") + .unwrap(); + assert_eq!(out, "https://x/?n=cba"); + assert_eq!(stub.count(), 4, "probe (iframe) + fresh body download"); + } + + #[test] + fn post_cooldown_probe_on_unchanged_player_skips_body_download() { + let _g = GLOBAL_DOWNLOADER_LOCK.lock(); + let stub = Arc::new(StubDownloader::new("aaaaaaaa", PLAYER_NSIG_BROKEN)); + install(&stub); + let mgr = PlayerManager::new(); + + assert!(mgr + .url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=AAA") + .is_err()); + assert_eq!(stub.count(), 2); + + mgr.test_expire_memos(); + // Probe: one iframe_api discovery, SAME hash → no body download, + // memo re-armed. + assert!(mgr + .url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=AAA") + .is_err()); + assert_eq!(stub.count(), 3); + + // Re-armed cooldown suppresses even the probe. + assert!(mgr + .url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=AAA") + .is_err()); + assert_eq!(stub.count(), 3); + } + + #[test] + fn transient_eval_failure_still_self_heals_via_refetch() { + let _g = GLOBAL_DOWNLOADER_LOCK.lock(); + let stub = Arc::new(StubDownloader::new("aaaaaaaa", PLAYER_EVAL_SELECTIVE)); + install(&stub); + let mgr = PlayerManager::new(); + + // Working nsig on the first video. + let out = mgr + .url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=good1") + .unwrap(); + assert_eq!(out, "https://x/?n=1doog"); + assert_eq!(stub.count(), 2); + + // A new-generation n-param chokes the CACHED (not fresh) snippet → + // invalidate, NO memo. No network spent on the failing call itself. + assert!(mgr + .url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=evil") + .is_err()); + assert_eq!(stub.count(), 2); + + // YouTube has rotated; the very next call re-fetches and succeeds — + // the transient-rotation self-heal must NOT be suppressed. + stub.rotate("bbbbbbbb", PLAYER_GOOD); + let out = mgr + .url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=evil") + .unwrap(); + assert_eq!(out, "https://x/?n=live"); + assert_eq!(stub.count(), 4); + } + + #[test] + fn persistent_eval_failure_on_fresh_player_memoizes() { + let _g = GLOBAL_DOWNLOADER_LOCK.lock(); + let stub = Arc::new(StubDownloader::new("aaaaaaaa", PLAYER_EVAL_THROWS)); + install(&stub); + let mgr = PlayerManager::new(); + + // Fresh download + successful build, then eval throws → memo. + let err = mgr + .url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=AAA") + .unwrap_err(); + assert!(matches!(err, DeobfError::JsRuntimeFailed(_)), "got {err:?}"); + assert_eq!(stub.count(), 2); + + // Every further format replays without network. + for _ in 0..5 { + assert!(mgr + .url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=BBB") + .is_err()); + } + assert_eq!(stub.count(), 2); + } + + #[test] + fn state_lock_stays_free_during_player_fetch() { + let _g = GLOBAL_DOWNLOADER_LOCK.lock(); + let (entered_tx, entered_rx) = mpsc::channel(); + let (release_tx, release_rx) = mpsc::channel(); + let mut stub = StubDownloader::new("aaaaaaaa", PLAYER_GOOD); + stub.block_player_fetch = Some((Mutex::new(entered_tx), Mutex::new(release_rx))); + let stub = Arc::new(stub); + install(&stub); + + let mgr: &'static PlayerManager = Box::leak(Box::new(PlayerManager::new())); + let worker = std::thread::spawn(move || { + mgr.url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=abc") + }); + + // Wait until the worker is INSIDE the player.js body download. + entered_rx + .recv_timeout(std::time::Duration::from_secs(10)) + .expect("worker never reached the player.js fetch"); + + // The whole point of the fetch_gate: readers aren't blocked while + // ~1.7 MB downloads. The old design held this mutex across the + // network and serialized everything behind it. + assert!( + mgr.state_lock_is_free(), + "state mutex must not be held across the player.js download" + ); + assert_eq!(mgr.throttling_parameter_cache_size(), 0); // must not block + + release_tx.send(()).unwrap(); + let out = worker.join().unwrap().unwrap(); + assert_eq!(out, "https://x/?n=cba"); + } + + // ---- S7 player_code byte-release lifecycle ---------------------------- + + #[test] + fn player_code_released_after_hot_artifacts_built_but_stays_installed() { + let _g = GLOBAL_DOWNLOADER_LOCK.lock(); + let stub = Arc::new(StubDownloader::new("aaaaaaaa", PLAYER_GOOD)); + install(&stub); + let mgr = PlayerManager::new(); + + // Timestamp built first (mirrors stream_info_with's worker). Only ONE of + // the two hot artifacts exists → bytes MUST still be retained so nsig can + // build without a re-download. + assert_eq!(mgr.signature_timestamp("vid").unwrap(), 20244); + assert_eq!(stub.count(), 2, "discovery + body download"); + assert!( + mgr.player_code_is_retained(), + "bytes kept until BOTH hot snippets exist" + ); + + // nsig built from the SAME retained bytes (no new network) → now both + // hot artifacts exist → bytes released, but the generation stays + // installed (player_url intact). + let out = mgr + .url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=abc") + .unwrap(); + assert_eq!(out, "https://x/?n=cba"); + assert_eq!(stub.count(), 2, "nsig built from retained bytes, no refetch"); + assert!( + !mgr.player_code_is_retained(), + "bytes dropped once timestamp + nsig are built" + ); + assert!( + mgr.player_url().is_some(), + "installed-generation sentinel survives the byte release" + ); + + // Warm re-request of BOTH hot artifacts after the release: pure cache + // hits, zero further network — the release is invisible to the hot path. + assert_eq!(mgr.signature_timestamp("vid").unwrap(), 20244); + let out = mgr + .url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=xyz") + .unwrap(); + assert_eq!(out, "https://x/?n=zyx"); + assert_eq!(stub.count(), 2, "warm path never re-downloads after release"); + assert!(!mgr.player_code_is_retained()); + } + + #[test] + fn build_failure_keeps_player_code_for_sibling_extraction() { + let _g = GLOBAL_DOWNLOADER_LOCK.lock(); + // nsig build fails on this player.js; the signature timestamp extracts. + let stub = Arc::new(StubDownloader::new("aaaaaaaa", PLAYER_NSIG_BROKEN)); + install(&stub); + let mgr = PlayerManager::new(); + + // nsig build FAILS → memo armed, bytes must be KEPT (never released on a + // failure) so the sibling timestamp artifact can still extract. + assert!(mgr + .url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=AAA") + .is_err()); + assert_eq!(stub.count(), 2); + assert!( + mgr.player_code_is_retained(), + "a build failure must NOT drop the bytes" + ); + + // Sibling timestamp extracts from the retained bytes — no re-download — + // and because nsig never built, the release condition stays unmet. + assert_eq!(mgr.signature_timestamp("vid").unwrap(), 19999); + assert_eq!(stub.count(), 2, "sibling built from retained bytes"); + assert!( + mgr.player_code_is_retained(), + "release requires the nsig snippet, which never built here" + ); + } } diff --git a/src/youtube/linkhandler/channel.rs b/src/youtube/linkhandler/channel.rs index d294655..33fde0e 100644 --- a/src/youtube/linkhandler/channel.rs +++ b/src/youtube/linkhandler/channel.rs @@ -10,7 +10,7 @@ use url::Url; -use crate::youtube::linkhandler::{host_is_youtube, LinkError}; +use crate::youtube::linkhandler::{host_is_youtube, redact_url, LinkError}; #[derive(Clone, Debug, Eq, PartialEq)] pub enum ChannelIdentifier { @@ -25,8 +25,10 @@ pub enum ChannelIdentifier { } pub fn parse(url_str: &str) -> Result { + // Privacy: never embed the raw input in the error (see stream.rs) — + // the parse reason alone suffices. let url = Url::parse(url_str) - .map_err(|e| LinkError::InvalidUrl(format!("{url_str}: {e}")))?; + .map_err(|e| LinkError::InvalidUrl(e.to_string()))?; let host = url .host_str() .ok_or_else(|| LinkError::InvalidUrl("no host".into()))?; @@ -37,32 +39,32 @@ pub fn parse(url_str: &str) -> Result { if let Some(rest) = path.strip_prefix("/channel/") { let id = rest.split('/').next().unwrap_or(""); if id.is_empty() { - return Err(LinkError::MissingId(url_str.into())); + return Err(LinkError::MissingId(redact_url(&url))); } return Ok(ChannelIdentifier::DirectId(id.into())); } if let Some(rest) = path.strip_prefix("/c/") { let s = rest.split('/').next().unwrap_or(""); if s.is_empty() { - return Err(LinkError::MissingId(url_str.into())); + return Err(LinkError::MissingId(redact_url(&url))); } return Ok(ChannelIdentifier::Custom(s.into())); } if let Some(rest) = path.strip_prefix("/user/") { let s = rest.split('/').next().unwrap_or(""); if s.is_empty() { - return Err(LinkError::MissingId(url_str.into())); + return Err(LinkError::MissingId(redact_url(&url))); } return Ok(ChannelIdentifier::LegacyUser(s.into())); } if let Some(rest) = path.strip_prefix("/@") { let s = rest.split('/').next().unwrap_or(""); if s.is_empty() { - return Err(LinkError::MissingId(url_str.into())); + return Err(LinkError::MissingId(redact_url(&url))); } return Ok(ChannelIdentifier::Handle(s.into())); } - Err(LinkError::MissingId(url_str.into())) + Err(LinkError::MissingId(redact_url(&url))) } pub fn channel_url(channel_id: &str) -> String { diff --git a/src/youtube/linkhandler/mod.rs b/src/youtube/linkhandler/mod.rs index 4f2f671..542f85d 100644 --- a/src/youtube/linkhandler/mod.rs +++ b/src/youtube/linkhandler/mod.rs @@ -34,6 +34,20 @@ pub const ACCEPTED_HOSTS: &[&str] = &[ "www.youtube-nocookie.com", ]; +/// Renders a URL as scheme+host only, for embedding in error strings. +/// Privacy: link errors reach Kotlin exception messages / logs. Query params +/// carry user browsing data (`v=`, `list=`) AND several +/// YouTube PATHS embed the id (`/embed/`, `/shorts/`, `/clip/`, +/// `/live/`, `youtu.be/`) — so drop the path too; the host alone says +/// which endpoint failed without leaking what the user was watching. +pub(crate) fn redact_url(url: &url::Url) -> String { + let mut u = url.clone(); + u.set_query(None); + u.set_fragment(None); + u.set_path("/"); + u.to_string() +} + pub fn host_is_youtube(host: &str) -> bool { let h = host.to_ascii_lowercase(); let h = h.strip_prefix("www.").unwrap_or(&h); @@ -66,4 +80,24 @@ mod tests { assert!(!host_is_youtube("piped.video")); assert!(!host_is_youtube("evil.com")); } + + #[test] + fn redact_url_drops_id_bearing_path_query_and_fragment() { + // Every one of these carries the video/playlist id somewhere the old + // query-only strip would have kept (the path, for embed/shorts/youtu.be). + let cases = [ + "https://www.youtube.com/embed/dQw4w9WgXcQ?v=secret#frag", + "https://www.youtube.com/shorts/dQw4w9WgXcQ", + "https://youtu.be/dQw4w9WgXcQ", + "https://www.youtube.com/watch?v=dQw4w9WgXcQ&list=PLsecretlist", + ]; + for c in cases { + let u = url::Url::parse(c).unwrap(); + let r = redact_url(&u); + assert!(!r.contains("dQw4w9WgXcQ"), "leaked video id: {r}"); + assert!(!r.contains("PLsecretlist"), "leaked list id: {r}"); + // scheme + host only (no port in any of these cases). + assert_eq!(r, format!("{}://{}/", u.scheme(), u.host_str().unwrap())); + } + } } diff --git a/src/youtube/linkhandler/stream.rs b/src/youtube/linkhandler/stream.rs index 9592138..dbeb732 100644 --- a/src/youtube/linkhandler/stream.rs +++ b/src/youtube/linkhandler/stream.rs @@ -15,7 +15,7 @@ use once_cell::sync::Lazy; use regex::Regex; use url::Url; -use crate::youtube::linkhandler::{host_is_youtube, LinkError}; +use crate::youtube::linkhandler::{host_is_youtube, redact_url, LinkError}; const VIDEO_ID_LEN: usize = 11; @@ -42,8 +42,11 @@ fn extract_video_id_inner(input_url: &str, depth: u8) -> Result Result String { ) } +/// visionOS user-agent — mirrors NPE `getVisionOsUserAgent`. +pub fn visionos_user_agent(country: &ContentCountry) -> String { + format!( + "com.google.visionos.youtube/{VISIONOS_CLIENT_VERSION}({VISIONOS_DEVICE_MODEL}; U; CPU visionOS {VISIONOS_USER_AGENT_VERSION} like Mac OS X; {})", + country.country_code() + ) +} + #[cfg(test)] mod tests { use super::*; @@ -134,4 +142,13 @@ mod tests { assert!(ua.contains("CPU iOS 18_7_2")); assert!(ua.contains("; US)")); } + + #[test] + fn visionos_ua_template() { + let ua = visionos_user_agent(&ContentCountry::new("US")); + assert!(ua.contains("com.google.visionos.youtube/1.02")); + assert!(ua.contains("RealityDevice14,1")); + assert!(ua.contains("CPU visionOS 25_6_0 like Mac OS X")); + assert!(ua.contains("; US)")); + } } diff --git a/src/youtube/stream_extractor.rs b/src/youtube/stream_extractor.rs index 8837537..36c4336 100644 --- a/src/youtube/stream_extractor.rs +++ b/src/youtube/stream_extractor.rs @@ -28,7 +28,8 @@ use crate::image::{Image, ResolutionLevel}; use crate::localization::{ContentCountry, Localization}; use crate::newpipe::NewPipe; use crate::stream::{ - AudioStream, DeliveryMethod, StreamInfo, StreamType, SubtitlesStream, VideoStream, + AudioStream, AudioTrackType, DeliveryMethod, StreamInfo, StreamType, SubtitlesStream, + VideoStream, }; use crate::youtube::itag::{lookup as itag_lookup, ItagType}; #[cfg(test)] @@ -36,6 +37,7 @@ use crate::youtube::itag::MediaFormat; use crate::youtube::js::PlayerManager; use crate::youtube::potoken::{po_token_provider, PoTokenResult}; use crate::youtube::stream_helper::{self, generate_content_playback_nonce}; +use crate::youtube::xtags; /// Shared `Value::Null` sentinel so a missing `streamingData` can be handed /// back as a borrowed `&Value` instead of cloning the (large) present @@ -49,9 +51,23 @@ pub enum FetchPolicy { AndroidWithPoToken, } -#[derive(Clone, Debug, Default)] +#[derive(Clone, Debug)] pub struct ExtractOptions { pub fetch_ios_client: bool, + /// Fetch the visionOS client (client 101) as an additional best-effort + /// /player call. Recovers >360p streams for SABR-only / "made for kids" + /// videos where ANDROID/IOS cap at 360p. Its formats are merged in + /// (android-preferred on itag ties); a failure changes nothing. + /// + /// Defaults to `false` (see `impl Default`) as of the 2026-07-29 playback + /// outage fix. Upstream NPE runs the visionOS fetch UNCONDITIONALLY in + /// `onFetchPage`, but its adaptive video-only + HLS output has no working + /// playback path in the Straw app yet, so the shipped path (`stream_info` → + /// `stream_info_with(.., ExtractOptions::default())`) keeps it OFF. Set to + /// `true` per-call to opt in (e.g. once the app's adaptive path is fixed and + /// verified via the NPE differential harness). The cheap `stream_metadata` + /// path does not build streams and never fetches visionOS regardless. + pub fetch_visionos_client: bool, pub android_streaming_pot: Option, pub ios_streaming_pot: Option, pub android_visitor_data: Option, @@ -60,6 +76,35 @@ pub struct ExtractOptions { pub ios_player_request_pot: Option, } +// Explicit manual impl (not a derive) so the load-bearing `fetch_visionos_client: +// false` default is spelled out and greppable — a silent flip of exactly this +// value caused the 2026-07-29 playback outage. clippy flags it as derivable only +// because every field currently equals its type default; keep it explicit on +// purpose so re-enabling visionOS is a deliberate, visible edit here. +#[allow(clippy::derivable_impls)] +impl Default for ExtractOptions { + fn default() -> Self { + Self { + // visionOS OFF by default (2026-07-29 outage fix). visionOS unlocks + // adaptive video-only + separate-audio streams + a VOD HLS manifest, + // but the Straw app has no working adaptive/HLS playback path — it + // has only ever received the muxed 360p stream — so those streams + // don't play and video playback breaks. Upstream NPE runs visionOS + // unconditionally; we can't re-enable until the client handles its + // output (fix + verify the app's adaptive path via the NPE + // differential harness first). + fetch_visionos_client: false, + fetch_ios_client: false, + android_streaming_pot: None, + ios_streaming_pot: None, + android_visitor_data: None, + ios_visitor_data: None, + android_player_request_pot: None, + ios_player_request_pot: None, + } + } +} + /// One-shot StreamInfo build for a video. Walks NPE's Android-primary /// fetch path, applies URL post-processing, returns the final shape. pub fn stream_info(video_id: &str) -> Result { @@ -90,19 +135,58 @@ pub fn stream_info_with( ); let android_cpn = generate_content_playback_nonce(); - let player_response = fetch_android( - video_id, - &localization, - &content_country, - &android_cpn, - android_token - .as_ref() - .map(|t| t.player_request_po_token.as_str()), - android_token.as_ref().map(|t| t.visitor_data.as_str()), - )?; - check_playability_status(&player_response)?; + // S2 (Option A — the conservative subset): overlap the player.js + // signature-timestamp resolution with the primary Android reel fetch. The + // two are fully independent — `signature_timestamp` needs only `video_id`, + // and the WEB metadata leg that consumes the timestamp still runs AFTER the + // decoy gate below. On a cold open (or after a player.js rotation) this + // overlaps the ~1.5 MB base.js download with the Android round-trip instead + // of paying them serially; warm, the timestamp is cached and the worker + // returns near-instantly. `scope` joins both before either result is read, + // so the borrowed locals stay valid (same shape as `channel.rs`). The + // worker leg is best-effort: a failed extraction OR a panicked worker yields + // 0 — identical to the prior serial `.unwrap_or(0)`. Android stays the + // gating primary (its playability/decoy check below is unchanged and still + // aborts before any WEB/visionOS work), the visitorData-first ordering is + // untouched (the player.js fetch never hits the visitor_id endpoint), and + // player.js concurrency is already guarded by PlayerManager's fetch_gate so + // a second concurrent open dedups the download rather than racing it. + // Trade-off (accepted): on a COLD cache, a *failing* open can't return its + // error until the speculative player.js leg joins (bounded by one download); + // the win is the warm/success path. Worst case is a rare, bounded cold-fail + // delay — never a change in what's extracted. + let (android_result, signature_timestamp) = std::thread::scope(|s| { + let sig_ts = s.spawn(|| { + PlayerManager::instance() + .signature_timestamp(video_id) + .unwrap_or(0) + }); + let android = fetch_android( + video_id, + &localization, + &content_country, + &android_cpn, + android_token + .as_ref() + .map(|t| t.player_request_po_token.as_str()), + android_token.as_ref().map(|t| t.visitor_data.as_str()), + ); + // A panicked worker joins as Err → 0, same as an extraction failure. + (android, sig_ts.join().unwrap_or(0)) + }); + let player_response = android_result?; + + // A primary-android rejection (playability failure or decoy) is exactly the + // symptom a poisoned visitorData would produce on the primary reel call — + // and since it aborts extraction the value would otherwise never self-heal. + // Drop the cached visitorData so the next extraction fetches a fresh one. + if let Err(e) = check_playability_status(&player_response) { + stream_helper::reset_visitor_data_cache(); + return Err(e); + } if is_player_response_not_valid(&player_response, video_id) { + stream_helper::reset_visitor_data_cache(); return Err(ExtractionError::Other( "ANDROID player response is not valid (decoy detected)".into(), )); @@ -152,6 +236,31 @@ pub fn stream_info_with( .and_then(|r| r.get("streamingData")) .unwrap_or(&NULL_VALUE); + // Optional visionOS — best-effort, gated (default OFF; see + // ExtractOptions::fetch_visionos_client). Recovers >360p streams for + // SABR-only / "made for kids" videos. Carries NO poToken and (like iOS) is + // decoy-checked before its streamingData is used. A failure leaves the + // output exactly as it is today. + let (visionos_response, visionos_cpn): (Option, Option) = + if options.fetch_visionos_client { + let cpn = generate_content_playback_nonce(); + match stream_helper::get_visionos_player_response( + video_id, + &localization, + &content_country, + &cpn, + ) { + Ok(r) if !is_player_response_not_valid(&r, video_id) => (Some(r), Some(cpn)), + _ => (None, None), + } + } else { + (None, None) + }; + let visionos_streaming_data: &Value = visionos_response + .as_ref() + .and_then(|r| r.get("streamingData")) + .unwrap_or(&NULL_VALUE); + let android_streaming_pot = android_token .as_ref() .map(|t| t.streaming_data_po_token.clone()) @@ -161,9 +270,9 @@ pub fn stream_info_with( .map(|t| t.streaming_data_po_token.clone()) .or_else(|| options.ios_streaming_pot.clone()); - let signature_timestamp = PlayerManager::instance() - .signature_timestamp(video_id) - .unwrap_or(0); + // `signature_timestamp` was resolved concurrently with the Android fetch + // above (S2). The WEB metadata leg is the sole consumer and runs only now, + // after the Android decoy gate has passed. let web_metadata = fetch_web_metadata(video_id, &localization, &content_country, signature_timestamp); let mut info = StreamInfo { @@ -176,19 +285,37 @@ pub fn stream_info_with( populate_video_details(&mut info, &player_response); populate_microformat(&mut info, &web_metadata); - populate_streams( - &mut info, - android_streaming_data, - ios_streaming_data, - video_id, - &android_cpn, - ios_cpn.as_deref(), - android_streaming_pot.as_deref(), - ios_streaming_pot.as_deref(), - )?; + + // Merge order mirrors NPE getItags: ANDROID, then VISIONOS, then IOS. The + // per-(itag,delivery) dedup is first-wins, so android formats win ties and + // visionOS/iOS only ADD formats android lacked (e.g. the >360p renditions + // for SABR-only videos). visionOS carries no poToken. + let sources = [ + FormatSource { + streaming_data: android_streaming_data, + client: "ANDROID", + cpn: &android_cpn, + pot: android_streaming_pot.as_deref(), + }, + FormatSource { + streaming_data: visionos_streaming_data, + client: "VISIONOS", + cpn: visionos_cpn.as_deref().unwrap_or(""), + pot: None, + }, + FormatSource { + streaming_data: ios_streaming_data, + client: "IOS", + cpn: ios_cpn.as_deref().unwrap_or(""), + pot: ios_streaming_pot.as_deref(), + }, + ]; + + populate_streams(&mut info, &sources, video_id)?; populate_manifests( &mut info, android_streaming_data, + visionos_streaming_data, ios_streaming_data, android_streaming_pot.as_deref(), ios_streaming_pot.as_deref(), @@ -231,7 +358,7 @@ pub fn stream_metadata(video_id: &str) -> Result { }); let android_cpn = generate_content_playback_nonce(); - let player_response = fetch_android( + let player_response = fetch_android_metadata( video_id, &localization, &content_country, @@ -242,8 +369,16 @@ pub fn stream_metadata(video_id: &str) -> Result { android_token.as_ref().map(|t| t.visitor_data.as_str()), )?; - check_playability_status(&player_response)?; + // A primary-android rejection (playability failure or decoy) is exactly the + // symptom a poisoned visitorData would produce on the primary reel call — + // and since it aborts extraction the value would otherwise never self-heal. + // Drop the cached visitorData so the next extraction fetches a fresh one. + if let Err(e) = check_playability_status(&player_response) { + stream_helper::reset_visitor_data_cache(); + return Err(e); + } if is_player_response_not_valid(&player_response, video_id) { + stream_helper::reset_visitor_data_cache(); return Err(ExtractionError::Other( "ANDROID player response is not valid (decoy detected)".into(), )); @@ -281,7 +416,7 @@ fn fetch_android( po_token: Option<&str>, visitor_data: Option<&str>, ) -> Result { - let result = if po_token.is_some() { + if po_token.is_some() { stream_helper::get_android_player_response( video_id, localization, @@ -298,9 +433,86 @@ fn fetch_android( cpn, )?; // The reel endpoint returns the `playerResponse` nested one level. - Ok(r.get("playerResponse").cloned().unwrap_or(r)) - }; - result + Ok(unwrap_reel_player_response(r)) + } +} + +/// Metadata-scoped Android fetch for the cheap `stream_metadata` feed-enrich +/// path — do NOT use on the full extraction path, which needs `streamingData`. +/// +/// On the anonymous reel path (the default — no PoTokenProvider registered) this +/// tries the NARROWED-`$fields` reel variant, which asks the endpoint for only +/// `playabilityStatus` + `videoDetails` (~3-6 KB vs the full ~150-500 KB reel +/// body). It is BEST-EFFORT with a HARD full-fetch fallback: `reel_item_watch` +/// may not honor the nested `$fields` selector, so if the narrowed request errors +/// (e.g. a 400 rejecting the selector) OR returns a shape MISSING `videoDetails` +/// (endpoint ignored it into an error/empty response), we fall back to the full +/// `fetch_android`, which is byte-for-byte today's behavior. Net: narrowed +/// honored → bytes saved; narrowed not honored → identical to today. Feed +/// enrichment can never regress from this optimization. +/// +/// The po_token `/player` path (non-default) has no `$fields` narrowing to apply, +/// so it goes straight to the shared `fetch_android`. +fn fetch_android_metadata( + video_id: &str, + localization: &Localization, + content_country: &ContentCountry, + cpn: &str, + po_token: Option<&str>, + visitor_data: Option<&str>, +) -> Result { + // Only the anonymous reel path carries a $fields selector to narrow. + if po_token.is_none() { + if let Ok(r) = stream_helper::get_android_reel_player_response_metadata( + video_id, + localization, + content_country, + cpn, + ) { + let unwrapped = unwrap_reel_player_response(r); + // Guard: only trust the narrowed response if its videoDetails is + // bound to the REQUESTED video — i.e. `videoDetails.videoId` is + // present AND equals `video_id`. A bare `is_some()` would accept a + // present-but-gutted `videoDetails: {}` (endpoint half-honored the + // nested selector) → silently-empty title/views/duration, and would + // also wave through a videoId-stripped decoy (the downstream decoy + // check treats a missing videoId as valid). Binding to the id + // subsumes the decoy check for this path and rejects every + // not-fully-honored shape into the full-fetch fallback. + let honored = unwrapped + .get("videoDetails") + .and_then(|vd| vd.get("videoId")) + .and_then(|id| id.as_str()) + == Some(video_id); + if honored { + return Ok(unwrapped); + } + } + } + // Fallback (and the po_token path): the full, unchanged fetch. + fetch_android( + video_id, + localization, + content_country, + cpn, + po_token, + visitor_data, + ) +} + +/// Unwrap the reel endpoint's one-level `playerResponse` nesting, MOVING the +/// (~150-500 KB) subtree out instead of deep-cloning it. `r` is owned and dead +/// after this call, so `serde_json::Map::remove` returns the owned subtree with +/// zero allocation — the old `.cloned().unwrap_or(r)` re-allocated every Map / +/// Vec / String node on every video open AND every feed-enrich (S1). Behavior +/// is identical, including the defensive "no `playerResponse` key → use the +/// whole response" fallback. Only the anonymous reel path calls this; the +/// token /player path returns the response already un-nested. +fn unwrap_reel_player_response(r: Value) -> Value { + match r { + Value::Object(mut m) => m.remove("playerResponse").unwrap_or(Value::Object(m)), + other => other, + } } fn fetch_web_metadata( @@ -461,66 +673,50 @@ fn populate_microformat(info: &mut StreamInfo, web_metadata: &Value) { } } -/// Merge the Android + iOS format objects under one streamingData array -/// key (`formats` / `adaptiveFormats`), each tagged with its client + cpn + -/// pot. Returns borrowed `&Value`s into the player responses: the format -/// objects are only ever read (`.get(...)`) downstream, so cloning the whole -/// array (~20-40 objects per video) was pure waste. -#[allow(clippy::too_many_arguments)] +/// One client's streamingData plus the tags every format from it carries. +/// Holds borrowed `&Value`s into the (function-lived) player responses. +struct FormatSource<'a> { + streaming_data: &'a Value, + client: &'static str, + cpn: &'a str, + pot: Option<&'a str>, +} + +/// Merge the per-client format objects under one streamingData array key +/// (`formats` / `adaptiveFormats`), each tagged with its client + cpn + pot, +/// in source order (ANDROID, VISIONOS, IOS). Returns borrowed `&Value`s into +/// the player responses: the format objects are only ever read (`.get(...)`) +/// downstream, so cloning the whole array (~20-40 objects per video) was pure +/// waste. fn merge_formats<'a>( - android: &'a Value, - ios: &'a Value, + sources: &[FormatSource<'a>], fmt_array_key: &str, - android_cpn: &'a str, - ios_cpn: Option<&'a str>, - android_pot: Option<&'a str>, - ios_pot: Option<&'a str>, ) -> Vec<(&'a Value, &'static str, &'a str, Option<&'a str>)> { let mut out = Vec::new(); - if let Some(arr) = android.get(fmt_array_key).and_then(|v| v.as_array()) { - for f in arr { - out.push((f, "ANDROID", android_cpn, android_pot)); - } - } - if let Some(arr) = ios.get(fmt_array_key).and_then(|v| v.as_array()) { - let cpn = ios_cpn.unwrap_or(""); - for f in arr { - out.push((f, "IOS", cpn, ios_pot)); + for src in sources { + if let Some(arr) = src.streaming_data.get(fmt_array_key).and_then(|v| v.as_array()) { + for f in arr { + out.push((f, src.client, src.cpn, src.pot)); + } } } out } -#[allow(clippy::too_many_arguments)] fn populate_streams( info: &mut StreamInfo, - android: &Value, - ios: &Value, + sources: &[FormatSource], video_id: &str, - android_cpn: &str, - ios_cpn: Option<&str>, - android_pot: Option<&str>, - ios_pot: Option<&str>, ) -> Result<(), ExtractionError> { // Progressive: streamingData.formats[] - for (fmt, _client, cpn, pot) in - merge_formats(android, ios, "formats", android_cpn, ios_cpn, android_pot, ios_pot) - { + for (fmt, _client, cpn, pot) in merge_formats(sources, "formats") { if let Some(stream) = build_video_progressive(fmt, video_id, cpn, pot)? { push_video_dedup(&mut info.video_streams, stream); } } // Adaptive: streamingData.adaptiveFormats[] - for (fmt, _client, cpn, pot) in merge_formats( - android, - ios, - "adaptiveFormats", - android_cpn, - ios_cpn, - android_pot, - ios_pot, - ) { + for (fmt, _client, cpn, pot) in merge_formats(sources, "adaptiveFormats") { let mime = fmt .get("mimeType") .and_then(|v| v.as_str()) @@ -541,18 +737,43 @@ fn populate_streams( fn populate_manifests( info: &mut StreamInfo, android: &Value, + visionos: &Value, ios: &Value, android_pot: Option<&str>, ios_pot: Option<&str>, ) { + // NPE getManifestUrl skips null-OR-EMPTY manifest URLs; the `!is_empty` + // filters matter because an EMPTY visionOS `hlsManifestUrl` would otherwise + // shadow a real iOS/Android one (`Some("")`), and the wrapper drops + // `Some("")` → the app loses HLS entirely. // DASH is Android-only. - if let Some(url) = android.get("dashManifestUrl").and_then(|v| v.as_str()) { + if let Some(url) = android + .get("dashManifestUrl") + .and_then(|v| v.as_str()) + .filter(|s| !s.is_empty()) + { info.dash_manifest_url = Some(append_pot_to_manifest(url, android_pot)); } - // HLS prefers iOS, falls back to Android. - if let Some(url) = ios.get("hlsManifestUrl").and_then(|v| v.as_str()) { + // HLS prefers an Apple client — visionOS, then iOS — because their HLS + // manifests carry separated audio/video for livestreams (NPE getHlsUrl: + // visionOS → iOS → android). visionOS carries no poToken. + if let Some(url) = visionos + .get("hlsManifestUrl") + .and_then(|v| v.as_str()) + .filter(|s| !s.is_empty()) + { + info.hls_manifest_url = Some(append_pot_to_manifest(url, None)); + } else if let Some(url) = ios + .get("hlsManifestUrl") + .and_then(|v| v.as_str()) + .filter(|s| !s.is_empty()) + { info.hls_manifest_url = Some(append_pot_to_manifest(url, ios_pot)); - } else if let Some(url) = android.get("hlsManifestUrl").and_then(|v| v.as_str()) { + } else if let Some(url) = android + .get("hlsManifestUrl") + .and_then(|v| v.as_str()) + .filter(|s| !s.is_empty()) + { info.hls_manifest_url = Some(append_pot_to_manifest(url, android_pot)); } } @@ -640,11 +861,15 @@ fn process_url( // nsig deobf — unconditional (quick-exits internally if no `n=` present). // On failure, fall back to the throttled ORIGINAL url rather than dropping - // the format or aborting the video: YouTube still serves it (rate-limited), - // and PlayerManager has already invalidated its cache so the NEXT format - // re-fetches a fresh player.js and deobfuscates cleanly. One routine - // player.js rotation therefore costs at most one throttled format, not the - // whole video. (NPE parity; straw audit 2026-07-04.) + // the format or aborting the video: YouTube still serves it (rate-limited). + // A stale-cache eval failure invalidates PlayerManager's cache, so the + // NEXT format re-fetches a fresh player.js and deobfuscates cleanly — one + // routine rotation costs at most one throttled format. If the FRESH + // player.js also fails (regex bank miss — a persistent breakage), the + // manager's failure memo replays the error for the remaining formats with + // zero network instead of re-downloading ~1.7 MB per format, and retries + // after a cooldown / as soon as a new player.js ships. (NPE parity; straw + // audits 2026-07-04 + 2026-07-28 H2.) url = PlayerManager::instance() .url_with_throttling_parameter_deobfuscated(video_id, &url) .unwrap_or(url); @@ -755,6 +980,35 @@ fn build_audio( return Ok(None); }; let audio_track = fmt.get("audioTrack"); + let has_track = audio_track + .and_then(|t| t.get("id")) + .and_then(|v| v.as_str()) + .map(|s| !s.is_empty()) + .unwrap_or(false); + + // audioTrackType from the format's `xtags` protobuf (NPE b0bca7e7/e3479a7c). + // Only meaningful for a real alternate track (matches NPE, which sets it + // only inside `if (!isNullOrEmpty(audioTrackId))`). + let track_type = if has_track { + fmt.get("xtags") + .and_then(|v| v.as_str()) + .and_then(xtags::extract_audio_track_type) + } else { + None + }; + + // `is_descriptive`: prefer the authoritative xtags-derived type; fall back + // to the legacy `audioTrack.audioIsDefault` heuristic when xtags is + // absent/unparseable (track_type == None). + let is_descriptive = match track_type { + Some(t) => t == AudioTrackType::Descriptive, + None => audio_track + .and_then(|t| t.get("audioIsDefault")) + .and_then(|v| v.as_bool()) + .map(|b| !b) + .unwrap_or(false), + }; + Ok(Some(AudioStream { itag: itag.id, url, @@ -783,11 +1037,8 @@ fn build_audio( .and_then(|v| v.as_str()) .and_then(|s| s.split('.').next()) .map(String::from), - is_descriptive: audio_track - .and_then(|t| t.get("audioIsDefault")) - .and_then(|v| v.as_bool()) - .map(|b| !b) - .unwrap_or(false), + is_descriptive, + track_type, itag_url_format: None, })) } @@ -886,6 +1137,27 @@ mod tests { assert!(!is_player_response_not_valid(&resp, "MATCHING")); } + #[test] + fn reel_player_response_moved_out_or_falls_back() { + // Nested playerResponse is unwrapped (moved out, not cloned) — the + // returned subtree is exactly the inner object. + let r = json!({ + "playerResponse": {"videoDetails": {"videoId": "abc"}, "streamingData": {}}, + "responseContext": {"x": 1} + }); + let out = unwrap_reel_player_response(r); + assert_eq!(out, json!({"videoDetails": {"videoId": "abc"}, "streamingData": {}})); + + // No playerResponse key → the whole response is returned unchanged + // (matches the prior `.cloned().unwrap_or(r)` fallback). + let r = json!({"videoDetails": {"videoId": "xyz"}}); + assert_eq!(unwrap_reel_player_response(r.clone()), r); + + // Defensive: a non-object value passes through as-is. + let r = json!("scalar"); + assert_eq!(unwrap_reel_player_response(r.clone()), r); + } + #[test] fn cipher_string_parsed() { let s = "s=AAA%3D&sp=sig&url=https%3A%2F%2Fexample.com%2Fpath%3Fa%3D1"; @@ -951,4 +1223,334 @@ mod tests { push_video_dedup(&mut list, s2); assert_eq!(list.len(), 2); } + + // ---- I-1 visionOS merge ------------------------------------------------ + + #[test] + fn absent_visionos_source_contributes_nothing() { + // Best-effort proof: a Null visionOS streamingData (fetch failed or the + // client disabled) adds ZERO formats — the merged set is android+ios + // only, exactly today's behavior. + let android = json!({ "formats": [ {"itag": 18} ] }); + let ios = json!({ "formats": [ {"itag": 22} ] }); + let null = Value::Null; + let sources = [ + FormatSource { streaming_data: &android, client: "ANDROID", cpn: "a", pot: None }, + FormatSource { streaming_data: &null, client: "VISIONOS", cpn: "", pot: None }, + FormatSource { streaming_data: &ios, client: "IOS", cpn: "i", pot: None }, + ]; + let merged = merge_formats(&sources, "formats"); + let itags: Vec = merged + .iter() + .map(|(f, _, _, _)| f["itag"].as_u64().unwrap()) + .collect(); + assert_eq!(itags, vec![18, 22]); + assert!(merged.iter().all(|(_, client, _, _)| *client != "VISIONOS")); + } + + #[test] + fn populate_streams_merges_visionos_extra_itags_android_wins_ties() { + // android has only itag 18 (360p); visionOS returns a dup 18 plus an + // extra 22 (720p) android lacked — the SABR-only coverage recovery. + let android = json!({ "formats": [ + {"itag": 18, "url": "https://r/vp?a=1", "mimeType": "video/mp4; codecs=\"avc1.42\""} + ]}); + let visionos = json!({ "formats": [ + {"itag": 18, "url": "https://r/vp?v=1", "mimeType": "video/mp4"}, + {"itag": 22, "url": "https://r/vp?v=2", "mimeType": "video/mp4"} + ]}); + let null = Value::Null; + let sources = [ + FormatSource { streaming_data: &android, client: "ANDROID", cpn: "acpn", pot: None }, + FormatSource { streaming_data: &visionos, client: "VISIONOS", cpn: "vcpn", pot: None }, + FormatSource { streaming_data: &null, client: "IOS", cpn: "", pot: None }, + ]; + let mut info = StreamInfo::default(); + populate_streams(&mut info, &sources, "vid").unwrap(); + let mut tags: Vec = info.video_streams.iter().map(|s| s.itag).collect(); + tags.sort_unstable(); + assert_eq!(tags, vec![18, 22]); // dup 18 deduped, 22 recovered + // itag 18 kept from ANDROID (first-wins): its URL carries android's cpn. + let s18 = info.video_streams.iter().find(|s| s.itag == 18).unwrap(); + assert!(s18.url.contains("a=1"), "android's itag-18 must win: {}", s18.url); + assert!(s18.url.contains("cpn=acpn")); + } + + #[test] + fn hls_prefers_visionos_then_ios_then_android() { + let android = json!({ "hlsManifestUrl": "https://a/hls" }); + let visionos = json!({ "hlsManifestUrl": "https://v/hls" }); + let ios = json!({ "hlsManifestUrl": "https://i/hls" }); + let null = Value::Null; + + let mut info = StreamInfo::default(); + populate_manifests(&mut info, &android, &visionos, &ios, None, None); + assert_eq!(info.hls_manifest_url.as_deref(), Some("https://v/hls")); + + // visionOS absent → iOS wins. + let mut info = StreamInfo::default(); + populate_manifests(&mut info, &android, &null, &ios, None, None); + assert_eq!(info.hls_manifest_url.as_deref(), Some("https://i/hls")); + + // Only android → android (unchanged from today). + let mut info = StreamInfo::default(); + populate_manifests(&mut info, &android, &null, &null, None, None); + assert_eq!(info.hls_manifest_url.as_deref(), Some("https://a/hls")); + } + + #[test] + fn empty_hls_url_is_skipped_not_shadowing_a_real_one() { + // Fix 3: an EMPTY visionOS hlsManifestUrl must NOT shadow iOS's real one + // (else the wrapper drops Some("") → the app loses HLS). + let android = json!({ "hlsManifestUrl": "https://a/hls" }); + let visionos_empty = json!({ "hlsManifestUrl": "" }); + let ios = json!({ "hlsManifestUrl": "https://i/hls" }); + + let mut info = StreamInfo::default(); + populate_manifests(&mut info, &android, &visionos_empty, &ios, None, None); + assert_eq!(info.hls_manifest_url.as_deref(), Some("https://i/hls")); + + // Empty visionOS AND empty iOS → falls through to android. + let ios_empty = json!({ "hlsManifestUrl": "" }); + let mut info = StreamInfo::default(); + populate_manifests(&mut info, &android, &visionos_empty, &ios_empty, None, None); + assert_eq!(info.hls_manifest_url.as_deref(), Some("https://a/hls")); + + // All empty → None (not Some("")). + let android_empty = json!({ "hlsManifestUrl": "", "dashManifestUrl": "" }); + let mut info = StreamInfo::default(); + populate_manifests(&mut info, &android_empty, &visionos_empty, &ios_empty, None, None); + assert!(info.hls_manifest_url.is_none()); + assert!(info.dash_manifest_url.is_none()); // empty DASH also skipped + } + + #[test] + fn default_options_visionos_off() { + // 2026-07-29 outage fix: the shipped app path (stream_info → + // ExtractOptions::default()) must keep visionOS OFF — its adaptive/HLS + // output has no working playback path in the app yet. Re-enable only + // after the app handles it and the NPE harness verifies the shape. + assert!(!ExtractOptions::default().fetch_visionos_client); + } + + // ---- I-3 xtags audioTrackType ----------------------------------------- + + // itag 140 (M4A audio) + a direct url with no `n=` param, so process_url + // never touches the network (nsig quick-exits). + fn audio_fmt(xtags: Option<&str>, audio_is_default: bool) -> Value { + let mut fmt = json!({ + "itag": 140, + "url": "https://r.googlevideo.com/videoplayback?foo=1", + "mimeType": "audio/mp4; codecs=\"mp4a.40.2\"", + "audioTrack": { "id": "en.4", "displayName": "English", "audioIsDefault": audio_is_default } + }); + if let Some(x) = xtags { + fmt["xtags"] = json!(x); + } + fmt + } + + #[test] + fn xtags_descriptive_sets_track_type_and_is_descriptive() { + let fmt = audio_fmt(Some("ChQKBWFjb250EgtkZXNjcmlwdGl2ZQ"), true); + let a = build_audio(&fmt, "vid", "cpn", None).unwrap().unwrap(); + assert_eq!(a.track_type, Some(AudioTrackType::Descriptive)); + assert!(a.is_descriptive); + } + + #[test] + fn xtags_overrides_audio_is_default_heuristic() { + // acont=dubbed → Dubbed + is_descriptive false, even though + // audioIsDefault=false would make the legacy heuristic say "descriptive". + let fmt = audio_fmt(Some("Cg8KBWFjb250EgZkdWJiZWQ"), false); + let a = build_audio(&fmt, "vid", "cpn", None).unwrap().unwrap(); + assert_eq!(a.track_type, Some(AudioTrackType::Dubbed)); + assert!(!a.is_descriptive); + } + + #[test] + fn xtags_original_maps_to_original() { + let fmt = audio_fmt(Some("ChEKBWFjb250EghvcmlnaW5hbA"), true); + let a = build_audio(&fmt, "vid", "cpn", None).unwrap().unwrap(); + assert_eq!(a.track_type, Some(AudioTrackType::Original)); + assert!(!a.is_descriptive); + } + + #[test] + fn absent_xtags_falls_back_to_audio_is_default_heuristic() { + // No xtags → track_type None → is_descriptive == !audioIsDefault. + let a = build_audio(&audio_fmt(None, false), "vid", "cpn", None) + .unwrap() + .unwrap(); + assert_eq!(a.track_type, None); + assert!(a.is_descriptive); // !false + + let a = build_audio(&audio_fmt(None, true), "vid", "cpn", None) + .unwrap() + .unwrap(); + assert_eq!(a.track_type, None); + assert!(!a.is_descriptive); // !true + } + + #[test] + fn unparseable_xtags_falls_back_to_heuristic() { + // Garbage xtags → extract_audio_track_type None → heuristic used. + let a = build_audio(&audio_fmt(Some("!!!not-b64!!!"), false), "vid", "cpn", None) + .unwrap() + .unwrap(); + assert_eq!(a.track_type, None); + assert!(a.is_descriptive); + } + + // ---- S3 narrowed-$fields feed-enrich fetch + HARD fallback ------------ + // + // These touch the process-global Downloader, so they serialize on + // DOWNLOADER_TEST_LOCK (shared with the player_manager / stream_helper + // stub tests). They drive `fetch_android_metadata` DIRECTLY with an + // explicit `po_token: None` so they exercise the anonymous narrowed-reel + // path without depending on the process-global PoTokenProvider state that + // the potoken tests mutate. + + use crate::downloader::request::Request; + use crate::downloader::response::Response; + use crate::downloader::Downloader; + use crate::exceptions::NetworkError; + use crate::newpipe::DOWNLOADER_TEST_LOCK; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::Arc; + + /// Serves the reel endpoint two ways, distinguished by the `$fields` + /// selector in the URL: the NARROWED metadata request (selector contains a + /// nested `playerResponse.` path) vs the FULL request (`$fields=playerResponse`). + /// Counts each so tests can assert whether the fallback fired. `visitor_id` + /// is declined (best-effort None) so the reel proceeds without it. + struct ReelStub { + narrowed_body: String, + full_body: String, + narrowed_hits: AtomicUsize, + full_hits: AtomicUsize, + } + + impl ReelStub { + fn new(narrowed_body: &str, full_body: &str) -> Self { + Self { + narrowed_body: narrowed_body.to_string(), + full_body: full_body.to_string(), + narrowed_hits: AtomicUsize::new(0), + full_hits: AtomicUsize::new(0), + } + } + } + + impl Downloader for ReelStub { + fn execute(&self, request: Request) -> Result { + let url = request.url().to_string(); + if url.contains("visitor_id") { + // Best-effort path: decline → reel proceeds without visitorData. + return Err(NetworkError::Transport("reel-stub: no visitor".into())); + } + if url.contains("reel_item_watch") { + // The nested dot-selector (`playerResponse.playabilityStatus,...`) + // only appears on the narrowed metadata request; the full request + // ends `&$fields=playerResponse` with no dot. + let (body, counter) = if url.contains("playerResponse.") { + (&self.narrowed_body, &self.narrowed_hits) + } else { + (&self.full_body, &self.full_hits) + }; + counter.fetch_add(1, Ordering::SeqCst); + return Ok(Response::new(200, "OK", Default::default(), body.clone(), url)); + } + Err(NetworkError::Transport("reel-stub: unexpected request".into())) + } + } + + #[test] + fn feed_metadata_falls_back_to_full_reel_when_narrowed_lacks_videodetails() { + let _g = DOWNLOADER_TEST_LOCK.lock(); + stream_helper::reset_visitor_data_cache(); + // Narrowed response has playabilityStatus but NO videoDetails — the + // shape the endpoint would return if it ignored/rejected the nested + // `$fields` selector into an error/empty body. The full response carries + // the real videoDetails + streamingData. + let stub = Arc::new(ReelStub::new( + r#"{"playerResponse":{"playabilityStatus":{"status":"OK"}}}"#, + r#"{"playerResponse":{"playabilityStatus":{"status":"OK"},"videoDetails":{"videoId":"vid","title":"FULL","lengthSeconds":"100","viewCount":"5"},"streamingData":{"formats":[]}}}"#, + )); + NewPipe::init(stub.clone() as Arc); + + let loc = NewPipe::preferred_localization(); + let cc = NewPipe::preferred_content_country(); + let out = fetch_android_metadata("vid", &loc, &cc, "cpn", None, None).unwrap(); + + // The narrowed request was tried, found videoDetails-less, and the full + // fetch was used as the safety fallback — feed enrich never regresses. + assert_eq!(stub.narrowed_hits.load(Ordering::SeqCst), 1, "narrowed tried"); + assert_eq!(stub.full_hits.load(Ordering::SeqCst), 1, "fell back to full"); + // The value returned is the FULL response (unwrapped playerResponse). + assert_eq!( + out.get("videoDetails").and_then(|v| v.get("title")).and_then(|v| v.as_str()), + Some("FULL") + ); + assert!(out.get("streamingData").is_some()); + stream_helper::reset_visitor_data_cache(); + } + + #[test] + fn feed_metadata_uses_narrowed_response_when_videodetails_present() { + let _g = DOWNLOADER_TEST_LOCK.lock(); + stream_helper::reset_visitor_data_cache(); + // Narrowed response DOES carry videoDetails → the endpoint honored the + // selector; use it directly, NO full-fetch fallback (this is the + // bytes-saved happy path). + let stub = Arc::new(ReelStub::new( + r#"{"playerResponse":{"playabilityStatus":{"status":"OK"},"videoDetails":{"videoId":"vid","title":"NARROWED","lengthSeconds":"42","viewCount":"7"}}}"#, + r#"{"playerResponse":{"playabilityStatus":{"status":"OK"},"videoDetails":{"videoId":"vid","title":"FULL"},"streamingData":{"formats":[]}}}"#, + )); + NewPipe::init(stub.clone() as Arc); + + let loc = NewPipe::preferred_localization(); + let cc = NewPipe::preferred_content_country(); + let out = fetch_android_metadata("vid", &loc, &cc, "cpn", None, None).unwrap(); + + assert_eq!(stub.narrowed_hits.load(Ordering::SeqCst), 1, "narrowed used"); + assert_eq!(stub.full_hits.load(Ordering::SeqCst), 0, "no fallback needed"); + assert_eq!( + out.get("videoDetails").and_then(|v| v.get("title")).and_then(|v| v.as_str()), + Some("NARROWED") + ); + // Narrowed body has no streamingData — proof we did NOT silently fall + // back to the full response. + assert!(out.get("streamingData").is_none()); + stream_helper::reset_visitor_data_cache(); + } + + #[test] + fn feed_metadata_falls_back_when_narrowed_videoid_mismatches() { + let _g = DOWNLOADER_TEST_LOCK.lock(); + stream_helper::reset_visitor_data_cache(); + // Narrowed response carries a videoDetails, but its videoId is NOT the + // requested one (a gutted/decoy shape the endpoint could return if it + // only half-honored the nested selector). The videoId-binding guard must + // REJECT it — a bare is_some() would have accepted this empty/decoy + // metadata — and fall through to the full fetch. + let stub = Arc::new(ReelStub::new( + r#"{"playerResponse":{"playabilityStatus":{"status":"OK"},"videoDetails":{"videoId":"DECOY","title":"WRONG"}}}"#, + r#"{"playerResponse":{"playabilityStatus":{"status":"OK"},"videoDetails":{"videoId":"vid","title":"FULL","lengthSeconds":"100","viewCount":"5"},"streamingData":{"formats":[]}}}"#, + )); + NewPipe::init(stub.clone() as Arc); + + let loc = NewPipe::preferred_localization(); + let cc = NewPipe::preferred_content_country(); + let out = fetch_android_metadata("vid", &loc, &cc, "cpn", None, None).unwrap(); + + assert_eq!(stub.narrowed_hits.load(Ordering::SeqCst), 1, "narrowed tried"); + assert_eq!(stub.full_hits.load(Ordering::SeqCst), 1, "mismatch -> fell back to full"); + assert_eq!( + out.get("videoDetails").and_then(|v| v.get("title")).and_then(|v| v.as_str()), + Some("FULL"), + "returned the full (correct-id) response, not the decoy" + ); + stream_helper::reset_visitor_data_cache(); + } } diff --git a/src/youtube/stream_helper.rs b/src/youtube/stream_helper.rs index 69b2571..e43c1f7 100644 --- a/src/youtube/stream_helper.rs +++ b/src/youtube/stream_helper.rs @@ -6,6 +6,8 @@ // serviceIntegrityDimensions for poToken), POSTs to the right URL with // the right headers, returns the parsed JSON. +use once_cell::sync::Lazy; +use parking_lot::RwLock; use serde_json::{json, Map, Value}; use crate::downloader::request::Request; @@ -15,7 +17,8 @@ use crate::newpipe::NewPipe; use crate::youtube::client_request::{build_envelope, InnertubeClientRequestInfo}; use crate::youtube::constants::*; use crate::youtube::parsing::{ - android_user_agent, ios_user_agent, mobile_post_headers, youtube_post_headers, + android_user_agent, ios_user_agent, mobile_post_headers, visionos_user_agent, + youtube_post_headers, }; /// Builds a 12-char alphanumeric `cpn` (content playback nonce). NPE uses @@ -72,6 +75,145 @@ fn envelope_to_body(envelope: Value) -> Map { } } +/// A cached `visitorData` plus the instant it was fetched (for TTL expiry). +struct CachedVisitorData { + value: String, + at: std::time::Instant, +} + +impl CachedVisitorData { + fn is_fresh(&self) -> bool { + self.at.elapsed() < VISITOR_DATA_TTL + } +} + +/// How long a cached `visitorData` is reused before a fresh one is fetched. +/// Upstream fetches a brand-new visitorData on EVERY call; we cache to avoid a +/// round-trip per video, but bound the reuse so (a) traffic isn't pinned to a +/// single visitor identity indefinitely (rate-flag risk) and (b) a +/// subtly-bad-but-non-empty value can't ride the PRIMARY android reel call — +/// whose failure aborts extraction — forever. 15 min is a small fraction of +/// YouTube's own visitorData lifetime while still rotating regularly. +const VISITOR_DATA_TTL: std::time::Duration = std::time::Duration::from_secs(15 * 60); + +/// Process-wide cached `visitorData`. See [`CachedVisitorData`] / TTL above. +static VISITOR_DATA_CACHE: Lazy>> = + Lazy::new(|| RwLock::new(None)); + +/// Best-effort `visitorData` for the InnerTube client context. +/// +/// Upstream (`YoutubeStreamHelper`) attaches a freshly-fetched visitorData to +/// every non-token reel/web/ios/visionos call — *"We must always pass a valid +/// visitorData to get valid player responses"* — via +/// `YoutubeParsingHelper.getVisitorDataFromInnertube` (POST to the +/// `visitor_id` endpoint, read `responseContext.visitorData`). +/// +/// DELIBERATE DEVIATION (Loop-2 best-effort mandate): upstream THROWS when +/// visitorData can't be obtained, which would fail the whole extraction. +/// strawcore's anonymous happy-path has always worked WITHOUT visitorData, so +/// any failure here returns `None` and the caller proceeds exactly as it does +/// today. This is a durability improvement that can never regress the current +/// path — sending visitorData when we can get it, silently skipping it when we +/// can't. +/// +/// The cache is bounded by [`VISITOR_DATA_TTL`] and can be dropped early via +/// [`reset_visitor_data_cache`] (called when the PRIMARY android response is +/// rejected, so a poisoned value self-heals). Locking mirrors the Loop-1 +/// `player_manager` discipline: the `RwLock` is NEVER held across the network. +/// read (fresh?) → (miss/expired) fetch unlocked → write. A rare concurrent +/// double-fetch simply stores one of two interchangeable values. +fn get_visitor_data( + info: &InnertubeClientRequestInfo, + localization: &Localization, + content_country: &ContentCountry, + headers: Vec<(String, String)>, + domain: &str, +) -> Option { + // Serve a still-fresh cached value; an expired one falls through to refetch. + if let Some(cached) = VISITOR_DATA_CACHE.read().as_ref() { + if cached.is_fresh() && !visitor_force_stale() { + return Some(cached.value.clone()); + } + } + let env = build_envelope(info, localization, content_country, None); + let body = Value::Object(envelope_to_body(env)); + let url = format!("{domain}visitor_id{DISABLE_PRETTY_PRINT_PARAM}"); + // post_youtube returns Err on any non-200 / transport / parse failure → + // best-effort None (the caller then proceeds without visitorData). An + // expired-but-present cache entry is deliberately NOT served on a failed + // refetch: a stale visitorData is the thing we're rotating away from. + let parsed = post_youtube(&url, &body, headers).ok()?; + let visitor = visitor_data_from_response(&parsed)?; + *VISITOR_DATA_CACHE.write() = Some(CachedVisitorData { + value: visitor.clone(), + at: std::time::Instant::now(), + }); + Some(visitor) +} + +/// Drop the cached `visitorData` so the next extraction fetches a fresh one. +/// Called when the PRIMARY android response is rejected (decoy / playability +/// failure): those are exactly the symptoms a bad visitorData would produce on +/// the primary path, and since that failure aborts extraction the value would +/// otherwise never self-heal. +pub(crate) fn reset_visitor_data_cache() { + *VISITOR_DATA_CACHE.write() = None; +} + +/// Pull `responseContext.visitorData` out of a visitor_id response, treating an +/// absent or empty value as failure (`None`) — the trigger for the best-effort +/// fallback to "no visitorData" (today's behavior). +fn visitor_data_from_response(parsed: &Value) -> Option { + parsed + .get("responseContext") + .and_then(|v| v.get("visitorData")) + .and_then(|v| v.as_str()) + .filter(|s| !s.is_empty()) + .map(str::to_string) +} + +/// Test-only override that forces a cached entry to be treated as stale, +/// exercising the TTL-expiry refetch branch deterministically (fabricating an +/// old `Instant` via `checked_sub` is process/host-uptime dependent). Compiled +/// out entirely in non-test builds. +#[cfg(test)] +static FORCE_VISITOR_STALE: std::sync::atomic::AtomicBool = + std::sync::atomic::AtomicBool::new(false); + +#[inline] +fn visitor_force_stale() -> bool { + #[cfg(test)] + { + FORCE_VISITOR_STALE.load(std::sync::atomic::Ordering::Relaxed) + } + #[cfg(not(test))] + { + false + } +} + +/// Test hook: pre-seed the visitorData cache with a fresh entry (at = now) so +/// the cache-hit read path can be exercised without a network round-trip. +#[cfg(test)] +fn seed_visitor_data_cache(v: &str) { + *VISITOR_DATA_CACHE.write() = Some(CachedVisitorData { + value: v.to_string(), + at: std::time::Instant::now(), + }); +} + +/// Test hook: read the current cached visitorData value (if any). +#[cfg(test)] +fn peek_visitor_data_cache() -> Option { + VISITOR_DATA_CACHE.read().as_ref().map(|c| c.value.clone()) +} + +/// Test hook: force/clear the stale override (see [`FORCE_VISITOR_STALE`]). +#[cfg(test)] +fn set_force_visitor_stale(on: bool) { + FORCE_VISITOR_STALE.store(on, std::sync::atomic::Ordering::Relaxed); +} + /// WEB-client metadata-only /player call — used for microformat + /// thumbnails only; never used as a stream URL source. pub fn get_web_metadata_player_response( @@ -80,7 +222,18 @@ pub fn get_web_metadata_player_response( content_country: &ContentCountry, signature_timestamp: i32, ) -> Result { - let info = InnertubeClientRequestInfo::of_web_client(); + let mut info = InnertubeClientRequestInfo::of_web_client(); + // Best-effort visitorData (upstream sends it on every non-token /player + // call). WEB uses the youtube.com domain + web headers for visitor_id. + if let Some(v) = get_visitor_data( + &info, + localization, + content_country, + youtube_post_headers(), + YOUTUBEI_V1_URL, + ) { + info.client_info.visitor_data = Some(v); + } let env = build_envelope(&info, localization, content_country, None); let mut body = envelope_to_body(env); add_player_body_fields(&mut body, video_id, &generate_content_playback_nonce()); @@ -123,14 +276,75 @@ pub fn get_android_player_response( /// ANDROID `/reel/reel_item_watch` fallback — used when no poToken is /// available. Returns a `playerResponse`-shaped JSON wrapped inside the -/// reel response. +/// reel response. Requests the FULL `playerResponse` (`$fields=playerResponse`) +/// — the stream-extraction path needs `streamingData` and everything else. pub fn get_android_reel_player_response( video_id: &str, localization: &Localization, content_country: &ContentCountry, cpn: &str, ) -> Result { - let info = InnertubeClientRequestInfo::of_android_client(); + reel_player_response_with_fields(video_id, localization, content_country, cpn, "playerResponse") +} + +/// Metadata-scoped variant of [`get_android_reel_player_response`] for the cheap +/// `stream_metadata` feed-enrich path. Requests a NARROWED `$fields` selector so +/// only `playabilityStatus` + `videoDetails` come back (~3-6 KB) instead of the +/// full ~150-500 KB reel body carrying every adaptive format's ~1.5 KB URL — the +/// feed path reads nothing outside those two subtrees (S3). +/// +/// BEST-EFFORT, with a full-fetch fallback owned by the caller +/// (`stream_extractor::fetch_android_metadata`): the `reel_item_watch` endpoint's +/// acceptance of the NESTED `$fields` selector (`playerResponse.videoDetails`) is +/// not guaranteed. If it honors the selector we save the bytes; if it rejects it +/// (400 → `Err` here) or returns a shape without `videoDetails`, the caller falls +/// back to the full reel fetch, so feed enrichment can NEVER regress. The +/// dot-form nested selector matches the spelling upstream/`get_web_metadata_player_response` +/// already use against the /player endpoint (`videoDetails.thumbnail.thumbnails`), +/// so the syntax itself is proven — only this endpoint's honoring of it is not. +pub fn get_android_reel_player_response_metadata( + video_id: &str, + localization: &Localization, + content_country: &ContentCountry, + cpn: &str, +) -> Result { + reel_player_response_with_fields( + video_id, + localization, + content_country, + cpn, + "playerResponse.playabilityStatus,playerResponse.videoDetails", + ) +} + +/// Shared body of the two reel helpers. `fields` is the `$fields` selector: the +/// full path passes `"playerResponse"` (byte-for-byte today's request), the +/// metadata path passes the narrowed nested selector. Everything else — the +/// best-effort visitorData, the envelope, the `playerRequest`/body shape, the +/// headers — is identical across both. +fn reel_player_response_with_fields( + video_id: &str, + localization: &Localization, + content_country: &ContentCountry, + cpn: &str, + fields: &str, +) -> Result { + let mut info = InnertubeClientRequestInfo::of_android_client(); + let ua = android_user_agent(content_country); + // Best-effort visitorData (upstream sends it on the reel call too). The reel + // path uses the gapis domain + android mobile headers for visitor_id. + // NOTE: the reel body shape is otherwise UNCHANGED from today (the upstream + // playerRequest/disablePlayerResponse realignment — recon W-1 — is out of + // scope for this pass; only touch it if a decoy/empty regression appears). + if let Some(v) = get_visitor_data( + &info, + localization, + content_country, + mobile_post_headers(&ua), + YOUTUBEI_V1_GAPIS_URL, + ) { + info.client_info.visitor_data = Some(v); + } let env = build_envelope(&info, localization, content_country, None); let mut body = envelope_to_body(env); body.insert( @@ -142,10 +356,9 @@ pub fn get_android_reel_player_response( ); add_player_body_fields(&mut body, video_id, cpn); let url = format!( - "{YOUTUBEI_V1_GAPIS_URL}reel/reel_item_watch{DISABLE_PRETTY_PRINT_PARAM}&t={t}&id={video_id}&$fields=playerResponse", + "{YOUTUBEI_V1_GAPIS_URL}reel/reel_item_watch{DISABLE_PRETTY_PRINT_PARAM}&t={t}&id={video_id}&$fields={fields}", t = generate_content_playback_nonce() ); - let ua = android_user_agent(content_country); post_youtube(&url, &Value::Object(body), mobile_post_headers(&ua)) } @@ -163,8 +376,20 @@ pub fn get_ios_player_response( visitor_data: Option<&str>, ) -> Result { let mut info = InnertubeClientRequestInfo::of_ios_client(); + let ua = ios_user_agent(content_country); + // Caller-supplied visitorData (paired with a poToken) wins; otherwise fetch + // it best-effort. iOS uses the youtube.com domain for visitor_id (upstream + // getIosPlayerResponse) even though the /player call goes to gapis. if let Some(v) = visitor_data { info.client_info.visitor_data = Some(v.into()); + } else if let Some(v) = get_visitor_data( + &info, + localization, + content_country, + mobile_post_headers(&ua), + YOUTUBEI_V1_URL, + ) { + info.client_info.visitor_data = Some(v); } let env = build_envelope(&info, localization, content_country, None); let mut body = envelope_to_body(env); @@ -176,7 +401,42 @@ pub fn get_ios_player_response( "{YOUTUBEI_V1_GAPIS_URL}player{DISABLE_PRETTY_PRINT_PARAM}&t={t}&id={video_id}", t = generate_content_playback_nonce() ); - let ua = ios_user_agent(content_country); + post_youtube(&url, &Value::Object(body), mobile_post_headers(&ua)) +} + +/// visionOS /player call (client 101). Best-effort 4th client that recovers +/// higher-than-360p streams for SABR-only / "made for kids" videos. Mirrors +/// NPE `getVisionOsPlayerResponse`: hits the gapis /player endpoint with the +/// visionOS mobile header set, carries a best-effort visitorData, and NO +/// poToken. +pub fn get_visionos_player_response( + video_id: &str, + localization: &Localization, + content_country: &ContentCountry, + cpn: &str, +) -> Result { + let mut info = InnertubeClientRequestInfo::of_visionos_client(); + let ua = visionos_user_agent(content_country); + // visionOS returns valid responses ONLY with a visitorData — but still + // best-effort: on failure we proceed without it (the whole visionOS fetch + // is itself swallowed best-effort by the caller). Uses the youtube.com + // domain for visitor_id (upstream), gapis for /player. + if let Some(v) = get_visitor_data( + &info, + localization, + content_country, + mobile_post_headers(&ua), + YOUTUBEI_V1_URL, + ) { + info.client_info.visitor_data = Some(v); + } + let env = build_envelope(&info, localization, content_country, None); + let mut body = envelope_to_body(env); + add_player_body_fields(&mut body, video_id, cpn); + let url = format!( + "{YOUTUBEI_V1_GAPIS_URL}player{DISABLE_PRETTY_PRINT_PARAM}&t={t}&id={video_id}", + t = generate_content_playback_nonce() + ); post_youtube(&url, &Value::Object(body), mobile_post_headers(&ua)) } @@ -195,8 +455,12 @@ fn post_youtube( } let resp = downloader.execute(builder.build())?; if resp.response_code() != 200 { + // Privacy: the full URL carries `id=` in the query — never + // put it in an error string (they reach Kotlin exception messages + // and logs). Scheme+host+path identifies the endpoint just fine. + let endpoint = url.split(['?', '#']).next().unwrap_or(url); return Err(ExtractionError::Network(NetworkError::Transport(format!( - "HTTP {} from {url}", + "HTTP {} from {endpoint}", resp.response_code() )))); } @@ -227,4 +491,133 @@ mod tests { let b = generate_content_playback_nonce(); assert_ne!(a, b); } + + #[test] + fn visitor_data_parsed_from_response() { + let resp = json!({ + "responseContext": { "visitorData": "Cgs1ZG1abc==" }, + "other": 1 + }); + assert_eq!( + visitor_data_from_response(&resp).as_deref(), + Some("Cgs1ZG1abc==") + ); + } + + #[test] + fn visitor_data_absent_or_empty_is_none_triggering_fallback() { + // Missing responseContext → None (→ caller proceeds without visitorData, + // i.e. exactly today's behavior). + assert!(visitor_data_from_response(&json!({})).is_none()); + // Present but empty string → None (empty is not a valid visitorData). + let resp = json!({ "responseContext": { "visitorData": "" } }); + assert!(visitor_data_from_response(&resp).is_none()); + // responseContext without the field → None. + let resp = json!({ "responseContext": { "foo": "bar" } }); + assert!(visitor_data_from_response(&resp).is_none()); + } + + // ---- visitorData cache: fresh-hit / TTL-expiry / reset ---------------- + // + // These touch the process-global visitorData cache and (for the refetch + // test) the global Downloader, so they serialize on DOWNLOADER_TEST_LOCK, + // shared with the player_manager test module. + + use crate::downloader::request::Request; + use crate::downloader::response::Response; + use crate::downloader::Downloader; + use crate::newpipe::DOWNLOADER_TEST_LOCK; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::Arc; + + /// Serves any `visitor_id` POST with a fixed visitorData; counts requests. + struct VisitorStub { + visitor: String, + requests: AtomicUsize, + } + + impl Downloader for VisitorStub { + fn execute(&self, request: Request) -> Result { + self.requests.fetch_add(1, Ordering::SeqCst); + let url = request.url().to_string(); + if url.contains("visitor_id") { + let body = format!( + r#"{{"responseContext":{{"visitorData":"{}"}}}}"#, + self.visitor + ); + Ok(Response::new(200, "OK", Default::default(), body, url)) + } else { + Err(NetworkError::Transport("visitor-stub: unexpected request".into())) + } + } + } + + #[test] + fn fresh_cached_visitor_data_served_without_network() { + let _g = DOWNLOADER_TEST_LOCK.lock(); + set_force_visitor_stale(false); + reset_visitor_data_cache(); + seed_visitor_data_cache("FRESH_VD"); + // No downloader is needed: a fresh cache hit returns before any fetch. + let info = InnertubeClientRequestInfo::of_android_client(); + let got = get_visitor_data( + &info, + &Localization::default(), + &ContentCountry::default(), + mobile_post_headers("ua/1.0"), + YOUTUBEI_V1_GAPIS_URL, + ); + assert_eq!(got.as_deref(), Some("FRESH_VD")); + reset_visitor_data_cache(); + } + + #[test] + fn expired_visitor_data_triggers_refetch() { + let _g = DOWNLOADER_TEST_LOCK.lock(); + let stub = Arc::new(VisitorStub { + visitor: "REFRESHED_VD".into(), + requests: AtomicUsize::new(0), + }); + NewPipe::init(stub.clone() as Arc); + + reset_visitor_data_cache(); + seed_visitor_data_cache("STALE_VD"); // fresh instant… + set_force_visitor_stale(true); // …but forced past its TTL + + let info = InnertubeClientRequestInfo::of_android_client(); + let got = get_visitor_data( + &info, + &Localization::default(), + &ContentCountry::default(), + mobile_post_headers("ua/1.0"), + YOUTUBEI_V1_GAPIS_URL, + ); + + assert_eq!(got.as_deref(), Some("REFRESHED_VD"), "expired entry must refetch"); + assert_eq!( + stub.requests.load(Ordering::SeqCst), + 1, + "exactly one visitor_id refetch" + ); + // The refetched value replaced the stale one, stamped fresh. + set_force_visitor_stale(false); + assert_eq!(peek_visitor_data_cache().as_deref(), Some("REFRESHED_VD")); + + set_force_visitor_stale(false); + reset_visitor_data_cache(); + } + + #[test] + fn reset_clears_visitor_data_cache_for_self_heal() { + // Fix 2b mechanism: on a primary-android rejection the extractor calls + // reset_visitor_data_cache() so a poisoned value doesn't ride the + // primary reel call forever. + let _g = DOWNLOADER_TEST_LOCK.lock(); + set_force_visitor_stale(false); + reset_visitor_data_cache(); + seed_visitor_data_cache("POISONED_VD"); + assert_eq!(peek_visitor_data_cache().as_deref(), Some("POISONED_VD")); + reset_visitor_data_cache(); + assert!(peek_visitor_data_cache().is_none()); + } } diff --git a/src/youtube/xtags.rs b/src/youtube/xtags.rs new file mode 100644 index 0000000..54c7d0b --- /dev/null +++ b/src/youtube/xtags.rs @@ -0,0 +1,297 @@ +// Minimal reader for an itag format's `xtags` field → AudioTrackType. +// +// Mirrors NPE YoutubeParsingHelper.extractAudioTrackType (commits b0bca7e7 + +// e3479a7c, 2026-06-01), which replaced the old `audioTrack.audioIsDefault` +// heuristic with the authoritative `acont` value carried in the format's +// `xtags` blob. +// +// `xtags` is a base64url-encoded `youtube.video.XTags` protobuf (upstream +// proto/youtube/video/xtags.proto): +// +// message KeyValuePair { optional string key = 1; optional string value = 2; } +// message XTags { repeated KeyValuePair xtags = 1; } +// +// The structure is trivial (two nested length-delimited string fields), so we +// read it with a ~40-line varint/field walker instead of pulling in a protobuf +// crate — matching the codebase's minimal-dependency approach. Everything is +// best-effort: any malformed input (bad base64, truncated protobuf, unknown +// `acont` value) yields `None`, and the caller falls back to the legacy +// `audioIsDefault` heuristic — never a panic, never a break. + +use crate::stream::AudioTrackType; + +/// Decode a format's `xtags` string and return its audio track type, mirroring +/// NPE `extractAudioTrackType`. Returns `None` when the blob is absent-shaped, +/// undecodable, carries no `acont` key, or maps to an unknown value. +pub fn extract_audio_track_type(xtags: &str) -> Option { + let bytes = base64url_decode(xtags)?; + let acont = xtags_find(&bytes, "acont")?; + match acont.as_str() { + "original" => Some(AudioTrackType::Original), + // NPE maps both "dubbed" and "dubbed-auto" → DUBBED. + "dubbed" | "dubbed-auto" => Some(AudioTrackType::Dubbed), + "descriptive" => Some(AudioTrackType::Descriptive), + "secondary" => Some(AudioTrackType::Secondary), + _ => None, + } +} + +/// Walk the top-level `XTags` message, returning the `value` of the first +/// `KeyValuePair` (field 1) whose `key` equals `wanted`. Unknown fields and +/// non-string wire types are skipped; any framing error bails to `None`. +fn xtags_find(buf: &[u8], wanted: &str) -> Option { + let mut i = 0; + while i < buf.len() { + let (tag, adv) = read_varint(buf, i)?; + i += adv; + let field = tag >> 3; + match wire_type(tag) { + // Length-delimited: field 1 is a KeyValuePair submessage. + 2 => { + let payload = read_len_delimited(buf, &mut i)?; + if field == 1 { + if let Some((k, v)) = parse_key_value_pair(payload) { + if k.as_deref() == Some(wanted) { + return v; + } + } + } + } + wt => skip_scalar(buf, &mut i, wt)?, + } + } + None +} + +/// Parse a `KeyValuePair` submessage into (key, value); either may be absent. +fn parse_key_value_pair(buf: &[u8]) -> Option<(Option, Option)> { + let mut key = None; + let mut val = None; + let mut i = 0; + while i < buf.len() { + let (tag, adv) = read_varint(buf, i)?; + i += adv; + let field = tag >> 3; + match wire_type(tag) { + 2 => { + let payload = read_len_delimited(buf, &mut i)?; + let s = std::str::from_utf8(payload).ok()?.to_string(); + match field { + 1 => key = Some(s), + 2 => val = Some(s), + _ => {} + } + } + wt => skip_scalar(buf, &mut i, wt)?, + } + } + Some((key, val)) +} + +#[inline] +fn wire_type(tag: u64) -> u8 { + (tag & 0x7) as u8 +} + +/// Read a length-delimited (wire type 2) payload, advancing `*i` past it. +fn read_len_delimited<'a>(buf: &'a [u8], i: &mut usize) -> Option<&'a [u8]> { + let (len, adv) = read_varint(buf, *i)?; + *i += adv; + // `usize::try_from` (not `as usize`) so a >4 GiB length can't wrap on a + // 32-bit target (armeabi-v7a is a real Straw ABI) — it fails closed instead. + let len = usize::try_from(len).ok()?; + let end = i.checked_add(len)?; + if end > buf.len() { + return None; + } + let payload = &buf[*i..end]; + *i = end; + Some(payload) +} + +/// Advance `*i` past a non-length-delimited field. Groups (3/4) and any +/// unrecognized wire type bail out — they never appear in a well-formed XTags. +fn skip_scalar(buf: &[u8], i: &mut usize, wire: u8) -> Option<()> { + match wire { + 0 => { + let (_, adv) = read_varint(buf, *i)?; + *i += adv; + } + 1 => { + *i = i.checked_add(8)?; + if *i > buf.len() { + return None; + } + } + 5 => { + *i = i.checked_add(4)?; + if *i > buf.len() { + return None; + } + } + _ => return None, + } + Some(()) +} + +/// Read a base-128 varint at `start`; returns (value, bytes_consumed). Caps at +/// 10 bytes (64 bits) so a malformed run can't spin. +fn read_varint(buf: &[u8], start: usize) -> Option<(u64, usize)> { + let mut result: u64 = 0; + let mut shift: u32 = 0; + let mut i = start; + loop { + if i >= buf.len() || shift >= 64 { + return None; + } + let byte = buf[i]; + result |= u64::from(byte & 0x7f) << shift; + i += 1; + if byte & 0x80 == 0 { + return Some((result, i - start)); + } + shift += 7; + } +} + +/// Strict base64url (RFC 4648 §5, no padding required) decoder. Rejects any +/// non-url-safe byte (`+`, `/`, whitespace) → `None`, matching upstream's use +/// of `Base64.getUrlDecoder()`. Stops at the first `=` padding byte. +fn base64url_decode(s: &str) -> Option> { + fn sextet(c: u8) -> Option { + match c { + b'A'..=b'Z' => Some(c - b'A'), + b'a'..=b'z' => Some(c - b'a' + 26), + b'0'..=b'9' => Some(c - b'0' + 52), + b'-' => Some(62), + b'_' => Some(63), + _ => None, + } + } + if s.is_empty() { + return None; + } + let mut out = Vec::with_capacity(s.len() * 3 / 4 + 3); + let mut acc: u32 = 0; + let mut bits: u32 = 0; + for &c in s.as_bytes() { + if c == b'=' { + break; + } + let v = u32::from(sextet(c)?); + acc = (acc << 6) | v; + bits += 6; + if bits >= 8 { + bits -= 8; + out.push((acc >> bits) as u8); + } + } + Some(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + // Independently-generated (Python urlsafe_b64encode of a hand-built XTags + // protobuf) so these vectors are an external oracle, not a round-trip of + // our own encoder. See the Loop-2 build notes for the generator. + const XT_ORIGINAL: &str = "ChEKBWFjb250EghvcmlnaW5hbA"; + const XT_DUBBED: &str = "Cg8KBWFjb250EgZkdWJiZWQ"; + const XT_DUBBED_AUTO: &str = "ChQKBWFjb250EgtkdWJiZWQtYXV0bw"; + const XT_DESCRIPTIVE: &str = "ChQKBWFjb250EgtkZXNjcmlwdGl2ZQ"; + const XT_SECONDARY: &str = "ChIKBWFjb250EglzZWNvbmRhcnk"; + // Two pairs: {lang=en},{acont=descriptive} — proves we scan past a + // non-matching pair to find `acont`. + const XT_MULTI: &str = "CgoKBGxhbmcSAmVuChQKBWFjb250EgtkZXNjcmlwdGl2ZQ"; + // Single pair {lang=en} — no `acont` key. + const XT_NO_ACONT: &str = "CgoKBGxhbmcSAmVu"; + + #[test] + fn maps_each_known_acont_value() { + assert_eq!( + extract_audio_track_type(XT_ORIGINAL), + Some(AudioTrackType::Original) + ); + assert_eq!( + extract_audio_track_type(XT_DUBBED), + Some(AudioTrackType::Dubbed) + ); + assert_eq!( + extract_audio_track_type(XT_DUBBED_AUTO), + Some(AudioTrackType::Dubbed) + ); + assert_eq!( + extract_audio_track_type(XT_DESCRIPTIVE), + Some(AudioTrackType::Descriptive) + ); + assert_eq!( + extract_audio_track_type(XT_SECONDARY), + Some(AudioTrackType::Secondary) + ); + } + + #[test] + fn finds_acont_among_multiple_pairs() { + assert_eq!( + extract_audio_track_type(XT_MULTI), + Some(AudioTrackType::Descriptive) + ); + } + + #[test] + fn absent_acont_key_yields_none() { + assert_eq!(extract_audio_track_type(XT_NO_ACONT), None); + } + + #[test] + fn malformed_input_yields_none_never_panics() { + // Empty, non-base64url chars, valid base64url but not a protobuf, + // and a truncated length-delimited frame. + assert_eq!(extract_audio_track_type(""), None); + assert_eq!(extract_audio_track_type("!!!not base64!!!"), None); + assert_eq!(extract_audio_track_type("++//"), None); // standard-b64 chars rejected + assert_eq!(extract_audio_track_type("Zm9vYmFy"), None); // "foobar", valid b64url, junk proto + // 0x0A (field1,LEN) claiming length 0x7F with no body → truncated. + assert_eq!(extract_audio_track_type("Cn8"), None); + } + + #[test] + fn unknown_acont_value_yields_none() { + // {acont=weird} — well-formed protobuf, value not in the enum. + // ld(1, ld(1,"acont")+ld(2,"weird")) + let bytes = { + let kv = [ + &[0x0a, 0x05][..], + b"acont", + &[0x12, 0x05], + b"weird", + ] + .concat(); + let mut top = vec![0x0a, kv.len() as u8]; + top.extend_from_slice(&kv); + top + }; + // sanity: our decoder round-trips what we assert on + let b64 = { + // encode without padding, url-safe + const A: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; + let mut out = String::new(); + for chunk in bytes.chunks(3) { + let b = [ + chunk[0], + *chunk.get(1).unwrap_or(&0), + *chunk.get(2).unwrap_or(&0), + ]; + let n = (u32::from(b[0]) << 16) | (u32::from(b[1]) << 8) | u32::from(b[2]); + let take = chunk.len() + 1; + for k in 0..take { + out.push(A[((n >> (18 - 6 * k)) & 0x3f) as usize] as char); + } + } + out + }; + assert_eq!(base64url_decode(&b64).unwrap(), bytes); + assert_eq!(extract_audio_track_type(&b64), None); + } +} diff --git a/tests/foundation_smoke.rs b/tests/foundation_smoke.rs index 1fb88f2..2544e87 100644 --- a/tests/foundation_smoke.rs +++ b/tests/foundation_smoke.rs @@ -9,16 +9,22 @@ use std::sync::Arc; -use strawcore::downloader::request::Request; -use strawcore::downloader::ReqwestDownloader; -use strawcore::exceptions::NetworkError; -use strawcore::localization::{ContentCountry, Localization}; -use strawcore::{Downloader, NewPipe}; +use strawcore_core::downloader::request::Request; +use strawcore_core::downloader::ReqwestDownloader; +use strawcore_core::exceptions::NetworkError; +use strawcore_core::localization::{ContentCountry, Localization}; +use strawcore_core::{Downloader, NewPipe}; + +/// The Downloader trait dropped its `get` convenience (commit f917e4a); +/// mirror it here so the suite reads as before. +fn get(dl: &impl Downloader, url: &str) -> Result { + dl.execute(Request::get(url).build()) +} #[test] fn get_through_default_downloader() { let dl = ReqwestDownloader::new().expect("build downloader"); - let resp = dl.get("https://httpbin.org/get").expect("transport"); + let resp = get(&dl, "https://httpbin.org/get").expect("transport"); assert_eq!(resp.response_code(), 200); assert!(resp.response_body().contains("\"url\"")); } @@ -26,9 +32,7 @@ fn get_through_default_downloader() { #[test] fn latest_url_follows_redirects() { let dl = ReqwestDownloader::new().expect("build downloader"); - let resp = dl - .get("https://httpbin.org/redirect/3") - .expect("transport"); + let resp = get(&dl, "https://httpbin.org/redirect/3").expect("transport"); assert_eq!(resp.response_code(), 200); assert!( resp.latest_url().ends_with("/get"), @@ -40,14 +44,14 @@ fn latest_url_follows_redirects() { #[test] fn non_2xx_returns_ok_not_err() { let dl = ReqwestDownloader::new().expect("build downloader"); - let resp = dl.get("https://httpbin.org/status/404").expect("transport"); + let resp = get(&dl, "https://httpbin.org/status/404").expect("transport"); assert_eq!(resp.response_code(), 404); } #[test] fn http_429_surfaces_as_recaptcha_err() { let dl = ReqwestDownloader::new().expect("build downloader"); - let err = dl.get("https://httpbin.org/status/429").expect_err("429 must be NetworkError"); + let err = get(&dl, "https://httpbin.org/status/429").expect_err("429 must be NetworkError"); match err { NetworkError::Recaptcha { url } => assert!(url.contains("/status/429")), other => panic!("expected Recaptcha, got {other:?}"), @@ -72,8 +76,8 @@ fn localization_header_attached_when_enabled() { #[test] fn header_keys_lowercased_in_response() { let dl = ReqwestDownloader::new().expect("build downloader"); - let resp = dl.get("https://httpbin.org/get").expect("transport"); - for (k, _) in resp.response_headers() { + let resp = get(&dl, "https://httpbin.org/get").expect("transport"); + for k in resp.response_headers().keys() { assert_eq!(k, &k.to_ascii_lowercase(), "header key {k} not lowercased"); } } @@ -88,7 +92,9 @@ fn newpipe_singleton_wires_downloader() { ); let from_global = NewPipe::downloader().expect("downloader registered"); - let resp = from_global.get("https://httpbin.org/get").expect("transport"); + let resp = from_global + .execute(Request::get("https://httpbin.org/get").build()) + .expect("transport"); assert_eq!(resp.response_code(), 200); assert_eq!(NewPipe::preferred_localization().localization_code(), "en-GB"); } diff --git a/tests/js_phase2_offline.rs b/tests/js_phase2_offline.rs index ea34f12..3e3a6ad 100644 --- a/tests/js_phase2_offline.rs +++ b/tests/js_phase2_offline.rs @@ -14,7 +14,7 @@ // * url_with_throttling_parameter_deobfuscated round-trip changes &n= // and caches the result -use strawcore::youtube::js::{signature, nsig, runtime, DeobfError}; +use strawcore_core::youtube::js::{signature, nsig, runtime, DeobfError}; // Synthetic minified player.js — replicates the shape of real YT player.js. //