diff --git a/Cargo.lock b/Cargo.lock index c4af244..82fb84a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -121,7 +121,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -711,9 +711,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.12.4" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" dependencies = [ "aho-corasick", "memchr", @@ -723,9 +723,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.14" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" dependencies = [ "aho-corasick", "memchr", @@ -891,9 +891,9 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "serde" -version = "1.0.229" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" dependencies = [ "serde_core", "serde_derive", @@ -901,29 +901,29 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.229" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.229" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn", ] [[package]] name = "serde_json" -version = "1.0.151" +version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ "itoa", "memchr", @@ -1017,17 +1017,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "syn" -version = "3.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - [[package]] name = "sync_wrapper" version = "1.0.2" @@ -1045,7 +1034,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -1074,7 +1063,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -1085,7 +1074,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -1327,7 +1316,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.117", + "syn", "wasm-bindgen-shared", ] @@ -1562,7 +1551,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", "synstructure", ] @@ -1583,7 +1572,7 @@ checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -1603,7 +1592,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", "synstructure", ] @@ -1643,7 +1632,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] diff --git a/src/downloader/default_impl.rs b/src/downloader/default_impl.rs index 9dcefea..9f3f5ba 100644 --- a/src/downloader/default_impl.rs +++ b/src/downloader/default_impl.rs @@ -81,17 +81,7 @@ impl Downloader for ReqwestDownloader { let url_after_redirects = resp.url().to_string(); if status.as_u16() == 429 { - // Privacy: this URL ends up in error strings (and from there in - // Kotlin exception messages / logs, and the deobf failure memo). - // YouTube URLs carry the video id in the query (`?v=` / `&id=`) - // AND in some paths (`/embed/`, `/shorts/`), so keep only - // scheme+host. The Response's latest_url below stays full — it's - // data consumed by redirect-tracking logic, not a message. - let mut stripped = resp.url().clone(); - stripped.set_query(None); - stripped.set_fragment(None); - stripped.set_path("/"); - return Err(NetworkError::Recaptcha { url: stripped.to_string() }); + return Err(NetworkError::Recaptcha { url: url_after_redirects }); } let code = status.as_u16(); diff --git a/src/downloader/response.rs b/src/downloader/response.rs index 4702277..cd34d32 100644 --- a/src/downloader/response.rs +++ b/src/downloader/response.rs @@ -54,15 +54,6 @@ impl Response { &self.response_body } - /// Consumes the response, handing back the owned body `String` with no - /// copy. Use at the last read of a body when the rest of the `Response` - /// is no longer needed — e.g. the ~1.5 MB player.js download, where - /// `response_body().to_string()` would clone the whole buffer only to - /// drop the original (S8 speed fix). - pub fn into_body(self) -> String { - self.response_body - } - pub fn latest_url(&self) -> &str { &self.latest_url } diff --git a/src/exceptions.rs b/src/exceptions.rs index cd19946..acdb096 100644 --- a/src/exceptions.rs +++ b/src/exceptions.rs @@ -83,12 +83,7 @@ pub enum ExtractionError { impl From for NetworkError { fn from(e: reqwest::Error) -> Self { - // Privacy: reqwest's Display appends " for url (…)" whenever the - // error carries a URL — for player requests that URL contains - // `id=`, i.e. what the user was watching. Error strings - // become Kotlin exception messages and land in logcat / exported - // logs, so strip the URL at this single choke point. - NetworkError::Transport(e.without_url().to_string()) + NetworkError::Transport(e.to_string()) } } diff --git a/src/newpipe.rs b/src/newpipe.rs index 0b9ead4..3fcb31c 100644 --- a/src/newpipe.rs +++ b/src/newpipe.rs @@ -66,10 +66,3 @@ impl NewPipe { *Self::instance().preferred_content_country.write() = content_country; } } - -/// Serializes every test that installs/uses a Downloader in the process-global -/// `NewPipe` singleton, so they don't clobber each other's registered -/// downloader (or corrupt request counts) when cargo runs tests in parallel. -/// Shared by the `player_manager` and `stream_helper` test modules. -#[cfg(test)] -pub(crate) static DOWNLOADER_TEST_LOCK: parking_lot::Mutex<()> = parking_lot::Mutex::new(()); diff --git a/src/stream/audio.rs b/src/stream/audio.rs index d40e1c0..079e806 100644 --- a/src/stream/audio.rs +++ b/src/stream/audio.rs @@ -3,21 +3,6 @@ use crate::stream::DeliveryMethod; use crate::youtube::itag::MediaFormat; -/// The track type of an [`AudioStream`]. Mirrors NPE -/// `stream/AudioTrackType.java`; derived from the format's `xtags` protobuf -/// (`acont` key) — see `youtube::xtags`. -#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] -pub enum AudioTrackType { - /// The original audio track of a video. - Original, - /// The original voices replaced, typically in a different language. - Dubbed, - /// A descriptive (audio-description) track for accessibility. - Descriptive, - /// A secondary track (e.g. an alternate/commentary track). - Secondary, -} - #[derive(Clone, Debug)] pub struct AudioStream { pub itag: u32, @@ -30,12 +15,6 @@ pub struct AudioStream { pub audio_track_id: Option, pub audio_track_name: Option, pub audio_locale: Option, - /// True iff this is a descriptive (audio-description) track. Preferentially - /// derived from `xtags` (`acont == descriptive`); falls back to the legacy - /// `audioTrack.audioIsDefault` heuristic when `xtags` is absent/unparseable. pub is_descriptive: bool, - /// The authoritative audio track type from the format's `xtags` blob, or - /// `None` for single-track audio / when `xtags` is absent or unparseable. - pub track_type: Option, pub itag_url_format: Option, } diff --git a/src/stream/mod.rs b/src/stream/mod.rs index c848cc7..9ee9b28 100644 --- a/src/stream/mod.rs +++ b/src/stream/mod.rs @@ -10,7 +10,7 @@ pub mod delivery; pub mod subtitles; pub mod video; -pub use audio::{AudioStream, AudioTrackType}; +pub use audio::AudioStream; pub use delivery::DeliveryMethod; pub use subtitles::SubtitlesStream; pub use video::VideoStream; diff --git a/src/youtube/client_request.rs b/src/youtube/client_request.rs index 15058bf..55577f4 100644 --- a/src/youtube/client_request.rs +++ b/src/youtube/client_request.rs @@ -111,29 +111,6 @@ impl InnertubeClientRequestInfo { }, } } - - /// visionOS client (client 101). Mirrors NPE - /// `InnertubeClientRequestInfo.ofVisionOsClient()`. Same MOBILE/Apple shape - /// as iOS but with the RealityDevice model + visionOS os fields. - pub fn of_visionos_client() -> Self { - Self { - client_info: ClientInfo { - client_name: VISIONOS_CLIENT_NAME.into(), - client_version: VISIONOS_CLIENT_VERSION.into(), - client_id: VISIONOS_CLIENT_ID.into(), - client_screen: Some(WATCH_CLIENT_SCREEN.into()), - visitor_data: None, - }, - device_info: DeviceInfo { - platform: Some(MOBILE_CLIENT_PLATFORM.into()), - device_make: Some("Apple".into()), - device_model: Some(VISIONOS_DEVICE_MODEL.into()), - os_name: Some("visionOS".into()), - os_version: Some(VISIONOS_OS_VERSION.into()), - android_sdk_version: -1, - }, - } - } } /// Builds the InnerTube request envelope mirroring NPE prepareJsonBuilder. @@ -298,30 +275,6 @@ mod tests { assert!(client.get("androidSdkVersion").is_none()); } - #[test] - fn visionos_client_envelope_shape() { - let info = InnertubeClientRequestInfo::of_visionos_client(); - let env = build_envelope( - &info, - &Localization::default(), - &ContentCountry::default(), - None, - ); - let client = &env["context"]["client"]; - assert_eq!(client["clientName"], "VISIONOS"); - assert_eq!(client["clientVersion"], "1.02"); - assert_eq!(client["platform"], "MOBILE"); - assert_eq!(client["deviceMake"], "Apple"); - assert_eq!(client["deviceModel"], "RealityDevice14,1"); - assert_eq!(client["osName"], "visionOS"); - assert_eq!(client["osVersion"], "25.6.0.23O471"); - assert_eq!(client["clientScreen"], "WATCH"); - // no androidSdkVersion for an Apple client - assert!(client.get("androidSdkVersion").is_none()); - // best-effort visitorData not set at envelope-build time - assert!(client.get("visitorData").is_none()); - } - #[test] fn embed_url_lands_in_third_party_block() { let info = InnertubeClientRequestInfo::of_web_embedded_player_client(); diff --git a/src/youtube/constants.rs b/src/youtube/constants.rs index cbf4d46..b5e2ac4 100644 --- a/src/youtube/constants.rs +++ b/src/youtube/constants.rs @@ -36,18 +36,6 @@ pub const ANDROID_CLIENT_VERSION: &str = "21.03.36"; pub const ANDROID_SDK_VERSION: u32 = 36; pub const ANDROID_OS_VERSION: &str = "16"; -// visionOS client (client 101). A best-effort 4th /player fetch that recovers -// >360p streams for SABR-only / "made for kids" videos where the ANDROID/IOS -// clients cap at a single 360p muxed stream. Values from NPE -// ClientsConstants.java (commit 82b7e410, "Workaround again SABR-only -// responses"). Carries NO poToken. -pub const VISIONOS_CLIENT_ID: &str = "101"; -pub const VISIONOS_CLIENT_NAME: &str = "VISIONOS"; -pub const VISIONOS_CLIENT_VERSION: &str = "1.02"; -pub const VISIONOS_DEVICE_MODEL: &str = "RealityDevice14,1"; -pub const VISIONOS_OS_VERSION: &str = "25.6.0.23O471"; -pub const VISIONOS_USER_AGENT_VERSION: &str = "25_6_0"; - // Base URLs (NPE YoutubeParsingHelper.java:91,96). pub const YOUTUBEI_V1_URL: &str = "https://www.youtube.com/youtubei/v1/"; pub const YOUTUBEI_V1_GAPIS_URL: &str = "https://youtubei.googleapis.com/youtubei/v1/"; diff --git a/src/youtube/js/extractor.rs b/src/youtube/js/extractor.rs index 27515aa..3ec0883 100644 --- a/src/youtube/js/extractor.rs +++ b/src/youtube/js/extractor.rs @@ -40,28 +40,15 @@ static SCRIPT_TAG: Lazy = /// Extracts the player.js URL + body. Tries iframe_api first, falls back /// to the embed page on any failure (matches NPE's try/catch flow). pub fn extract_javascript_player_code(video_id: &str) -> Result<(String, String), DeobfError> { - let url = extract_javascript_player_url(video_id)?; - let body = download_player_code(&url)?; - Ok((url, body)) -} - -/// Discovers the CURRENT player.js URL only (a few-KB iframe_api fetch — -/// cheap next to the ~1.7 MB body). Split out so PlayerManager's failure -/// memo can probe "did YouTube rotate player.js?" without paying for a -/// body download that would deterministically re-fail extraction. -pub fn extract_javascript_player_url(video_id: &str) -> Result { let downloader = NewPipe::downloader().ok_or(DeobfError::DownloaderMissing)?; + let url = match extract_from_iframe(&*downloader) { Ok(u) => u, Err(_iframe_err) => extract_from_embed(&*downloader, video_id)?, }; - clean_javascript_url(&url) -} - -/// Downloads the player.js body for an already-discovered URL. -pub fn download_player_code(url: &str) -> Result { - let downloader = NewPipe::downloader().ok_or(DeobfError::DownloaderMissing)?; - download_javascript_code(&*downloader, url) + let cleaned = clean_javascript_url(&url)?; + let body = download_javascript_code(&*downloader, &cleaned)?; + Ok((cleaned, body)) } fn extract_from_iframe(downloader: &dyn Downloader) -> Result { @@ -136,9 +123,7 @@ fn download_javascript_code(downloader: &dyn Downloader, url: &str) -> Result Result // NPE's `groupCount()` excludes group 0, so: // len() == 2 → 1 capture → direct name // len() == 3 → 2 captures → array indirection - // - // BUT: `caps.len()` is pattern-static — it counts DECLARED groups, - // not the ones that PARTICIPATED in the match. Regexes 6/7 declare - // the array-access group inside an optional `(?:@ARRAY@)?`, so a - // direct-call player.js shape matches with group 2 absent. Route on - // participation: group 2 unmatched → the group-1 name IS the answer. - // (NPE's Java would NPE on `Integer.parseInt(matcher.group(2))` here - // — a latent upstream bug; the optional group's evident intent, and - // yt-dlp's guarded equivalent, is the direct name.) match caps.len() { 2 => { if let Some(m) = caps.get(1) { @@ -128,14 +119,8 @@ fn deobfuscation_function_name(player_code: &str) -> Result } } 3 => { - let Some(index_m) = caps.get(2) else { - if let Some(m) = caps.get(1) { - return Ok(m.as_str().to_string()); - } - continue; - }; let array_name = caps.get(1).map(|m| m.as_str()).unwrap_or_default(); - let index_str = index_m.as_str(); + let index_str = caps.get(2).map(|m| m.as_str()).unwrap_or_default(); let index: usize = index_str.parse().map_err(|_| { DeobfError::NsigArrayLookupFailed(format!("bad index: {index_str}")) })?; @@ -200,18 +185,8 @@ fn deobfuscation_function_body_regex( /// Strips `if(typeof X==="undefined")return ;` so the function /// actually runs standalone. NPE adds this 2024-12-29 (`56595bd9d`). -/// -/// Must handle BOTH body shapes this crate produces: -/// * lexer path: `name=function(a){…};` -/// * regex-fallback path: `function name(a){…}` -/// -/// NPE's FUNCTION_ARGUMENTS_REGEX requires the `=` (its regex-fallback -/// shape would throw from matchGroup1); ours accepts both so the fallback -/// path doesn't silently no-op — a surviving guard makes the function -/// return its input unchanged, which then gets CACHED as the deobfuscated -/// n-param → permanent silent throttling. pub fn fixup_function(function: &str) -> Result { - let args_re = Regex::new(r"(?:=\s*)?function\s*[a-zA-Z0-9$_]*\s*\(\s*([^)]*)\s*\)") + let args_re = Regex::new(r"=\s*function\s*\(\s*([^)]*)\s*\)") .map_err(|e| DeobfError::NsigBodyParseFailed(e.to_string()))?; let first_arg = args_re .captures(function) @@ -228,18 +203,13 @@ pub fn fixup_function(function: &str) -> Result { // Substitute with an alternation of fully-quoted `"undefined"` / // `'undefined'` forms. Loosens slightly (allows `"undefined'`) but // real player.js always uses balanced quotes; harmless. - // - // NPE's EARLY_RETURN_REGEX anchors on a preceding `;` only. When the - // guard is the FIRST statement of the body (regex-fallback shape: - // `function name(a){if(typeof X==="undefined")return a;…}`) the - // preceding token is `{` — accept both and re-emit whichever matched. let early_return_re_src = format!( - r#"(?s)([;{{])\s*if\s*\(\s*typeof\s+[a-zA-Z0-9$_]+\s*===?\s*(?:"undefined"|'undefined')\s*\)\s*return\s+{};"#, + r#"(?s);\s*if\s*\(\s*typeof\s+[a-zA-Z0-9$_]+\s*===?\s*(?:"undefined"|'undefined')\s*\)\s*return\s+{};"#, regex::escape(&first_arg) ); let er_re = Regex::new(&early_return_re_src) .map_err(|e| DeobfError::NsigBodyParseFailed(e.to_string()))?; - Ok(er_re.replace(function, "$1").to_string()) + Ok(er_re.replace(function, ";").to_string()) } #[cfg(test)] @@ -316,63 +286,4 @@ mod tests { Err(e) => panic!("expected name match, got {e:?}"), } } - - #[test] - fn regex_6_direct_call_without_array_access() { - // Regex 6 (String.fromCharCode(110)) with a DIRECT call — the - // optional `(?:\[(\d+)])?` group does not participate. Must route - // to the group-1 name, not the array-indirection branch (which - // would hard-fail with NsigArrayLookupFailed on the empty index). - let src = r#"WL=function(a){a.j=1};(b=String.fromCharCode(110),c=a.get(b))&&(c=mfn(c),a.set(b,c))"#; - match deobfuscation_function_name(src) { - Ok(n) => assert_eq!(n, "mfn"), - Err(e) => panic!("direct-call shape must yield the direct name, got {e:?}"), - } - } - - #[test] - fn regex_7_direct_call_without_array_access() { - // Regex 7 (.get("n")) with a direct call — same non-participating - // optional group. - let src = r#"(c=d.get("n"))&&(e=Nfn(e),d.set("n",e))"#; - match deobfuscation_function_name(src) { - Ok(n) => assert_eq!(n, "Nfn"), - Err(e) => panic!("direct-call shape must yield the direct name, got {e:?}"), - } - } - - #[test] - fn fixup_strips_guard_on_function_declaration_shape() { - // The regex-fallback body shape: `function name(a){…}` — no `=` - // before `function`, and the guard sits right after `{` with no - // preceding `;`. The old fixup no-op'd on both counts, letting the - // early return survive → nsig returns its input unchanged → the - // identity result gets cached → permanent silent throttling. - let body = r#"function m85(p){if(typeof RUQ==="undefined")return p;var a=p.split("");a.reverse();return a.join("");}"#; - let fixed = fixup_function(body).unwrap(); - assert!( - !fixed.contains("typeof RUQ"), - "guard must be stripped on the function-declaration shape, got: {fixed}" - ); - assert!(fixed.contains(r#"var a=p.split("");"#)); - } - - #[test] - fn fixup_function_declaration_shape_runs_non_identity() { - // End-to-end: the fixed fallback-shape body must actually transform - // its input when run (i.e. the guard is gone, not just renamed). - let body = r#"function m85(p){if(typeof RUQ==="undefined")return p;var a=p.split("");a.reverse();return a.join("");}"#; - let fixed = fixup_function(body).unwrap(); - let out = crate::youtube::js::runtime::run(&fixed, "m85", "abc123").unwrap(); - assert_eq!(out, "321cba"); - } - - #[test] - fn fixup_lexer_shape_guard_at_start_of_body() { - // Lexer shape (`name=function(...)`) whose guard is the FIRST - // statement — preceded by `{`, not `;`. Also must strip. - let body = r#"m85=function(p){if(typeof RUQ==="undefined")return p;var a=p.split("");return a.join("");}"#; - let fixed = fixup_function(body).unwrap(); - assert!(!fixed.contains("typeof RUQ")); - } } diff --git a/src/youtube/js/player_manager.rs b/src/youtube/js/player_manager.rs index 9dc65e9..9c233dd 100644 --- a/src/youtube/js/player_manager.rs +++ b/src/youtube/js/player_manager.rs @@ -3,60 +3,27 @@ // sole public class in the JS subsystem). // // Cache layout: -// * cached_player_code — the raw ~1.5 MB player.js. Retained only -// until the hot-path snippets (signature timestamp + nsig) are built, -// then DROPPED to reclaim native heap (S7); `player_url` remains as the -// installed-generation sentinel. Re-fetched on demand if a later, -// un-built artifact (the dead-in-android sig path) ever needs the bytes. +// * cached_player_code — process-lifetime, until invalidate() // * cached_signature_timestamp // * cached_sig_snippet — assembled JS, ready for runtime::run // * cached_nsig_name + snippet // * cached_throttling_params — obfuscated → deobfuscated cache (per-session) // -// Failure handling — three cases, each tuned so a single player.js rotation -// self-heals while a persistently-unparseable player.js can't trigger a -// per-format refetch storm (~20-25 × ~1.7 MB per video open — the H2 finding -// of the 2026-07-28 audit; the previous invalidate-on-any-error design had no -// memory of "I just refetched and it STILL failed"): +// Failure handling: on ANY deobf/extraction error, invalidate() drops the +// fetched player.js and everything derived from it, so the NEXT call re-fetches +// a fresh player.js and retries. YouTube rotates player.js every few hours, so +// a failure is far more often "our cached player.js went stale" than "this +// player.js is unparseable" — invalidating lets the next attempt self-heal. +// (This replaces the previous sticky-error flags, which were set on failure and +// never cleared by any code path, permanently wedging ALL extraction for the +// process lifetime after a single routine rotation. straw audit, 2026-07-04.) // -// * Artifact extraction fails (regex bank miss / body parse): arm a -// per-artifact failure memo keyed by the player_url that failed — -// mirrors NPE's cached ParsingException fields (sigTimestampExtractionEx -// et al.), except NPE's are sticky until clearAllCaches() while ours -// expire after FAILURE_COOLDOWN. The installed player.js is KEPT so -// sibling artifacts (e.g. a working signature timestamp) stay usable. -// While the memo is live the original error is replayed with zero -// network. After the cooldown, discovery re-runs (a few-KB iframe_api -// probe): a rotated player.js is fetched fresh and retried; an -// unchanged one just re-arms the memo — extraction is deterministic, -// so re-downloading ~1.7 MB to re-fail is pure waste. -// -// * Eval (QuickJS run) fails: invalidate() everything so the next call -// re-fetches — the stale-rotation self-heal (a cached player.js whose -// function chokes on a new-generation n-param is fixed by fetching the -// current player.js). If the snippet came from a player.js downloaded -// by THIS call, a refetch would rebuild the identical snippet, so the -// memo is armed instead — bounding the storm to one refetch per open. -// -// * Fetch fails (network): plain error, no memo — transient by nature. -// -// Locking: `state` (Mutex) is NEVER held across the network. -// `fetch_gate` serializes player.js downloads so concurrent extractions -// don't each re-download (~1.7 MB); waiters re-check `state` after taking -// the gate and find the code already installed. Lock order is always -// fetch_gate → state; state is released before any fetch. (The previous -// design held the state mutex across ensure_player_code's downloads, -// serializing every concurrent extraction behind up-to-30 s requests — -// the S6 finding.) -// -// NPE uses static fields and is not thread-safe; callers serialize. We give -// the same shape via the state mutex — call sites can still hammer it from +// NPE uses static fields and is not thread-safe; callers serialize. We give the +// same shape via a `Mutex` — call sites can still hammer it from // multiple threads safely. use parking_lot::Mutex; use std::collections::HashMap; -use std::sync::Arc; -use std::time::{Duration, Instant}; use crate::youtube::js::extractor; use crate::youtube::js::nsig; @@ -64,159 +31,23 @@ use crate::youtube::js::runtime; use crate::youtube::js::signature; use crate::youtube::js::DeobfError; -/// How long a failure memo suppresses re-fetch/re-parse attempts for the -/// player.js that produced it. YouTube rotates player.js every few hours, -/// so five minutes bounds the retry storm without meaningfully delaying -/// recovery once a genuinely new player.js ships (recovery is also -/// immediate whenever a different player_url is discovered). -const FAILURE_COOLDOWN: Duration = Duration::from_secs(5 * 60); - -/// One derived artifact's failure record. `player_url` pins the player.js -/// generation the failure belongs to: a different discovered URL retires -/// the memo instantly (rotation self-heal), an identical one replays -/// `error` without re-downloading (extraction is deterministic). -struct FailMemo { - player_url: String, - error: DeobfError, - at: Instant, -} - -/// The three derived artifacts, each with its own cache + failure memo — -/// mirrors NPE's three cached exception fields. Kept independent so e.g. -/// a signature-timestamp regex miss doesn't block nsig extraction that -/// would succeed on the same player.js. -#[derive(Clone, Copy)] -enum Derived { - SigTimestamp, - Sig, - Nsig, -} - -impl Derived { - fn memo<'a>(&self, state: &'a ManagerState) -> &'a Option { - match self { - Derived::SigTimestamp => &state.ts_fail, - Derived::Sig => &state.sig_fail, - Derived::Nsig => &state.nsig_fail, - } - } - - fn memo_mut<'a>(&self, state: &'a mut ManagerState) -> &'a mut Option { - match self { - Derived::SigTimestamp => &mut state.ts_fail, - Derived::Sig => &mut state.sig_fail, - Derived::Nsig => &mut state.nsig_fail, - } - } -} - -/// Verdict of a memo check, decided under the state lock. -enum MemoVerdict { - /// No live memo applies — proceed normally. - Proceed, - /// Memo cooldown expired — proceed, but treat this URL as known-bad: - /// discover first and skip the ~1.7 MB body download if it's unchanged. - Probe(String), - /// Memo is live — replay the memoized error, zero work. - Replay(DeobfError), -} - -fn memo_verdict(state: &ManagerState, memo: &Option) -> MemoVerdict { - let Some(m) = memo else { - return MemoVerdict::Proceed; - }; - let live = m.at.elapsed() < FAILURE_COOLDOWN; - // Installed-generation sentinel: `player_url` marks WHICH player.js is - // installed, set/cleared in lockstep with install/invalidate and keyed the - // same way the memos are. It is DECOUPLED from `player_code` (the raw bytes), - // which S7 drops once the hot-path snippets are built — so a byte-released - // player.js is still "installed" for memo purposes. (ensure_player_code keeps - // keying off `player_code` because IT needs the bytes to build from.) - let installed_same = state.player_url.as_deref() == Some(m.player_url.as_str()); - if installed_same { - if live { - MemoVerdict::Replay(m.error.clone()) - } else { - MemoVerdict::Probe(m.player_url.clone()) - } - } else if state.player_url.is_some() { - // A different player.js has been installed since the failure — - // retry this artifact against it. - MemoVerdict::Proceed - } else if live { - MemoVerdict::Replay(m.error.clone()) - } else { - MemoVerdict::Probe(m.player_url.clone()) - } -} - -/// S7: reclaim the raw ~1.5 MB player.js once the hot-path artifacts are built. -/// -/// The bytes exist only to build the derived snippets. The android-primary flow -/// needs exactly two — the signature timestamp and the nsig deobfuscator — and -/// both are always built per open (timestamp by `stream_info_with`, nsig by the -/// first throttled format). Once both are present, every later call hits the -/// snippet/timestamp cache and never touches the bytes again, so we drop them -/// and keep `player_url` as the installed-generation sentinel. -/// -/// Deliberately keyed on {timestamp, nsig} and NOT on the sig snippet: the sig -/// path is dead in the android-primary flow (formats carry direct URLs), so -/// requiring it would pin the bytes forever. If that dead path IS ever hit after -/// a release, `ensure_player_code` (which keys off `player_code`) simply -/// re-fetches — the audit-sanctioned worst case. Called ONLY on a clean build -/// success; a build FAILURE keeps the bytes so sibling artifacts can still -/// extract from them. -fn release_player_code_if_built(state: &mut ManagerState) { - if state.signature_timestamp.is_some() && state.nsig_snippet.is_some() { - state.player_code = None; - } -} - -/// Outcome of ensure_player_code. -enum Ensured { - /// player_code is installed; `fresh` = this call downloaded it. - Ready { fresh: bool }, - /// The probe found YouTube still serving the caller's known-bad URL — - /// no body was downloaded. - StillBad, -} - #[derive(Default)] struct ManagerState { - // `player_url` is the installed-GENERATION sentinel — Some iff a player.js - // has been discovered+installed and its derived snippets are the current - // generation. `player_code` is the raw ~1.5 MB bytes, dropped by - // `release_player_code_if_built` once the hot-path snippets exist (S7), so - // it can be None while `player_url` is Some. memo/rotation logic keys off - // `player_url`; only `ensure_player_code` (which needs to build) keys off - // `player_code`, re-fetching when the bytes are gone. player_url: Option, player_code: Option, signature_timestamp: Option, - // Snippets held as `Arc` so the per-eval hand-off out of the state - // lock is a pointer bump instead of a ~30-120 KB `String` copy (S10). Only - // the cloned payload type changes — the lock/memo/fetch_gate discipline is - // exactly as Loop-1 left it. - sig_snippet: Option>, - nsig_name: Option>, - nsig_snippet: Option>, + sig_snippet: Option, + nsig_name: Option, + nsig_snippet: Option, throttling_param_cache: HashMap, - - // Failure memos — deliberately NOT cleared by invalidate(): they must - // survive the invalidate that accompanies the failure they record. - // Retired by: a different player_url being installed, their artifact - // building successfully, clear_all_caches(), or cooldown expiry. - ts_fail: Option, - sig_fail: Option, - nsig_fail: Option, } impl ManagerState { /// Drop the fetched player.js and everything derived from it, so the next - /// call re-fetches a fresh player.js and re-extracts. Called on eval - /// failures (stale-player.js self-heal) and before installing a newly - /// downloaded player.js. Failure memos are kept — see the field comment. + /// call re-fetches a fresh player.js and re-extracts. Called on any + /// deobf/extraction failure so a stale-player.js error self-heals on retry + /// instead of wedging extraction for the process lifetime. fn invalidate(&mut self) { self.player_url = None; self.player_code = None; @@ -231,17 +62,13 @@ impl ManagerState { } pub struct PlayerManager { - state: Mutex, - /// Serializes player.js downloads WITHOUT blocking `state` readers. - /// Held only while discovering/downloading; never while `state` is held. - fetch_gate: Mutex<()>, + inner: Mutex, } impl PlayerManager { pub fn new() -> Self { Self { - state: Mutex::new(ManagerState::default()), - fetch_gate: Mutex::new(()), + inner: Mutex::new(ManagerState::default()), } } @@ -252,14 +79,23 @@ impl PlayerManager { } pub fn signature_timestamp(&self, video_id: &str) -> Result { - self.derived_artifact( - video_id, - Derived::SigTimestamp, - |state| state.signature_timestamp, - signature::signature_timestamp, - |state, ts| state.signature_timestamp = Some(*ts), - ) - .map(|(ts, _)| ts) + let r = self.signature_timestamp_inner(video_id); + if r.is_err() { + self.inner.lock().invalidate(); + } + r + } + + fn signature_timestamp_inner(&self, video_id: &str) -> Result { + let mut state = self.inner.lock(); + if let Some(ts) = state.signature_timestamp { + return Ok(ts); + } + Self::ensure_player_code(&mut state, video_id)?; + let code = state.player_code.as_deref().unwrap(); + let ts = signature::signature_timestamp(code)?; + state.signature_timestamp = Some(ts); + Ok(ts) } pub fn deobfuscate_signature( @@ -267,37 +103,56 @@ impl PlayerManager { video_id: &str, obfuscated_signature: &str, ) -> Result { - let (snippet, fresh) = self.derived_artifact( - video_id, - Derived::Sig, - |state| state.sig_snippet.clone(), - |code| signature::build_deobfuscator(code).map(Arc::::from), - |state, snippet| state.sig_snippet = Some(snippet.clone()), - )?; + let r = self.deobfuscate_signature_inner(video_id, obfuscated_signature); + if r.is_err() { + self.inner.lock().invalidate(); + } + r + } - match runtime::run( + fn deobfuscate_signature_inner( + &self, + video_id: &str, + obfuscated_signature: &str, + ) -> Result { + let snippet = { + let mut state = self.inner.lock(); + if state.sig_snippet.is_none() { + Self::ensure_player_code(&mut state, video_id)?; + let code = state.player_code.as_deref().unwrap(); + let s = signature::build_deobfuscator(code)?; + state.sig_snippet = Some(s); + } + state.sig_snippet.clone().unwrap() + }; + + let result = runtime::run( &snippet, signature::DEOBFUSCATION_FUNCTION_NAME, obfuscated_signature, - ) { - Ok(result) => { - if result == "null" { - Ok(String::new()) // NPE: Objects.requireNonNullElse(..., "") - } else { - Ok(result) - } - } - Err(e) => { - self.on_eval_failure(Derived::Sig, fresh, &e); - Err(e) - } + )?; + if result == "null" { + return Ok(String::new()); // NPE: Objects.requireNonNullElse(..., "") } + Ok(result) } pub fn url_with_throttling_parameter_deobfuscated( &self, video_id: &str, streaming_url: &str, + ) -> Result { + let r = self.url_with_throttling_parameter_deobfuscated_inner(video_id, streaming_url); + if r.is_err() { + self.inner.lock().invalidate(); + } + r + } + + fn url_with_throttling_parameter_deobfuscated_inner( + &self, + video_id: &str, + streaming_url: &str, ) -> Result { let obf = match nsig::throttling_parameter_from_url(streaming_url) { Some(s) => s, @@ -305,44 +160,34 @@ impl PlayerManager { }; { - let state = self.state.lock(); + let state = self.inner.lock(); if let Some(cached) = state.throttling_param_cache.get(&obf) { return Ok(streaming_url.replace(&obf, cached)); } } - let ((name, snippet), fresh) = self.derived_artifact( - video_id, - Derived::Nsig, - |state| match (&state.nsig_name, &state.nsig_snippet) { - (Some(n), Some(s)) => Some((n.clone(), s.clone())), - _ => None, - }, - |code| { - nsig::build_deobfuscator(code) - .map(|(n, s)| (Arc::::from(n), Arc::::from(s))) - }, - |state, (n, s)| { - state.nsig_name = Some(n.clone()); - state.nsig_snippet = Some(s.clone()); - }, - )?; - - let deobf = match runtime::run(&snippet, &name, &obf) { - Ok(d) => d, - Err(e) => { - self.on_eval_failure(Derived::Nsig, fresh, &e); - return Err(e); + let (name, snippet) = { + let mut state = self.inner.lock(); + if state.nsig_snippet.is_none() { + Self::ensure_player_code(&mut state, video_id)?; + let code = state.player_code.as_deref().unwrap(); + let (n, s) = nsig::build_deobfuscator(code)?; + state.nsig_name = Some(n); + state.nsig_snippet = Some(s); } + ( + state.nsig_name.clone().unwrap(), + state.nsig_snippet.clone().unwrap(), + ) }; + + let deobf = runtime::run(&snippet, &name, &obf)?; if deobf.is_empty() { - let e = DeobfError::NsigEmpty; - self.on_eval_failure(Derived::Nsig, fresh, &e); - return Err(e); + return Err(DeobfError::NsigEmpty); } { - let mut state = self.state.lock(); + let mut state = self.inner.lock(); state .throttling_param_cache .insert(obf.clone(), deobf.clone()); @@ -352,240 +197,29 @@ impl PlayerManager { } pub fn throttling_parameter_cache_size(&self) -> usize { - self.state.lock().throttling_param_cache.len() + self.inner.lock().throttling_param_cache.len() } pub fn clear_all_caches(&self) { - let mut state = self.state.lock(); - state.invalidate(); - state.ts_fail = None; - state.sig_fail = None; - state.nsig_fail = None; + self.inner.lock().invalidate(); } pub fn clear_throttling_parameters_cache(&self) { - self.state.lock().throttling_param_cache.clear(); + self.inner.lock().throttling_param_cache.clear(); } pub fn player_url(&self) -> Option { - self.state.lock().player_url.clone() + self.inner.lock().player_url.clone() } - /// Shared spine for the three derived artifacts: cache fast path → - /// failure-memo verdict → make player code available (fetching outside - /// the state lock) → build + store under the lock, arming the memo on - /// extraction failure. Returns the artifact plus whether the player.js - /// it was built from was downloaded by THIS call (`fresh` — feeds the - /// eval-failure policy). - fn derived_artifact( - &self, - video_id: &str, - which: Derived, - cached: impl Fn(&ManagerState) -> Option, - build: impl Fn(&str) -> Result, - store: impl Fn(&mut ManagerState, &T), - ) -> Result<(T, bool), DeobfError> { - // Two passes: the second absorbs the narrow races (concurrent - // invalidate between ensure and the build lock; memo retired by - // clear_all_caches between verdict and probe). - for _ in 0..2 { - let known_bad: Option = { - let state = self.state.lock(); - if let Some(v) = cached(&state) { - return Ok((v, false)); - } - match memo_verdict(&state, which.memo(&state)) { - MemoVerdict::Replay(e) => return Err(e), - MemoVerdict::Probe(url) => Some(url), - MemoVerdict::Proceed => None, - } - }; - - let ensured = match self.ensure_player_code(video_id, known_bad.as_deref()) { - Ok(e) => e, - Err(fetch_err) => { - if known_bad.is_some() { - // The post-cooldown probe itself failed (network). - // Re-arm the cooldown so we don't probe on every - // subsequent call while the network is down. - let mut state = self.state.lock(); - if let Some(m) = which.memo_mut(&mut state).as_mut() { - m.at = Instant::now(); - } - } - return Err(fetch_err); - } - }; - let fresh = match ensured { - Ensured::Ready { fresh } => fresh, - Ensured::StillBad => { - // YouTube still serves the player.js this artifact - // failed on — replay the original error and restart - // the cooldown. - let mut state = self.state.lock(); - match which.memo_mut(&mut state).as_mut() { - Some(m) => { - m.at = Instant::now(); - return Err(m.error.clone()); - } - // Memo retired concurrently (clear_all_caches) — - // loop and take the fresh-slate path. - None => continue, - } - } - }; - - let mut state = self.state.lock(); - let built = match state.player_code.as_deref() { - Some(code) => build(code), - // Invalidated between ensure and here — loop to re-fetch. - None => continue, - }; - return match built { - Ok(v) => { - store(&mut state, &v); - // A successful build proves any lingering memo for this - // artifact records a superseded player.js. - *which.memo_mut(&mut state) = None; - // S7: once the hot-path snippets exist, the raw player.js - // bytes are dead weight — release them (keeping player_url). - release_player_code_if_built(&mut state); - Ok((v, fresh)) - } - Err(e) => { - // Extraction failed. Keep the installed player.js — - // sibling artifacts may extract fine from it — and arm - // the memo so every further call within the cooldown - // replays instead of re-fetching ~1.7 MB per format. - let player_url = state.player_url.clone().unwrap_or_default(); - *which.memo_mut(&mut state) = Some(FailMemo { - player_url, - error: e.clone(), - at: Instant::now(), - }); - Err(e) - } - }; - } - Err(DeobfError::FetchPlayerCode( - "player.js unavailable after retry (concurrent cache invalidation)".into(), - )) - } - - /// Eval (QuickJS) failures: invalidate so the next call re-fetches — - /// the stale-rotation self-heal (a cached player.js whose function - /// chokes on a new-generation n-param is fixed by fetching the current - /// player.js). When the failing snippet came from a player.js this very - /// call downloaded (`fresh`), a refetch would rebuild the identical - /// snippet — arm the memo instead so the remaining ~20-25 formats of - /// the open replay the error instead of each re-downloading. - fn on_eval_failure(&self, which: Derived, fresh: bool, error: &DeobfError) { - let mut state = self.state.lock(); - let player_url = state.player_url.clone().unwrap_or_default(); - state.invalidate(); - if fresh { - *which.memo_mut(&mut state) = Some(FailMemo { - player_url, - error: error.clone(), - at: Instant::now(), - }); - } - } - - /// Makes `state.player_code` available. The network work happens with - /// only `fetch_gate` held — never the state lock — so cache-hit readers - /// stay unblocked and concurrent fetchers collapse into one download. - /// - /// `known_bad_url`: the caller's memoized failing player URL. Discovery - /// still runs, but if YouTube is serving that exact URL the ~1.7 MB body - /// download is skipped (`StillBad`) — extraction is deterministic, so it - /// could only re-fail. - fn ensure_player_code( - &self, - video_id: &str, - known_bad_url: Option<&str>, - ) -> Result { - // Fast path — usable code already installed. - { - let state = self.state.lock(); - if state.player_code.is_some() - && (known_bad_url.is_none() || state.player_url.as_deref() != known_bad_url) - { - return Ok(Ensured::Ready { fresh: false }); - } - } - - // One fetcher at a time. Waiters block HERE (not on `state`), then - // usually find the winner's code installed. - let _gate = self.fetch_gate.lock(); - { - let state = self.state.lock(); - if state.player_code.is_some() - && (known_bad_url.is_none() || state.player_url.as_deref() != known_bad_url) - { - return Ok(Ensured::Ready { fresh: false }); - } - } - - // Discovery (iframe_api / embed fallback) — a few KB. - let url = extractor::extract_javascript_player_url(video_id)?; - if Some(url.as_str()) == known_bad_url { - return Ok(Ensured::StillBad); - } - // Body download — ~1.7 MB. - let code = extractor::download_player_code(&url)?; - - let mut state = self.state.lock(); - let state: &mut ManagerState = &mut state; - // Everything derived from any previously installed player.js dies - // with it; memos recorded against OTHER player URLs are retired so - // a genuinely new player.js gets a clean slate for every artifact. - state.invalidate(); - for memo in [ - &mut state.ts_fail, - &mut state.sig_fail, - &mut state.nsig_fail, - ] { - if memo.as_ref().is_some_and(|m| m.player_url != url) { - *memo = None; - } + fn ensure_player_code(state: &mut ManagerState, video_id: &str) -> Result<(), DeobfError> { + if state.player_code.is_some() { + return Ok(()); } + let (url, code) = extractor::extract_javascript_player_code(video_id)?; state.player_url = Some(url); state.player_code = Some(code); - Ok(Ensured::Ready { fresh: true }) - } - - /// Test hook: age every armed memo past FAILURE_COOLDOWN so the probe - /// path can be exercised without a five-minute sleep. - #[cfg(test)] - fn test_expire_memos(&self) { - let mut state = self.state.lock(); - let state: &mut ManagerState = &mut state; - for memo in [ - &mut state.ts_fail, - &mut state.sig_fail, - &mut state.nsig_fail, - ] { - if let Some(m) = memo.as_mut() { - if let Some(aged) = Instant::now().checked_sub(FAILURE_COOLDOWN) { - m.at = aged; - } - } - } - } - - /// Test hook: is the state mutex currently free? Used to prove the - /// player.js fetch runs without holding it. - #[cfg(test)] - fn state_lock_is_free(&self) -> bool { - self.state.try_lock().is_some() - } - - /// Test hook: are the raw player.js bytes still retained? (S7 drops them - /// once the hot-path snippets are built, keeping `player_url`.) - #[cfg(test)] - fn player_code_is_retained(&self) -> bool { - self.state.lock().player_code.is_some() + Ok(()) } } @@ -598,13 +232,6 @@ impl Default for PlayerManager { #[cfg(test)] mod tests { use super::*; - use crate::downloader::request::Request; - use crate::downloader::response::Response; - use crate::downloader::Downloader; - use crate::exceptions::NetworkError; - use crate::newpipe::NewPipe; - use std::sync::atomic::{AtomicUsize, Ordering}; - use std::sync::{mpsc, Arc}; #[test] fn assembled_sig_snippet_runs_against_synthetic_player() { @@ -637,359 +264,4 @@ mod tests { mgr.clear_all_caches(); assert!(mgr.player_url().is_none()); } - - // ---- failure-memo / refetch-storm tests ------------------------------- - // - // These register a stub Downloader in the process-global NewPipe - // singleton, so they serialize on GLOBAL_DOWNLOADER_LOCK (no other lib - // test touches the downloader). Each uses its own PlayerManager - // instance, so manager state never crosses tests. - - // Shared with the stream_helper test module so downloader-touching tests - // across both modules serialize (they all mutate the same NewPipe global). - use crate::newpipe::DOWNLOADER_TEST_LOCK as GLOBAL_DOWNLOADER_LOCK; - - /// Both channel endpoints wrapped in Mutex for the Downloader Sync bound. - type FetchBlocker = (Mutex>, Mutex>); - - /// Serves a synthetic iframe_api (announcing `hash`) and a synthetic - /// player.js body; counts every request. - struct StubDownloader { - hash: Mutex, - player_js: Mutex, - requests: AtomicUsize, - /// When set, the player.js body request signals `entered` then - /// blocks until `release` fires — used by the lock-freedom test. - block_player_fetch: Option, - } - - impl StubDownloader { - fn new(hash: &str, player_js: &str) -> Self { - Self { - hash: Mutex::new(hash.to_string()), - player_js: Mutex::new(player_js.to_string()), - requests: AtomicUsize::new(0), - block_player_fetch: None, - } - } - - fn rotate(&self, hash: &str, player_js: &str) { - *self.hash.lock() = hash.to_string(); - *self.player_js.lock() = player_js.to_string(); - } - - fn count(&self) -> usize { - self.requests.load(Ordering::SeqCst) - } - } - - impl Downloader for StubDownloader { - fn execute(&self, request: Request) -> Result { - self.requests.fetch_add(1, Ordering::SeqCst); - let url = request.url().to_string(); - if url.contains("iframe_api") { - let h = self.hash.lock().clone(); - // Shape matched by IFRAME_RES_JS_BASE_PLAYER_HASH (escaped - // `\/` form, 8-char [a-z0-9] hash). - let body = format!( - r#"src:"https://www.youtube.com/s/player\/{h}\/player_ias.vflset/en_US/www-embed.js""# - ); - Ok(Response::new(200, "OK", Default::default(), body, url)) - } else if url.contains("/s/player/") { - if let Some((entered, release)) = &self.block_player_fetch { - let _ = entered.lock().send(()); - let _ = release - .lock() - .recv_timeout(std::time::Duration::from_secs(10)); - } - let body = self.player_js.lock().clone(); - Ok(Response::new(200, "OK", Default::default(), body, url)) - } else { - Err(NetworkError::Transport("stub: unexpected request".into())) - } - } - } - - // Parseable by ts + NOTHING nsig-shaped → nsig build fails while the - // signature timestamp extracts fine. - const PLAYER_NSIG_BROKEN: &str = r#"var foo={signatureTimestamp:19999};var unrelated=1;"#; - - // nsig name via regex 0 (m85 … the greedy `.*` reaches dummy's - // `return Q[1]` on the same line); body via lexer; self-contained - // (no outer-global references), so eval succeeds for any input. - const PLAYER_GOOD: &str = r#"m85=function(p){var a=p.split("");a.reverse();return a.join("");};dummy=function(q){return Q[1];};var foo={signatureTimestamp:20244};"#; - - // Parses fine but throws for EVERY input at eval time. - const PLAYER_EVAL_THROWS: &str = r#"m85=function(p){throw Error("boom");return Q[1];};"#; - - // Parses fine; throws only for the input "evil". - const PLAYER_EVAL_SELECTIVE: &str = r#"m85=function(p){if(p==="evil")throw Error("bad");var a=p.split("");a.reverse();return a.join("");};dummy=function(q){return Q[1];};"#; - - fn install(stub: &Arc) { - NewPipe::init(stub.clone() as Arc); - } - - #[test] - fn nsig_build_failure_memoizes_instead_of_refetching_per_format() { - let _g = GLOBAL_DOWNLOADER_LOCK.lock(); - let stub = Arc::new(StubDownloader::new("aaaaaaaa", PLAYER_NSIG_BROKEN)); - install(&stub); - let mgr = PlayerManager::new(); - - // First format: discovery + body download (2 requests), build fails. - let err = mgr - .url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=AAA") - .unwrap_err(); - assert!(matches!(err, DeobfError::NsigFuncNotFound), "got {err:?}"); - assert_eq!(stub.count(), 2); - - // Remaining formats of the same open: replayed error, ZERO network. - for _ in 0..5 { - let err = mgr - .url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=BBB") - .unwrap_err(); - assert!(matches!(err, DeobfError::NsigFuncNotFound)); - } - assert_eq!(stub.count(), 2, "memo must suppress refetching"); - } - - #[test] - fn artifact_memos_are_independent() { - let _g = GLOBAL_DOWNLOADER_LOCK.lock(); - let stub = Arc::new(StubDownloader::new("aaaaaaaa", PLAYER_NSIG_BROKEN)); - install(&stub); - let mgr = PlayerManager::new(); - - // nsig fails and memoizes… - assert!(mgr - .url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=AAA") - .is_err()); - assert_eq!(stub.count(), 2); - // …but the player.js stays installed, so the signature timestamp - // extracts from it without any further network. - assert_eq!(mgr.signature_timestamp("vid").unwrap(), 19999); - assert_eq!(stub.count(), 2); - } - - #[test] - fn rotation_after_cooldown_fetches_fresh_and_recovers() { - let _g = GLOBAL_DOWNLOADER_LOCK.lock(); - let stub = Arc::new(StubDownloader::new("aaaaaaaa", PLAYER_NSIG_BROKEN)); - install(&stub); - let mgr = PlayerManager::new(); - - assert!(mgr - .url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=AAA") - .is_err()); - assert_eq!(stub.count(), 2); - - // YouTube rotates to a player.js our bank parses. - stub.rotate("bbbbbbbb", PLAYER_GOOD); - - // Within the cooldown the memo still replays — no network. - assert!(mgr - .url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=AAA") - .is_err()); - assert_eq!(stub.count(), 2); - - // After the cooldown: probe discovers the NEW url → full fetch → - // extraction + eval succeed. - mgr.test_expire_memos(); - let out = mgr - .url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=abc") - .unwrap(); - assert_eq!(out, "https://x/?n=cba"); - assert_eq!(stub.count(), 4, "probe (iframe) + fresh body download"); - } - - #[test] - fn post_cooldown_probe_on_unchanged_player_skips_body_download() { - let _g = GLOBAL_DOWNLOADER_LOCK.lock(); - let stub = Arc::new(StubDownloader::new("aaaaaaaa", PLAYER_NSIG_BROKEN)); - install(&stub); - let mgr = PlayerManager::new(); - - assert!(mgr - .url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=AAA") - .is_err()); - assert_eq!(stub.count(), 2); - - mgr.test_expire_memos(); - // Probe: one iframe_api discovery, SAME hash → no body download, - // memo re-armed. - assert!(mgr - .url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=AAA") - .is_err()); - assert_eq!(stub.count(), 3); - - // Re-armed cooldown suppresses even the probe. - assert!(mgr - .url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=AAA") - .is_err()); - assert_eq!(stub.count(), 3); - } - - #[test] - fn transient_eval_failure_still_self_heals_via_refetch() { - let _g = GLOBAL_DOWNLOADER_LOCK.lock(); - let stub = Arc::new(StubDownloader::new("aaaaaaaa", PLAYER_EVAL_SELECTIVE)); - install(&stub); - let mgr = PlayerManager::new(); - - // Working nsig on the first video. - let out = mgr - .url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=good1") - .unwrap(); - assert_eq!(out, "https://x/?n=1doog"); - assert_eq!(stub.count(), 2); - - // A new-generation n-param chokes the CACHED (not fresh) snippet → - // invalidate, NO memo. No network spent on the failing call itself. - assert!(mgr - .url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=evil") - .is_err()); - assert_eq!(stub.count(), 2); - - // YouTube has rotated; the very next call re-fetches and succeeds — - // the transient-rotation self-heal must NOT be suppressed. - stub.rotate("bbbbbbbb", PLAYER_GOOD); - let out = mgr - .url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=evil") - .unwrap(); - assert_eq!(out, "https://x/?n=live"); - assert_eq!(stub.count(), 4); - } - - #[test] - fn persistent_eval_failure_on_fresh_player_memoizes() { - let _g = GLOBAL_DOWNLOADER_LOCK.lock(); - let stub = Arc::new(StubDownloader::new("aaaaaaaa", PLAYER_EVAL_THROWS)); - install(&stub); - let mgr = PlayerManager::new(); - - // Fresh download + successful build, then eval throws → memo. - let err = mgr - .url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=AAA") - .unwrap_err(); - assert!(matches!(err, DeobfError::JsRuntimeFailed(_)), "got {err:?}"); - assert_eq!(stub.count(), 2); - - // Every further format replays without network. - for _ in 0..5 { - assert!(mgr - .url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=BBB") - .is_err()); - } - assert_eq!(stub.count(), 2); - } - - #[test] - fn state_lock_stays_free_during_player_fetch() { - let _g = GLOBAL_DOWNLOADER_LOCK.lock(); - let (entered_tx, entered_rx) = mpsc::channel(); - let (release_tx, release_rx) = mpsc::channel(); - let mut stub = StubDownloader::new("aaaaaaaa", PLAYER_GOOD); - stub.block_player_fetch = Some((Mutex::new(entered_tx), Mutex::new(release_rx))); - let stub = Arc::new(stub); - install(&stub); - - let mgr: &'static PlayerManager = Box::leak(Box::new(PlayerManager::new())); - let worker = std::thread::spawn(move || { - mgr.url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=abc") - }); - - // Wait until the worker is INSIDE the player.js body download. - entered_rx - .recv_timeout(std::time::Duration::from_secs(10)) - .expect("worker never reached the player.js fetch"); - - // The whole point of the fetch_gate: readers aren't blocked while - // ~1.7 MB downloads. The old design held this mutex across the - // network and serialized everything behind it. - assert!( - mgr.state_lock_is_free(), - "state mutex must not be held across the player.js download" - ); - assert_eq!(mgr.throttling_parameter_cache_size(), 0); // must not block - - release_tx.send(()).unwrap(); - let out = worker.join().unwrap().unwrap(); - assert_eq!(out, "https://x/?n=cba"); - } - - // ---- S7 player_code byte-release lifecycle ---------------------------- - - #[test] - fn player_code_released_after_hot_artifacts_built_but_stays_installed() { - let _g = GLOBAL_DOWNLOADER_LOCK.lock(); - let stub = Arc::new(StubDownloader::new("aaaaaaaa", PLAYER_GOOD)); - install(&stub); - let mgr = PlayerManager::new(); - - // Timestamp built first (mirrors stream_info_with's worker). Only ONE of - // the two hot artifacts exists → bytes MUST still be retained so nsig can - // build without a re-download. - assert_eq!(mgr.signature_timestamp("vid").unwrap(), 20244); - assert_eq!(stub.count(), 2, "discovery + body download"); - assert!( - mgr.player_code_is_retained(), - "bytes kept until BOTH hot snippets exist" - ); - - // nsig built from the SAME retained bytes (no new network) → now both - // hot artifacts exist → bytes released, but the generation stays - // installed (player_url intact). - let out = mgr - .url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=abc") - .unwrap(); - assert_eq!(out, "https://x/?n=cba"); - assert_eq!(stub.count(), 2, "nsig built from retained bytes, no refetch"); - assert!( - !mgr.player_code_is_retained(), - "bytes dropped once timestamp + nsig are built" - ); - assert!( - mgr.player_url().is_some(), - "installed-generation sentinel survives the byte release" - ); - - // Warm re-request of BOTH hot artifacts after the release: pure cache - // hits, zero further network — the release is invisible to the hot path. - assert_eq!(mgr.signature_timestamp("vid").unwrap(), 20244); - let out = mgr - .url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=xyz") - .unwrap(); - assert_eq!(out, "https://x/?n=zyx"); - assert_eq!(stub.count(), 2, "warm path never re-downloads after release"); - assert!(!mgr.player_code_is_retained()); - } - - #[test] - fn build_failure_keeps_player_code_for_sibling_extraction() { - let _g = GLOBAL_DOWNLOADER_LOCK.lock(); - // nsig build fails on this player.js; the signature timestamp extracts. - let stub = Arc::new(StubDownloader::new("aaaaaaaa", PLAYER_NSIG_BROKEN)); - install(&stub); - let mgr = PlayerManager::new(); - - // nsig build FAILS → memo armed, bytes must be KEPT (never released on a - // failure) so the sibling timestamp artifact can still extract. - assert!(mgr - .url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=AAA") - .is_err()); - assert_eq!(stub.count(), 2); - assert!( - mgr.player_code_is_retained(), - "a build failure must NOT drop the bytes" - ); - - // Sibling timestamp extracts from the retained bytes — no re-download — - // and because nsig never built, the release condition stays unmet. - assert_eq!(mgr.signature_timestamp("vid").unwrap(), 19999); - assert_eq!(stub.count(), 2, "sibling built from retained bytes"); - assert!( - mgr.player_code_is_retained(), - "release requires the nsig snippet, which never built here" - ); - } } diff --git a/src/youtube/linkhandler/channel.rs b/src/youtube/linkhandler/channel.rs index 33fde0e..d294655 100644 --- a/src/youtube/linkhandler/channel.rs +++ b/src/youtube/linkhandler/channel.rs @@ -10,7 +10,7 @@ use url::Url; -use crate::youtube::linkhandler::{host_is_youtube, redact_url, LinkError}; +use crate::youtube::linkhandler::{host_is_youtube, LinkError}; #[derive(Clone, Debug, Eq, PartialEq)] pub enum ChannelIdentifier { @@ -25,10 +25,8 @@ pub enum ChannelIdentifier { } pub fn parse(url_str: &str) -> Result { - // Privacy: never embed the raw input in the error (see stream.rs) — - // the parse reason alone suffices. let url = Url::parse(url_str) - .map_err(|e| LinkError::InvalidUrl(e.to_string()))?; + .map_err(|e| LinkError::InvalidUrl(format!("{url_str}: {e}")))?; let host = url .host_str() .ok_or_else(|| LinkError::InvalidUrl("no host".into()))?; @@ -39,32 +37,32 @@ pub fn parse(url_str: &str) -> Result { if let Some(rest) = path.strip_prefix("/channel/") { let id = rest.split('/').next().unwrap_or(""); if id.is_empty() { - return Err(LinkError::MissingId(redact_url(&url))); + return Err(LinkError::MissingId(url_str.into())); } return Ok(ChannelIdentifier::DirectId(id.into())); } if let Some(rest) = path.strip_prefix("/c/") { let s = rest.split('/').next().unwrap_or(""); if s.is_empty() { - return Err(LinkError::MissingId(redact_url(&url))); + return Err(LinkError::MissingId(url_str.into())); } return Ok(ChannelIdentifier::Custom(s.into())); } if let Some(rest) = path.strip_prefix("/user/") { let s = rest.split('/').next().unwrap_or(""); if s.is_empty() { - return Err(LinkError::MissingId(redact_url(&url))); + return Err(LinkError::MissingId(url_str.into())); } return Ok(ChannelIdentifier::LegacyUser(s.into())); } if let Some(rest) = path.strip_prefix("/@") { let s = rest.split('/').next().unwrap_or(""); if s.is_empty() { - return Err(LinkError::MissingId(redact_url(&url))); + return Err(LinkError::MissingId(url_str.into())); } return Ok(ChannelIdentifier::Handle(s.into())); } - Err(LinkError::MissingId(redact_url(&url))) + Err(LinkError::MissingId(url_str.into())) } pub fn channel_url(channel_id: &str) -> String { diff --git a/src/youtube/linkhandler/mod.rs b/src/youtube/linkhandler/mod.rs index 542f85d..4f2f671 100644 --- a/src/youtube/linkhandler/mod.rs +++ b/src/youtube/linkhandler/mod.rs @@ -34,20 +34,6 @@ pub const ACCEPTED_HOSTS: &[&str] = &[ "www.youtube-nocookie.com", ]; -/// Renders a URL as scheme+host only, for embedding in error strings. -/// Privacy: link errors reach Kotlin exception messages / logs. Query params -/// carry user browsing data (`v=`, `list=`) AND several -/// YouTube PATHS embed the id (`/embed/`, `/shorts/`, `/clip/`, -/// `/live/`, `youtu.be/`) — so drop the path too; the host alone says -/// which endpoint failed without leaking what the user was watching. -pub(crate) fn redact_url(url: &url::Url) -> String { - let mut u = url.clone(); - u.set_query(None); - u.set_fragment(None); - u.set_path("/"); - u.to_string() -} - pub fn host_is_youtube(host: &str) -> bool { let h = host.to_ascii_lowercase(); let h = h.strip_prefix("www.").unwrap_or(&h); @@ -80,24 +66,4 @@ mod tests { assert!(!host_is_youtube("piped.video")); assert!(!host_is_youtube("evil.com")); } - - #[test] - fn redact_url_drops_id_bearing_path_query_and_fragment() { - // Every one of these carries the video/playlist id somewhere the old - // query-only strip would have kept (the path, for embed/shorts/youtu.be). - let cases = [ - "https://www.youtube.com/embed/dQw4w9WgXcQ?v=secret#frag", - "https://www.youtube.com/shorts/dQw4w9WgXcQ", - "https://youtu.be/dQw4w9WgXcQ", - "https://www.youtube.com/watch?v=dQw4w9WgXcQ&list=PLsecretlist", - ]; - for c in cases { - let u = url::Url::parse(c).unwrap(); - let r = redact_url(&u); - assert!(!r.contains("dQw4w9WgXcQ"), "leaked video id: {r}"); - assert!(!r.contains("PLsecretlist"), "leaked list id: {r}"); - // scheme + host only (no port in any of these cases). - assert_eq!(r, format!("{}://{}/", u.scheme(), u.host_str().unwrap())); - } - } } diff --git a/src/youtube/linkhandler/stream.rs b/src/youtube/linkhandler/stream.rs index dbeb732..9592138 100644 --- a/src/youtube/linkhandler/stream.rs +++ b/src/youtube/linkhandler/stream.rs @@ -15,7 +15,7 @@ use once_cell::sync::Lazy; use regex::Regex; use url::Url; -use crate::youtube::linkhandler::{host_is_youtube, redact_url, LinkError}; +use crate::youtube::linkhandler::{host_is_youtube, LinkError}; const VIDEO_ID_LEN: usize = 11; @@ -42,11 +42,8 @@ fn extract_video_id_inner(input_url: &str, depth: u8) -> Result Result String { ) } -/// visionOS user-agent — mirrors NPE `getVisionOsUserAgent`. -pub fn visionos_user_agent(country: &ContentCountry) -> String { - format!( - "com.google.visionos.youtube/{VISIONOS_CLIENT_VERSION}({VISIONOS_DEVICE_MODEL}; U; CPU visionOS {VISIONOS_USER_AGENT_VERSION} like Mac OS X; {})", - country.country_code() - ) -} - #[cfg(test)] mod tests { use super::*; @@ -142,13 +134,4 @@ mod tests { assert!(ua.contains("CPU iOS 18_7_2")); assert!(ua.contains("; US)")); } - - #[test] - fn visionos_ua_template() { - let ua = visionos_user_agent(&ContentCountry::new("US")); - assert!(ua.contains("com.google.visionos.youtube/1.02")); - assert!(ua.contains("RealityDevice14,1")); - assert!(ua.contains("CPU visionOS 25_6_0 like Mac OS X")); - assert!(ua.contains("; US)")); - } } diff --git a/src/youtube/stream_extractor.rs b/src/youtube/stream_extractor.rs index 36c4336..8837537 100644 --- a/src/youtube/stream_extractor.rs +++ b/src/youtube/stream_extractor.rs @@ -28,8 +28,7 @@ use crate::image::{Image, ResolutionLevel}; use crate::localization::{ContentCountry, Localization}; use crate::newpipe::NewPipe; use crate::stream::{ - AudioStream, AudioTrackType, DeliveryMethod, StreamInfo, StreamType, SubtitlesStream, - VideoStream, + AudioStream, DeliveryMethod, StreamInfo, StreamType, SubtitlesStream, VideoStream, }; use crate::youtube::itag::{lookup as itag_lookup, ItagType}; #[cfg(test)] @@ -37,7 +36,6 @@ use crate::youtube::itag::MediaFormat; use crate::youtube::js::PlayerManager; use crate::youtube::potoken::{po_token_provider, PoTokenResult}; use crate::youtube::stream_helper::{self, generate_content_playback_nonce}; -use crate::youtube::xtags; /// Shared `Value::Null` sentinel so a missing `streamingData` can be handed /// back as a borrowed `&Value` instead of cloning the (large) present @@ -51,23 +49,9 @@ pub enum FetchPolicy { AndroidWithPoToken, } -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Default)] pub struct ExtractOptions { pub fetch_ios_client: bool, - /// Fetch the visionOS client (client 101) as an additional best-effort - /// /player call. Recovers >360p streams for SABR-only / "made for kids" - /// videos where ANDROID/IOS cap at 360p. Its formats are merged in - /// (android-preferred on itag ties); a failure changes nothing. - /// - /// Defaults to `false` (see `impl Default`) as of the 2026-07-29 playback - /// outage fix. Upstream NPE runs the visionOS fetch UNCONDITIONALLY in - /// `onFetchPage`, but its adaptive video-only + HLS output has no working - /// playback path in the Straw app yet, so the shipped path (`stream_info` → - /// `stream_info_with(.., ExtractOptions::default())`) keeps it OFF. Set to - /// `true` per-call to opt in (e.g. once the app's adaptive path is fixed and - /// verified via the NPE differential harness). The cheap `stream_metadata` - /// path does not build streams and never fetches visionOS regardless. - pub fetch_visionos_client: bool, pub android_streaming_pot: Option, pub ios_streaming_pot: Option, pub android_visitor_data: Option, @@ -76,35 +60,6 @@ pub struct ExtractOptions { pub ios_player_request_pot: Option, } -// Explicit manual impl (not a derive) so the load-bearing `fetch_visionos_client: -// false` default is spelled out and greppable — a silent flip of exactly this -// value caused the 2026-07-29 playback outage. clippy flags it as derivable only -// because every field currently equals its type default; keep it explicit on -// purpose so re-enabling visionOS is a deliberate, visible edit here. -#[allow(clippy::derivable_impls)] -impl Default for ExtractOptions { - fn default() -> Self { - Self { - // visionOS OFF by default (2026-07-29 outage fix). visionOS unlocks - // adaptive video-only + separate-audio streams + a VOD HLS manifest, - // but the Straw app has no working adaptive/HLS playback path — it - // has only ever received the muxed 360p stream — so those streams - // don't play and video playback breaks. Upstream NPE runs visionOS - // unconditionally; we can't re-enable until the client handles its - // output (fix + verify the app's adaptive path via the NPE - // differential harness first). - fetch_visionos_client: false, - fetch_ios_client: false, - android_streaming_pot: None, - ios_streaming_pot: None, - android_visitor_data: None, - ios_visitor_data: None, - android_player_request_pot: None, - ios_player_request_pot: None, - } - } -} - /// One-shot StreamInfo build for a video. Walks NPE's Android-primary /// fetch path, applies URL post-processing, returns the final shape. pub fn stream_info(video_id: &str) -> Result { @@ -135,58 +90,19 @@ pub fn stream_info_with( ); let android_cpn = generate_content_playback_nonce(); + let player_response = fetch_android( + video_id, + &localization, + &content_country, + &android_cpn, + android_token + .as_ref() + .map(|t| t.player_request_po_token.as_str()), + android_token.as_ref().map(|t| t.visitor_data.as_str()), + )?; - // S2 (Option A — the conservative subset): overlap the player.js - // signature-timestamp resolution with the primary Android reel fetch. The - // two are fully independent — `signature_timestamp` needs only `video_id`, - // and the WEB metadata leg that consumes the timestamp still runs AFTER the - // decoy gate below. On a cold open (or after a player.js rotation) this - // overlaps the ~1.5 MB base.js download with the Android round-trip instead - // of paying them serially; warm, the timestamp is cached and the worker - // returns near-instantly. `scope` joins both before either result is read, - // so the borrowed locals stay valid (same shape as `channel.rs`). The - // worker leg is best-effort: a failed extraction OR a panicked worker yields - // 0 — identical to the prior serial `.unwrap_or(0)`. Android stays the - // gating primary (its playability/decoy check below is unchanged and still - // aborts before any WEB/visionOS work), the visitorData-first ordering is - // untouched (the player.js fetch never hits the visitor_id endpoint), and - // player.js concurrency is already guarded by PlayerManager's fetch_gate so - // a second concurrent open dedups the download rather than racing it. - // Trade-off (accepted): on a COLD cache, a *failing* open can't return its - // error until the speculative player.js leg joins (bounded by one download); - // the win is the warm/success path. Worst case is a rare, bounded cold-fail - // delay — never a change in what's extracted. - let (android_result, signature_timestamp) = std::thread::scope(|s| { - let sig_ts = s.spawn(|| { - PlayerManager::instance() - .signature_timestamp(video_id) - .unwrap_or(0) - }); - let android = fetch_android( - video_id, - &localization, - &content_country, - &android_cpn, - android_token - .as_ref() - .map(|t| t.player_request_po_token.as_str()), - android_token.as_ref().map(|t| t.visitor_data.as_str()), - ); - // A panicked worker joins as Err → 0, same as an extraction failure. - (android, sig_ts.join().unwrap_or(0)) - }); - let player_response = android_result?; - - // A primary-android rejection (playability failure or decoy) is exactly the - // symptom a poisoned visitorData would produce on the primary reel call — - // and since it aborts extraction the value would otherwise never self-heal. - // Drop the cached visitorData so the next extraction fetches a fresh one. - if let Err(e) = check_playability_status(&player_response) { - stream_helper::reset_visitor_data_cache(); - return Err(e); - } + check_playability_status(&player_response)?; if is_player_response_not_valid(&player_response, video_id) { - stream_helper::reset_visitor_data_cache(); return Err(ExtractionError::Other( "ANDROID player response is not valid (decoy detected)".into(), )); @@ -236,31 +152,6 @@ pub fn stream_info_with( .and_then(|r| r.get("streamingData")) .unwrap_or(&NULL_VALUE); - // Optional visionOS — best-effort, gated (default OFF; see - // ExtractOptions::fetch_visionos_client). Recovers >360p streams for - // SABR-only / "made for kids" videos. Carries NO poToken and (like iOS) is - // decoy-checked before its streamingData is used. A failure leaves the - // output exactly as it is today. - let (visionos_response, visionos_cpn): (Option, Option) = - if options.fetch_visionos_client { - let cpn = generate_content_playback_nonce(); - match stream_helper::get_visionos_player_response( - video_id, - &localization, - &content_country, - &cpn, - ) { - Ok(r) if !is_player_response_not_valid(&r, video_id) => (Some(r), Some(cpn)), - _ => (None, None), - } - } else { - (None, None) - }; - let visionos_streaming_data: &Value = visionos_response - .as_ref() - .and_then(|r| r.get("streamingData")) - .unwrap_or(&NULL_VALUE); - let android_streaming_pot = android_token .as_ref() .map(|t| t.streaming_data_po_token.clone()) @@ -270,9 +161,9 @@ pub fn stream_info_with( .map(|t| t.streaming_data_po_token.clone()) .or_else(|| options.ios_streaming_pot.clone()); - // `signature_timestamp` was resolved concurrently with the Android fetch - // above (S2). The WEB metadata leg is the sole consumer and runs only now, - // after the Android decoy gate has passed. + let signature_timestamp = PlayerManager::instance() + .signature_timestamp(video_id) + .unwrap_or(0); let web_metadata = fetch_web_metadata(video_id, &localization, &content_country, signature_timestamp); let mut info = StreamInfo { @@ -285,37 +176,19 @@ pub fn stream_info_with( populate_video_details(&mut info, &player_response); populate_microformat(&mut info, &web_metadata); - - // Merge order mirrors NPE getItags: ANDROID, then VISIONOS, then IOS. The - // per-(itag,delivery) dedup is first-wins, so android formats win ties and - // visionOS/iOS only ADD formats android lacked (e.g. the >360p renditions - // for SABR-only videos). visionOS carries no poToken. - let sources = [ - FormatSource { - streaming_data: android_streaming_data, - client: "ANDROID", - cpn: &android_cpn, - pot: android_streaming_pot.as_deref(), - }, - FormatSource { - streaming_data: visionos_streaming_data, - client: "VISIONOS", - cpn: visionos_cpn.as_deref().unwrap_or(""), - pot: None, - }, - FormatSource { - streaming_data: ios_streaming_data, - client: "IOS", - cpn: ios_cpn.as_deref().unwrap_or(""), - pot: ios_streaming_pot.as_deref(), - }, - ]; - - populate_streams(&mut info, &sources, video_id)?; + populate_streams( + &mut info, + android_streaming_data, + ios_streaming_data, + video_id, + &android_cpn, + ios_cpn.as_deref(), + android_streaming_pot.as_deref(), + ios_streaming_pot.as_deref(), + )?; populate_manifests( &mut info, android_streaming_data, - visionos_streaming_data, ios_streaming_data, android_streaming_pot.as_deref(), ios_streaming_pot.as_deref(), @@ -358,7 +231,7 @@ pub fn stream_metadata(video_id: &str) -> Result { }); let android_cpn = generate_content_playback_nonce(); - let player_response = fetch_android_metadata( + let player_response = fetch_android( video_id, &localization, &content_country, @@ -369,16 +242,8 @@ pub fn stream_metadata(video_id: &str) -> Result { android_token.as_ref().map(|t| t.visitor_data.as_str()), )?; - // A primary-android rejection (playability failure or decoy) is exactly the - // symptom a poisoned visitorData would produce on the primary reel call — - // and since it aborts extraction the value would otherwise never self-heal. - // Drop the cached visitorData so the next extraction fetches a fresh one. - if let Err(e) = check_playability_status(&player_response) { - stream_helper::reset_visitor_data_cache(); - return Err(e); - } + check_playability_status(&player_response)?; if is_player_response_not_valid(&player_response, video_id) { - stream_helper::reset_visitor_data_cache(); return Err(ExtractionError::Other( "ANDROID player response is not valid (decoy detected)".into(), )); @@ -416,7 +281,7 @@ fn fetch_android( po_token: Option<&str>, visitor_data: Option<&str>, ) -> Result { - if po_token.is_some() { + let result = if po_token.is_some() { stream_helper::get_android_player_response( video_id, localization, @@ -433,86 +298,9 @@ fn fetch_android( cpn, )?; // The reel endpoint returns the `playerResponse` nested one level. - Ok(unwrap_reel_player_response(r)) - } -} - -/// Metadata-scoped Android fetch for the cheap `stream_metadata` feed-enrich -/// path — do NOT use on the full extraction path, which needs `streamingData`. -/// -/// On the anonymous reel path (the default — no PoTokenProvider registered) this -/// tries the NARROWED-`$fields` reel variant, which asks the endpoint for only -/// `playabilityStatus` + `videoDetails` (~3-6 KB vs the full ~150-500 KB reel -/// body). It is BEST-EFFORT with a HARD full-fetch fallback: `reel_item_watch` -/// may not honor the nested `$fields` selector, so if the narrowed request errors -/// (e.g. a 400 rejecting the selector) OR returns a shape MISSING `videoDetails` -/// (endpoint ignored it into an error/empty response), we fall back to the full -/// `fetch_android`, which is byte-for-byte today's behavior. Net: narrowed -/// honored → bytes saved; narrowed not honored → identical to today. Feed -/// enrichment can never regress from this optimization. -/// -/// The po_token `/player` path (non-default) has no `$fields` narrowing to apply, -/// so it goes straight to the shared `fetch_android`. -fn fetch_android_metadata( - video_id: &str, - localization: &Localization, - content_country: &ContentCountry, - cpn: &str, - po_token: Option<&str>, - visitor_data: Option<&str>, -) -> Result { - // Only the anonymous reel path carries a $fields selector to narrow. - if po_token.is_none() { - if let Ok(r) = stream_helper::get_android_reel_player_response_metadata( - video_id, - localization, - content_country, - cpn, - ) { - let unwrapped = unwrap_reel_player_response(r); - // Guard: only trust the narrowed response if its videoDetails is - // bound to the REQUESTED video — i.e. `videoDetails.videoId` is - // present AND equals `video_id`. A bare `is_some()` would accept a - // present-but-gutted `videoDetails: {}` (endpoint half-honored the - // nested selector) → silently-empty title/views/duration, and would - // also wave through a videoId-stripped decoy (the downstream decoy - // check treats a missing videoId as valid). Binding to the id - // subsumes the decoy check for this path and rejects every - // not-fully-honored shape into the full-fetch fallback. - let honored = unwrapped - .get("videoDetails") - .and_then(|vd| vd.get("videoId")) - .and_then(|id| id.as_str()) - == Some(video_id); - if honored { - return Ok(unwrapped); - } - } - } - // Fallback (and the po_token path): the full, unchanged fetch. - fetch_android( - video_id, - localization, - content_country, - cpn, - po_token, - visitor_data, - ) -} - -/// Unwrap the reel endpoint's one-level `playerResponse` nesting, MOVING the -/// (~150-500 KB) subtree out instead of deep-cloning it. `r` is owned and dead -/// after this call, so `serde_json::Map::remove` returns the owned subtree with -/// zero allocation — the old `.cloned().unwrap_or(r)` re-allocated every Map / -/// Vec / String node on every video open AND every feed-enrich (S1). Behavior -/// is identical, including the defensive "no `playerResponse` key → use the -/// whole response" fallback. Only the anonymous reel path calls this; the -/// token /player path returns the response already un-nested. -fn unwrap_reel_player_response(r: Value) -> Value { - match r { - Value::Object(mut m) => m.remove("playerResponse").unwrap_or(Value::Object(m)), - other => other, - } + Ok(r.get("playerResponse").cloned().unwrap_or(r)) + }; + result } fn fetch_web_metadata( @@ -673,50 +461,66 @@ fn populate_microformat(info: &mut StreamInfo, web_metadata: &Value) { } } -/// One client's streamingData plus the tags every format from it carries. -/// Holds borrowed `&Value`s into the (function-lived) player responses. -struct FormatSource<'a> { - streaming_data: &'a Value, - client: &'static str, - cpn: &'a str, - pot: Option<&'a str>, -} - -/// Merge the per-client format objects under one streamingData array key -/// (`formats` / `adaptiveFormats`), each tagged with its client + cpn + pot, -/// in source order (ANDROID, VISIONOS, IOS). Returns borrowed `&Value`s into -/// the player responses: the format objects are only ever read (`.get(...)`) -/// downstream, so cloning the whole array (~20-40 objects per video) was pure -/// waste. +/// Merge the Android + iOS format objects under one streamingData array +/// key (`formats` / `adaptiveFormats`), each tagged with its client + cpn + +/// pot. Returns borrowed `&Value`s into the player responses: the format +/// objects are only ever read (`.get(...)`) downstream, so cloning the whole +/// array (~20-40 objects per video) was pure waste. +#[allow(clippy::too_many_arguments)] fn merge_formats<'a>( - sources: &[FormatSource<'a>], + android: &'a Value, + ios: &'a Value, fmt_array_key: &str, + android_cpn: &'a str, + ios_cpn: Option<&'a str>, + android_pot: Option<&'a str>, + ios_pot: Option<&'a str>, ) -> Vec<(&'a Value, &'static str, &'a str, Option<&'a str>)> { let mut out = Vec::new(); - for src in sources { - if let Some(arr) = src.streaming_data.get(fmt_array_key).and_then(|v| v.as_array()) { - for f in arr { - out.push((f, src.client, src.cpn, src.pot)); - } + if let Some(arr) = android.get(fmt_array_key).and_then(|v| v.as_array()) { + for f in arr { + out.push((f, "ANDROID", android_cpn, android_pot)); + } + } + if let Some(arr) = ios.get(fmt_array_key).and_then(|v| v.as_array()) { + let cpn = ios_cpn.unwrap_or(""); + for f in arr { + out.push((f, "IOS", cpn, ios_pot)); } } out } +#[allow(clippy::too_many_arguments)] fn populate_streams( info: &mut StreamInfo, - sources: &[FormatSource], + android: &Value, + ios: &Value, video_id: &str, + android_cpn: &str, + ios_cpn: Option<&str>, + android_pot: Option<&str>, + ios_pot: Option<&str>, ) -> Result<(), ExtractionError> { // Progressive: streamingData.formats[] - for (fmt, _client, cpn, pot) in merge_formats(sources, "formats") { + for (fmt, _client, cpn, pot) in + merge_formats(android, ios, "formats", android_cpn, ios_cpn, android_pot, ios_pot) + { if let Some(stream) = build_video_progressive(fmt, video_id, cpn, pot)? { push_video_dedup(&mut info.video_streams, stream); } } // Adaptive: streamingData.adaptiveFormats[] - for (fmt, _client, cpn, pot) in merge_formats(sources, "adaptiveFormats") { + for (fmt, _client, cpn, pot) in merge_formats( + android, + ios, + "adaptiveFormats", + android_cpn, + ios_cpn, + android_pot, + ios_pot, + ) { let mime = fmt .get("mimeType") .and_then(|v| v.as_str()) @@ -737,43 +541,18 @@ fn populate_streams( fn populate_manifests( info: &mut StreamInfo, android: &Value, - visionos: &Value, ios: &Value, android_pot: Option<&str>, ios_pot: Option<&str>, ) { - // NPE getManifestUrl skips null-OR-EMPTY manifest URLs; the `!is_empty` - // filters matter because an EMPTY visionOS `hlsManifestUrl` would otherwise - // shadow a real iOS/Android one (`Some("")`), and the wrapper drops - // `Some("")` → the app loses HLS entirely. // DASH is Android-only. - if let Some(url) = android - .get("dashManifestUrl") - .and_then(|v| v.as_str()) - .filter(|s| !s.is_empty()) - { + if let Some(url) = android.get("dashManifestUrl").and_then(|v| v.as_str()) { info.dash_manifest_url = Some(append_pot_to_manifest(url, android_pot)); } - // HLS prefers an Apple client — visionOS, then iOS — because their HLS - // manifests carry separated audio/video for livestreams (NPE getHlsUrl: - // visionOS → iOS → android). visionOS carries no poToken. - if let Some(url) = visionos - .get("hlsManifestUrl") - .and_then(|v| v.as_str()) - .filter(|s| !s.is_empty()) - { - info.hls_manifest_url = Some(append_pot_to_manifest(url, None)); - } else if let Some(url) = ios - .get("hlsManifestUrl") - .and_then(|v| v.as_str()) - .filter(|s| !s.is_empty()) - { + // HLS prefers iOS, falls back to Android. + if let Some(url) = ios.get("hlsManifestUrl").and_then(|v| v.as_str()) { info.hls_manifest_url = Some(append_pot_to_manifest(url, ios_pot)); - } else if let Some(url) = android - .get("hlsManifestUrl") - .and_then(|v| v.as_str()) - .filter(|s| !s.is_empty()) - { + } else if let Some(url) = android.get("hlsManifestUrl").and_then(|v| v.as_str()) { info.hls_manifest_url = Some(append_pot_to_manifest(url, android_pot)); } } @@ -861,15 +640,11 @@ fn process_url( // nsig deobf — unconditional (quick-exits internally if no `n=` present). // On failure, fall back to the throttled ORIGINAL url rather than dropping - // the format or aborting the video: YouTube still serves it (rate-limited). - // A stale-cache eval failure invalidates PlayerManager's cache, so the - // NEXT format re-fetches a fresh player.js and deobfuscates cleanly — one - // routine rotation costs at most one throttled format. If the FRESH - // player.js also fails (regex bank miss — a persistent breakage), the - // manager's failure memo replays the error for the remaining formats with - // zero network instead of re-downloading ~1.7 MB per format, and retries - // after a cooldown / as soon as a new player.js ships. (NPE parity; straw - // audits 2026-07-04 + 2026-07-28 H2.) + // the format or aborting the video: YouTube still serves it (rate-limited), + // and PlayerManager has already invalidated its cache so the NEXT format + // re-fetches a fresh player.js and deobfuscates cleanly. One routine + // player.js rotation therefore costs at most one throttled format, not the + // whole video. (NPE parity; straw audit 2026-07-04.) url = PlayerManager::instance() .url_with_throttling_parameter_deobfuscated(video_id, &url) .unwrap_or(url); @@ -980,35 +755,6 @@ fn build_audio( return Ok(None); }; let audio_track = fmt.get("audioTrack"); - let has_track = audio_track - .and_then(|t| t.get("id")) - .and_then(|v| v.as_str()) - .map(|s| !s.is_empty()) - .unwrap_or(false); - - // audioTrackType from the format's `xtags` protobuf (NPE b0bca7e7/e3479a7c). - // Only meaningful for a real alternate track (matches NPE, which sets it - // only inside `if (!isNullOrEmpty(audioTrackId))`). - let track_type = if has_track { - fmt.get("xtags") - .and_then(|v| v.as_str()) - .and_then(xtags::extract_audio_track_type) - } else { - None - }; - - // `is_descriptive`: prefer the authoritative xtags-derived type; fall back - // to the legacy `audioTrack.audioIsDefault` heuristic when xtags is - // absent/unparseable (track_type == None). - let is_descriptive = match track_type { - Some(t) => t == AudioTrackType::Descriptive, - None => audio_track - .and_then(|t| t.get("audioIsDefault")) - .and_then(|v| v.as_bool()) - .map(|b| !b) - .unwrap_or(false), - }; - Ok(Some(AudioStream { itag: itag.id, url, @@ -1037,8 +783,11 @@ fn build_audio( .and_then(|v| v.as_str()) .and_then(|s| s.split('.').next()) .map(String::from), - is_descriptive, - track_type, + is_descriptive: audio_track + .and_then(|t| t.get("audioIsDefault")) + .and_then(|v| v.as_bool()) + .map(|b| !b) + .unwrap_or(false), itag_url_format: None, })) } @@ -1137,27 +886,6 @@ mod tests { assert!(!is_player_response_not_valid(&resp, "MATCHING")); } - #[test] - fn reel_player_response_moved_out_or_falls_back() { - // Nested playerResponse is unwrapped (moved out, not cloned) — the - // returned subtree is exactly the inner object. - let r = json!({ - "playerResponse": {"videoDetails": {"videoId": "abc"}, "streamingData": {}}, - "responseContext": {"x": 1} - }); - let out = unwrap_reel_player_response(r); - assert_eq!(out, json!({"videoDetails": {"videoId": "abc"}, "streamingData": {}})); - - // No playerResponse key → the whole response is returned unchanged - // (matches the prior `.cloned().unwrap_or(r)` fallback). - let r = json!({"videoDetails": {"videoId": "xyz"}}); - assert_eq!(unwrap_reel_player_response(r.clone()), r); - - // Defensive: a non-object value passes through as-is. - let r = json!("scalar"); - assert_eq!(unwrap_reel_player_response(r.clone()), r); - } - #[test] fn cipher_string_parsed() { let s = "s=AAA%3D&sp=sig&url=https%3A%2F%2Fexample.com%2Fpath%3Fa%3D1"; @@ -1223,334 +951,4 @@ mod tests { push_video_dedup(&mut list, s2); assert_eq!(list.len(), 2); } - - // ---- I-1 visionOS merge ------------------------------------------------ - - #[test] - fn absent_visionos_source_contributes_nothing() { - // Best-effort proof: a Null visionOS streamingData (fetch failed or the - // client disabled) adds ZERO formats — the merged set is android+ios - // only, exactly today's behavior. - let android = json!({ "formats": [ {"itag": 18} ] }); - let ios = json!({ "formats": [ {"itag": 22} ] }); - let null = Value::Null; - let sources = [ - FormatSource { streaming_data: &android, client: "ANDROID", cpn: "a", pot: None }, - FormatSource { streaming_data: &null, client: "VISIONOS", cpn: "", pot: None }, - FormatSource { streaming_data: &ios, client: "IOS", cpn: "i", pot: None }, - ]; - let merged = merge_formats(&sources, "formats"); - let itags: Vec = merged - .iter() - .map(|(f, _, _, _)| f["itag"].as_u64().unwrap()) - .collect(); - assert_eq!(itags, vec![18, 22]); - assert!(merged.iter().all(|(_, client, _, _)| *client != "VISIONOS")); - } - - #[test] - fn populate_streams_merges_visionos_extra_itags_android_wins_ties() { - // android has only itag 18 (360p); visionOS returns a dup 18 plus an - // extra 22 (720p) android lacked — the SABR-only coverage recovery. - let android = json!({ "formats": [ - {"itag": 18, "url": "https://r/vp?a=1", "mimeType": "video/mp4; codecs=\"avc1.42\""} - ]}); - let visionos = json!({ "formats": [ - {"itag": 18, "url": "https://r/vp?v=1", "mimeType": "video/mp4"}, - {"itag": 22, "url": "https://r/vp?v=2", "mimeType": "video/mp4"} - ]}); - let null = Value::Null; - let sources = [ - FormatSource { streaming_data: &android, client: "ANDROID", cpn: "acpn", pot: None }, - FormatSource { streaming_data: &visionos, client: "VISIONOS", cpn: "vcpn", pot: None }, - FormatSource { streaming_data: &null, client: "IOS", cpn: "", pot: None }, - ]; - let mut info = StreamInfo::default(); - populate_streams(&mut info, &sources, "vid").unwrap(); - let mut tags: Vec = info.video_streams.iter().map(|s| s.itag).collect(); - tags.sort_unstable(); - assert_eq!(tags, vec![18, 22]); // dup 18 deduped, 22 recovered - // itag 18 kept from ANDROID (first-wins): its URL carries android's cpn. - let s18 = info.video_streams.iter().find(|s| s.itag == 18).unwrap(); - assert!(s18.url.contains("a=1"), "android's itag-18 must win: {}", s18.url); - assert!(s18.url.contains("cpn=acpn")); - } - - #[test] - fn hls_prefers_visionos_then_ios_then_android() { - let android = json!({ "hlsManifestUrl": "https://a/hls" }); - let visionos = json!({ "hlsManifestUrl": "https://v/hls" }); - let ios = json!({ "hlsManifestUrl": "https://i/hls" }); - let null = Value::Null; - - let mut info = StreamInfo::default(); - populate_manifests(&mut info, &android, &visionos, &ios, None, None); - assert_eq!(info.hls_manifest_url.as_deref(), Some("https://v/hls")); - - // visionOS absent → iOS wins. - let mut info = StreamInfo::default(); - populate_manifests(&mut info, &android, &null, &ios, None, None); - assert_eq!(info.hls_manifest_url.as_deref(), Some("https://i/hls")); - - // Only android → android (unchanged from today). - let mut info = StreamInfo::default(); - populate_manifests(&mut info, &android, &null, &null, None, None); - assert_eq!(info.hls_manifest_url.as_deref(), Some("https://a/hls")); - } - - #[test] - fn empty_hls_url_is_skipped_not_shadowing_a_real_one() { - // Fix 3: an EMPTY visionOS hlsManifestUrl must NOT shadow iOS's real one - // (else the wrapper drops Some("") → the app loses HLS). - let android = json!({ "hlsManifestUrl": "https://a/hls" }); - let visionos_empty = json!({ "hlsManifestUrl": "" }); - let ios = json!({ "hlsManifestUrl": "https://i/hls" }); - - let mut info = StreamInfo::default(); - populate_manifests(&mut info, &android, &visionos_empty, &ios, None, None); - assert_eq!(info.hls_manifest_url.as_deref(), Some("https://i/hls")); - - // Empty visionOS AND empty iOS → falls through to android. - let ios_empty = json!({ "hlsManifestUrl": "" }); - let mut info = StreamInfo::default(); - populate_manifests(&mut info, &android, &visionos_empty, &ios_empty, None, None); - assert_eq!(info.hls_manifest_url.as_deref(), Some("https://a/hls")); - - // All empty → None (not Some("")). - let android_empty = json!({ "hlsManifestUrl": "", "dashManifestUrl": "" }); - let mut info = StreamInfo::default(); - populate_manifests(&mut info, &android_empty, &visionos_empty, &ios_empty, None, None); - assert!(info.hls_manifest_url.is_none()); - assert!(info.dash_manifest_url.is_none()); // empty DASH also skipped - } - - #[test] - fn default_options_visionos_off() { - // 2026-07-29 outage fix: the shipped app path (stream_info → - // ExtractOptions::default()) must keep visionOS OFF — its adaptive/HLS - // output has no working playback path in the app yet. Re-enable only - // after the app handles it and the NPE harness verifies the shape. - assert!(!ExtractOptions::default().fetch_visionos_client); - } - - // ---- I-3 xtags audioTrackType ----------------------------------------- - - // itag 140 (M4A audio) + a direct url with no `n=` param, so process_url - // never touches the network (nsig quick-exits). - fn audio_fmt(xtags: Option<&str>, audio_is_default: bool) -> Value { - let mut fmt = json!({ - "itag": 140, - "url": "https://r.googlevideo.com/videoplayback?foo=1", - "mimeType": "audio/mp4; codecs=\"mp4a.40.2\"", - "audioTrack": { "id": "en.4", "displayName": "English", "audioIsDefault": audio_is_default } - }); - if let Some(x) = xtags { - fmt["xtags"] = json!(x); - } - fmt - } - - #[test] - fn xtags_descriptive_sets_track_type_and_is_descriptive() { - let fmt = audio_fmt(Some("ChQKBWFjb250EgtkZXNjcmlwdGl2ZQ"), true); - let a = build_audio(&fmt, "vid", "cpn", None).unwrap().unwrap(); - assert_eq!(a.track_type, Some(AudioTrackType::Descriptive)); - assert!(a.is_descriptive); - } - - #[test] - fn xtags_overrides_audio_is_default_heuristic() { - // acont=dubbed → Dubbed + is_descriptive false, even though - // audioIsDefault=false would make the legacy heuristic say "descriptive". - let fmt = audio_fmt(Some("Cg8KBWFjb250EgZkdWJiZWQ"), false); - let a = build_audio(&fmt, "vid", "cpn", None).unwrap().unwrap(); - assert_eq!(a.track_type, Some(AudioTrackType::Dubbed)); - assert!(!a.is_descriptive); - } - - #[test] - fn xtags_original_maps_to_original() { - let fmt = audio_fmt(Some("ChEKBWFjb250EghvcmlnaW5hbA"), true); - let a = build_audio(&fmt, "vid", "cpn", None).unwrap().unwrap(); - assert_eq!(a.track_type, Some(AudioTrackType::Original)); - assert!(!a.is_descriptive); - } - - #[test] - fn absent_xtags_falls_back_to_audio_is_default_heuristic() { - // No xtags → track_type None → is_descriptive == !audioIsDefault. - let a = build_audio(&audio_fmt(None, false), "vid", "cpn", None) - .unwrap() - .unwrap(); - assert_eq!(a.track_type, None); - assert!(a.is_descriptive); // !false - - let a = build_audio(&audio_fmt(None, true), "vid", "cpn", None) - .unwrap() - .unwrap(); - assert_eq!(a.track_type, None); - assert!(!a.is_descriptive); // !true - } - - #[test] - fn unparseable_xtags_falls_back_to_heuristic() { - // Garbage xtags → extract_audio_track_type None → heuristic used. - let a = build_audio(&audio_fmt(Some("!!!not-b64!!!"), false), "vid", "cpn", None) - .unwrap() - .unwrap(); - assert_eq!(a.track_type, None); - assert!(a.is_descriptive); - } - - // ---- S3 narrowed-$fields feed-enrich fetch + HARD fallback ------------ - // - // These touch the process-global Downloader, so they serialize on - // DOWNLOADER_TEST_LOCK (shared with the player_manager / stream_helper - // stub tests). They drive `fetch_android_metadata` DIRECTLY with an - // explicit `po_token: None` so they exercise the anonymous narrowed-reel - // path without depending on the process-global PoTokenProvider state that - // the potoken tests mutate. - - use crate::downloader::request::Request; - use crate::downloader::response::Response; - use crate::downloader::Downloader; - use crate::exceptions::NetworkError; - use crate::newpipe::DOWNLOADER_TEST_LOCK; - use std::sync::atomic::{AtomicUsize, Ordering}; - use std::sync::Arc; - - /// Serves the reel endpoint two ways, distinguished by the `$fields` - /// selector in the URL: the NARROWED metadata request (selector contains a - /// nested `playerResponse.` path) vs the FULL request (`$fields=playerResponse`). - /// Counts each so tests can assert whether the fallback fired. `visitor_id` - /// is declined (best-effort None) so the reel proceeds without it. - struct ReelStub { - narrowed_body: String, - full_body: String, - narrowed_hits: AtomicUsize, - full_hits: AtomicUsize, - } - - impl ReelStub { - fn new(narrowed_body: &str, full_body: &str) -> Self { - Self { - narrowed_body: narrowed_body.to_string(), - full_body: full_body.to_string(), - narrowed_hits: AtomicUsize::new(0), - full_hits: AtomicUsize::new(0), - } - } - } - - impl Downloader for ReelStub { - fn execute(&self, request: Request) -> Result { - let url = request.url().to_string(); - if url.contains("visitor_id") { - // Best-effort path: decline → reel proceeds without visitorData. - return Err(NetworkError::Transport("reel-stub: no visitor".into())); - } - if url.contains("reel_item_watch") { - // The nested dot-selector (`playerResponse.playabilityStatus,...`) - // only appears on the narrowed metadata request; the full request - // ends `&$fields=playerResponse` with no dot. - let (body, counter) = if url.contains("playerResponse.") { - (&self.narrowed_body, &self.narrowed_hits) - } else { - (&self.full_body, &self.full_hits) - }; - counter.fetch_add(1, Ordering::SeqCst); - return Ok(Response::new(200, "OK", Default::default(), body.clone(), url)); - } - Err(NetworkError::Transport("reel-stub: unexpected request".into())) - } - } - - #[test] - fn feed_metadata_falls_back_to_full_reel_when_narrowed_lacks_videodetails() { - let _g = DOWNLOADER_TEST_LOCK.lock(); - stream_helper::reset_visitor_data_cache(); - // Narrowed response has playabilityStatus but NO videoDetails — the - // shape the endpoint would return if it ignored/rejected the nested - // `$fields` selector into an error/empty body. The full response carries - // the real videoDetails + streamingData. - let stub = Arc::new(ReelStub::new( - r#"{"playerResponse":{"playabilityStatus":{"status":"OK"}}}"#, - r#"{"playerResponse":{"playabilityStatus":{"status":"OK"},"videoDetails":{"videoId":"vid","title":"FULL","lengthSeconds":"100","viewCount":"5"},"streamingData":{"formats":[]}}}"#, - )); - NewPipe::init(stub.clone() as Arc); - - let loc = NewPipe::preferred_localization(); - let cc = NewPipe::preferred_content_country(); - let out = fetch_android_metadata("vid", &loc, &cc, "cpn", None, None).unwrap(); - - // The narrowed request was tried, found videoDetails-less, and the full - // fetch was used as the safety fallback — feed enrich never regresses. - assert_eq!(stub.narrowed_hits.load(Ordering::SeqCst), 1, "narrowed tried"); - assert_eq!(stub.full_hits.load(Ordering::SeqCst), 1, "fell back to full"); - // The value returned is the FULL response (unwrapped playerResponse). - assert_eq!( - out.get("videoDetails").and_then(|v| v.get("title")).and_then(|v| v.as_str()), - Some("FULL") - ); - assert!(out.get("streamingData").is_some()); - stream_helper::reset_visitor_data_cache(); - } - - #[test] - fn feed_metadata_uses_narrowed_response_when_videodetails_present() { - let _g = DOWNLOADER_TEST_LOCK.lock(); - stream_helper::reset_visitor_data_cache(); - // Narrowed response DOES carry videoDetails → the endpoint honored the - // selector; use it directly, NO full-fetch fallback (this is the - // bytes-saved happy path). - let stub = Arc::new(ReelStub::new( - r#"{"playerResponse":{"playabilityStatus":{"status":"OK"},"videoDetails":{"videoId":"vid","title":"NARROWED","lengthSeconds":"42","viewCount":"7"}}}"#, - r#"{"playerResponse":{"playabilityStatus":{"status":"OK"},"videoDetails":{"videoId":"vid","title":"FULL"},"streamingData":{"formats":[]}}}"#, - )); - NewPipe::init(stub.clone() as Arc); - - let loc = NewPipe::preferred_localization(); - let cc = NewPipe::preferred_content_country(); - let out = fetch_android_metadata("vid", &loc, &cc, "cpn", None, None).unwrap(); - - assert_eq!(stub.narrowed_hits.load(Ordering::SeqCst), 1, "narrowed used"); - assert_eq!(stub.full_hits.load(Ordering::SeqCst), 0, "no fallback needed"); - assert_eq!( - out.get("videoDetails").and_then(|v| v.get("title")).and_then(|v| v.as_str()), - Some("NARROWED") - ); - // Narrowed body has no streamingData — proof we did NOT silently fall - // back to the full response. - assert!(out.get("streamingData").is_none()); - stream_helper::reset_visitor_data_cache(); - } - - #[test] - fn feed_metadata_falls_back_when_narrowed_videoid_mismatches() { - let _g = DOWNLOADER_TEST_LOCK.lock(); - stream_helper::reset_visitor_data_cache(); - // Narrowed response carries a videoDetails, but its videoId is NOT the - // requested one (a gutted/decoy shape the endpoint could return if it - // only half-honored the nested selector). The videoId-binding guard must - // REJECT it — a bare is_some() would have accepted this empty/decoy - // metadata — and fall through to the full fetch. - let stub = Arc::new(ReelStub::new( - r#"{"playerResponse":{"playabilityStatus":{"status":"OK"},"videoDetails":{"videoId":"DECOY","title":"WRONG"}}}"#, - r#"{"playerResponse":{"playabilityStatus":{"status":"OK"},"videoDetails":{"videoId":"vid","title":"FULL","lengthSeconds":"100","viewCount":"5"},"streamingData":{"formats":[]}}}"#, - )); - NewPipe::init(stub.clone() as Arc); - - let loc = NewPipe::preferred_localization(); - let cc = NewPipe::preferred_content_country(); - let out = fetch_android_metadata("vid", &loc, &cc, "cpn", None, None).unwrap(); - - assert_eq!(stub.narrowed_hits.load(Ordering::SeqCst), 1, "narrowed tried"); - assert_eq!(stub.full_hits.load(Ordering::SeqCst), 1, "mismatch -> fell back to full"); - assert_eq!( - out.get("videoDetails").and_then(|v| v.get("title")).and_then(|v| v.as_str()), - Some("FULL"), - "returned the full (correct-id) response, not the decoy" - ); - stream_helper::reset_visitor_data_cache(); - } } diff --git a/src/youtube/stream_helper.rs b/src/youtube/stream_helper.rs index e43c1f7..69b2571 100644 --- a/src/youtube/stream_helper.rs +++ b/src/youtube/stream_helper.rs @@ -6,8 +6,6 @@ // serviceIntegrityDimensions for poToken), POSTs to the right URL with // the right headers, returns the parsed JSON. -use once_cell::sync::Lazy; -use parking_lot::RwLock; use serde_json::{json, Map, Value}; use crate::downloader::request::Request; @@ -17,8 +15,7 @@ use crate::newpipe::NewPipe; use crate::youtube::client_request::{build_envelope, InnertubeClientRequestInfo}; use crate::youtube::constants::*; use crate::youtube::parsing::{ - android_user_agent, ios_user_agent, mobile_post_headers, visionos_user_agent, - youtube_post_headers, + android_user_agent, ios_user_agent, mobile_post_headers, youtube_post_headers, }; /// Builds a 12-char alphanumeric `cpn` (content playback nonce). NPE uses @@ -75,145 +72,6 @@ fn envelope_to_body(envelope: Value) -> Map { } } -/// A cached `visitorData` plus the instant it was fetched (for TTL expiry). -struct CachedVisitorData { - value: String, - at: std::time::Instant, -} - -impl CachedVisitorData { - fn is_fresh(&self) -> bool { - self.at.elapsed() < VISITOR_DATA_TTL - } -} - -/// How long a cached `visitorData` is reused before a fresh one is fetched. -/// Upstream fetches a brand-new visitorData on EVERY call; we cache to avoid a -/// round-trip per video, but bound the reuse so (a) traffic isn't pinned to a -/// single visitor identity indefinitely (rate-flag risk) and (b) a -/// subtly-bad-but-non-empty value can't ride the PRIMARY android reel call — -/// whose failure aborts extraction — forever. 15 min is a small fraction of -/// YouTube's own visitorData lifetime while still rotating regularly. -const VISITOR_DATA_TTL: std::time::Duration = std::time::Duration::from_secs(15 * 60); - -/// Process-wide cached `visitorData`. See [`CachedVisitorData`] / TTL above. -static VISITOR_DATA_CACHE: Lazy>> = - Lazy::new(|| RwLock::new(None)); - -/// Best-effort `visitorData` for the InnerTube client context. -/// -/// Upstream (`YoutubeStreamHelper`) attaches a freshly-fetched visitorData to -/// every non-token reel/web/ios/visionos call — *"We must always pass a valid -/// visitorData to get valid player responses"* — via -/// `YoutubeParsingHelper.getVisitorDataFromInnertube` (POST to the -/// `visitor_id` endpoint, read `responseContext.visitorData`). -/// -/// DELIBERATE DEVIATION (Loop-2 best-effort mandate): upstream THROWS when -/// visitorData can't be obtained, which would fail the whole extraction. -/// strawcore's anonymous happy-path has always worked WITHOUT visitorData, so -/// any failure here returns `None` and the caller proceeds exactly as it does -/// today. This is a durability improvement that can never regress the current -/// path — sending visitorData when we can get it, silently skipping it when we -/// can't. -/// -/// The cache is bounded by [`VISITOR_DATA_TTL`] and can be dropped early via -/// [`reset_visitor_data_cache`] (called when the PRIMARY android response is -/// rejected, so a poisoned value self-heals). Locking mirrors the Loop-1 -/// `player_manager` discipline: the `RwLock` is NEVER held across the network. -/// read (fresh?) → (miss/expired) fetch unlocked → write. A rare concurrent -/// double-fetch simply stores one of two interchangeable values. -fn get_visitor_data( - info: &InnertubeClientRequestInfo, - localization: &Localization, - content_country: &ContentCountry, - headers: Vec<(String, String)>, - domain: &str, -) -> Option { - // Serve a still-fresh cached value; an expired one falls through to refetch. - if let Some(cached) = VISITOR_DATA_CACHE.read().as_ref() { - if cached.is_fresh() && !visitor_force_stale() { - return Some(cached.value.clone()); - } - } - let env = build_envelope(info, localization, content_country, None); - let body = Value::Object(envelope_to_body(env)); - let url = format!("{domain}visitor_id{DISABLE_PRETTY_PRINT_PARAM}"); - // post_youtube returns Err on any non-200 / transport / parse failure → - // best-effort None (the caller then proceeds without visitorData). An - // expired-but-present cache entry is deliberately NOT served on a failed - // refetch: a stale visitorData is the thing we're rotating away from. - let parsed = post_youtube(&url, &body, headers).ok()?; - let visitor = visitor_data_from_response(&parsed)?; - *VISITOR_DATA_CACHE.write() = Some(CachedVisitorData { - value: visitor.clone(), - at: std::time::Instant::now(), - }); - Some(visitor) -} - -/// Drop the cached `visitorData` so the next extraction fetches a fresh one. -/// Called when the PRIMARY android response is rejected (decoy / playability -/// failure): those are exactly the symptoms a bad visitorData would produce on -/// the primary path, and since that failure aborts extraction the value would -/// otherwise never self-heal. -pub(crate) fn reset_visitor_data_cache() { - *VISITOR_DATA_CACHE.write() = None; -} - -/// Pull `responseContext.visitorData` out of a visitor_id response, treating an -/// absent or empty value as failure (`None`) — the trigger for the best-effort -/// fallback to "no visitorData" (today's behavior). -fn visitor_data_from_response(parsed: &Value) -> Option { - parsed - .get("responseContext") - .and_then(|v| v.get("visitorData")) - .and_then(|v| v.as_str()) - .filter(|s| !s.is_empty()) - .map(str::to_string) -} - -/// Test-only override that forces a cached entry to be treated as stale, -/// exercising the TTL-expiry refetch branch deterministically (fabricating an -/// old `Instant` via `checked_sub` is process/host-uptime dependent). Compiled -/// out entirely in non-test builds. -#[cfg(test)] -static FORCE_VISITOR_STALE: std::sync::atomic::AtomicBool = - std::sync::atomic::AtomicBool::new(false); - -#[inline] -fn visitor_force_stale() -> bool { - #[cfg(test)] - { - FORCE_VISITOR_STALE.load(std::sync::atomic::Ordering::Relaxed) - } - #[cfg(not(test))] - { - false - } -} - -/// Test hook: pre-seed the visitorData cache with a fresh entry (at = now) so -/// the cache-hit read path can be exercised without a network round-trip. -#[cfg(test)] -fn seed_visitor_data_cache(v: &str) { - *VISITOR_DATA_CACHE.write() = Some(CachedVisitorData { - value: v.to_string(), - at: std::time::Instant::now(), - }); -} - -/// Test hook: read the current cached visitorData value (if any). -#[cfg(test)] -fn peek_visitor_data_cache() -> Option { - VISITOR_DATA_CACHE.read().as_ref().map(|c| c.value.clone()) -} - -/// Test hook: force/clear the stale override (see [`FORCE_VISITOR_STALE`]). -#[cfg(test)] -fn set_force_visitor_stale(on: bool) { - FORCE_VISITOR_STALE.store(on, std::sync::atomic::Ordering::Relaxed); -} - /// WEB-client metadata-only /player call — used for microformat + /// thumbnails only; never used as a stream URL source. pub fn get_web_metadata_player_response( @@ -222,18 +80,7 @@ pub fn get_web_metadata_player_response( content_country: &ContentCountry, signature_timestamp: i32, ) -> Result { - let mut info = InnertubeClientRequestInfo::of_web_client(); - // Best-effort visitorData (upstream sends it on every non-token /player - // call). WEB uses the youtube.com domain + web headers for visitor_id. - if let Some(v) = get_visitor_data( - &info, - localization, - content_country, - youtube_post_headers(), - YOUTUBEI_V1_URL, - ) { - info.client_info.visitor_data = Some(v); - } + let info = InnertubeClientRequestInfo::of_web_client(); let env = build_envelope(&info, localization, content_country, None); let mut body = envelope_to_body(env); add_player_body_fields(&mut body, video_id, &generate_content_playback_nonce()); @@ -276,75 +123,14 @@ pub fn get_android_player_response( /// ANDROID `/reel/reel_item_watch` fallback — used when no poToken is /// available. Returns a `playerResponse`-shaped JSON wrapped inside the -/// reel response. Requests the FULL `playerResponse` (`$fields=playerResponse`) -/// — the stream-extraction path needs `streamingData` and everything else. +/// reel response. pub fn get_android_reel_player_response( video_id: &str, localization: &Localization, content_country: &ContentCountry, cpn: &str, ) -> Result { - reel_player_response_with_fields(video_id, localization, content_country, cpn, "playerResponse") -} - -/// Metadata-scoped variant of [`get_android_reel_player_response`] for the cheap -/// `stream_metadata` feed-enrich path. Requests a NARROWED `$fields` selector so -/// only `playabilityStatus` + `videoDetails` come back (~3-6 KB) instead of the -/// full ~150-500 KB reel body carrying every adaptive format's ~1.5 KB URL — the -/// feed path reads nothing outside those two subtrees (S3). -/// -/// BEST-EFFORT, with a full-fetch fallback owned by the caller -/// (`stream_extractor::fetch_android_metadata`): the `reel_item_watch` endpoint's -/// acceptance of the NESTED `$fields` selector (`playerResponse.videoDetails`) is -/// not guaranteed. If it honors the selector we save the bytes; if it rejects it -/// (400 → `Err` here) or returns a shape without `videoDetails`, the caller falls -/// back to the full reel fetch, so feed enrichment can NEVER regress. The -/// dot-form nested selector matches the spelling upstream/`get_web_metadata_player_response` -/// already use against the /player endpoint (`videoDetails.thumbnail.thumbnails`), -/// so the syntax itself is proven — only this endpoint's honoring of it is not. -pub fn get_android_reel_player_response_metadata( - video_id: &str, - localization: &Localization, - content_country: &ContentCountry, - cpn: &str, -) -> Result { - reel_player_response_with_fields( - video_id, - localization, - content_country, - cpn, - "playerResponse.playabilityStatus,playerResponse.videoDetails", - ) -} - -/// Shared body of the two reel helpers. `fields` is the `$fields` selector: the -/// full path passes `"playerResponse"` (byte-for-byte today's request), the -/// metadata path passes the narrowed nested selector. Everything else — the -/// best-effort visitorData, the envelope, the `playerRequest`/body shape, the -/// headers — is identical across both. -fn reel_player_response_with_fields( - video_id: &str, - localization: &Localization, - content_country: &ContentCountry, - cpn: &str, - fields: &str, -) -> Result { - let mut info = InnertubeClientRequestInfo::of_android_client(); - let ua = android_user_agent(content_country); - // Best-effort visitorData (upstream sends it on the reel call too). The reel - // path uses the gapis domain + android mobile headers for visitor_id. - // NOTE: the reel body shape is otherwise UNCHANGED from today (the upstream - // playerRequest/disablePlayerResponse realignment — recon W-1 — is out of - // scope for this pass; only touch it if a decoy/empty regression appears). - if let Some(v) = get_visitor_data( - &info, - localization, - content_country, - mobile_post_headers(&ua), - YOUTUBEI_V1_GAPIS_URL, - ) { - info.client_info.visitor_data = Some(v); - } + let info = InnertubeClientRequestInfo::of_android_client(); let env = build_envelope(&info, localization, content_country, None); let mut body = envelope_to_body(env); body.insert( @@ -356,9 +142,10 @@ fn reel_player_response_with_fields( ); add_player_body_fields(&mut body, video_id, cpn); let url = format!( - "{YOUTUBEI_V1_GAPIS_URL}reel/reel_item_watch{DISABLE_PRETTY_PRINT_PARAM}&t={t}&id={video_id}&$fields={fields}", + "{YOUTUBEI_V1_GAPIS_URL}reel/reel_item_watch{DISABLE_PRETTY_PRINT_PARAM}&t={t}&id={video_id}&$fields=playerResponse", t = generate_content_playback_nonce() ); + let ua = android_user_agent(content_country); post_youtube(&url, &Value::Object(body), mobile_post_headers(&ua)) } @@ -376,20 +163,8 @@ pub fn get_ios_player_response( visitor_data: Option<&str>, ) -> Result { let mut info = InnertubeClientRequestInfo::of_ios_client(); - let ua = ios_user_agent(content_country); - // Caller-supplied visitorData (paired with a poToken) wins; otherwise fetch - // it best-effort. iOS uses the youtube.com domain for visitor_id (upstream - // getIosPlayerResponse) even though the /player call goes to gapis. if let Some(v) = visitor_data { info.client_info.visitor_data = Some(v.into()); - } else if let Some(v) = get_visitor_data( - &info, - localization, - content_country, - mobile_post_headers(&ua), - YOUTUBEI_V1_URL, - ) { - info.client_info.visitor_data = Some(v); } let env = build_envelope(&info, localization, content_country, None); let mut body = envelope_to_body(env); @@ -401,42 +176,7 @@ pub fn get_ios_player_response( "{YOUTUBEI_V1_GAPIS_URL}player{DISABLE_PRETTY_PRINT_PARAM}&t={t}&id={video_id}", t = generate_content_playback_nonce() ); - post_youtube(&url, &Value::Object(body), mobile_post_headers(&ua)) -} - -/// visionOS /player call (client 101). Best-effort 4th client that recovers -/// higher-than-360p streams for SABR-only / "made for kids" videos. Mirrors -/// NPE `getVisionOsPlayerResponse`: hits the gapis /player endpoint with the -/// visionOS mobile header set, carries a best-effort visitorData, and NO -/// poToken. -pub fn get_visionos_player_response( - video_id: &str, - localization: &Localization, - content_country: &ContentCountry, - cpn: &str, -) -> Result { - let mut info = InnertubeClientRequestInfo::of_visionos_client(); - let ua = visionos_user_agent(content_country); - // visionOS returns valid responses ONLY with a visitorData — but still - // best-effort: on failure we proceed without it (the whole visionOS fetch - // is itself swallowed best-effort by the caller). Uses the youtube.com - // domain for visitor_id (upstream), gapis for /player. - if let Some(v) = get_visitor_data( - &info, - localization, - content_country, - mobile_post_headers(&ua), - YOUTUBEI_V1_URL, - ) { - info.client_info.visitor_data = Some(v); - } - let env = build_envelope(&info, localization, content_country, None); - let mut body = envelope_to_body(env); - add_player_body_fields(&mut body, video_id, cpn); - let url = format!( - "{YOUTUBEI_V1_GAPIS_URL}player{DISABLE_PRETTY_PRINT_PARAM}&t={t}&id={video_id}", - t = generate_content_playback_nonce() - ); + let ua = ios_user_agent(content_country); post_youtube(&url, &Value::Object(body), mobile_post_headers(&ua)) } @@ -455,12 +195,8 @@ fn post_youtube( } let resp = downloader.execute(builder.build())?; if resp.response_code() != 200 { - // Privacy: the full URL carries `id=` in the query — never - // put it in an error string (they reach Kotlin exception messages - // and logs). Scheme+host+path identifies the endpoint just fine. - let endpoint = url.split(['?', '#']).next().unwrap_or(url); return Err(ExtractionError::Network(NetworkError::Transport(format!( - "HTTP {} from {endpoint}", + "HTTP {} from {url}", resp.response_code() )))); } @@ -491,133 +227,4 @@ mod tests { let b = generate_content_playback_nonce(); assert_ne!(a, b); } - - #[test] - fn visitor_data_parsed_from_response() { - let resp = json!({ - "responseContext": { "visitorData": "Cgs1ZG1abc==" }, - "other": 1 - }); - assert_eq!( - visitor_data_from_response(&resp).as_deref(), - Some("Cgs1ZG1abc==") - ); - } - - #[test] - fn visitor_data_absent_or_empty_is_none_triggering_fallback() { - // Missing responseContext → None (→ caller proceeds without visitorData, - // i.e. exactly today's behavior). - assert!(visitor_data_from_response(&json!({})).is_none()); - // Present but empty string → None (empty is not a valid visitorData). - let resp = json!({ "responseContext": { "visitorData": "" } }); - assert!(visitor_data_from_response(&resp).is_none()); - // responseContext without the field → None. - let resp = json!({ "responseContext": { "foo": "bar" } }); - assert!(visitor_data_from_response(&resp).is_none()); - } - - // ---- visitorData cache: fresh-hit / TTL-expiry / reset ---------------- - // - // These touch the process-global visitorData cache and (for the refetch - // test) the global Downloader, so they serialize on DOWNLOADER_TEST_LOCK, - // shared with the player_manager test module. - - use crate::downloader::request::Request; - use crate::downloader::response::Response; - use crate::downloader::Downloader; - use crate::newpipe::DOWNLOADER_TEST_LOCK; - use std::sync::atomic::{AtomicUsize, Ordering}; - use std::sync::Arc; - - /// Serves any `visitor_id` POST with a fixed visitorData; counts requests. - struct VisitorStub { - visitor: String, - requests: AtomicUsize, - } - - impl Downloader for VisitorStub { - fn execute(&self, request: Request) -> Result { - self.requests.fetch_add(1, Ordering::SeqCst); - let url = request.url().to_string(); - if url.contains("visitor_id") { - let body = format!( - r#"{{"responseContext":{{"visitorData":"{}"}}}}"#, - self.visitor - ); - Ok(Response::new(200, "OK", Default::default(), body, url)) - } else { - Err(NetworkError::Transport("visitor-stub: unexpected request".into())) - } - } - } - - #[test] - fn fresh_cached_visitor_data_served_without_network() { - let _g = DOWNLOADER_TEST_LOCK.lock(); - set_force_visitor_stale(false); - reset_visitor_data_cache(); - seed_visitor_data_cache("FRESH_VD"); - // No downloader is needed: a fresh cache hit returns before any fetch. - let info = InnertubeClientRequestInfo::of_android_client(); - let got = get_visitor_data( - &info, - &Localization::default(), - &ContentCountry::default(), - mobile_post_headers("ua/1.0"), - YOUTUBEI_V1_GAPIS_URL, - ); - assert_eq!(got.as_deref(), Some("FRESH_VD")); - reset_visitor_data_cache(); - } - - #[test] - fn expired_visitor_data_triggers_refetch() { - let _g = DOWNLOADER_TEST_LOCK.lock(); - let stub = Arc::new(VisitorStub { - visitor: "REFRESHED_VD".into(), - requests: AtomicUsize::new(0), - }); - NewPipe::init(stub.clone() as Arc); - - reset_visitor_data_cache(); - seed_visitor_data_cache("STALE_VD"); // fresh instant… - set_force_visitor_stale(true); // …but forced past its TTL - - let info = InnertubeClientRequestInfo::of_android_client(); - let got = get_visitor_data( - &info, - &Localization::default(), - &ContentCountry::default(), - mobile_post_headers("ua/1.0"), - YOUTUBEI_V1_GAPIS_URL, - ); - - assert_eq!(got.as_deref(), Some("REFRESHED_VD"), "expired entry must refetch"); - assert_eq!( - stub.requests.load(Ordering::SeqCst), - 1, - "exactly one visitor_id refetch" - ); - // The refetched value replaced the stale one, stamped fresh. - set_force_visitor_stale(false); - assert_eq!(peek_visitor_data_cache().as_deref(), Some("REFRESHED_VD")); - - set_force_visitor_stale(false); - reset_visitor_data_cache(); - } - - #[test] - fn reset_clears_visitor_data_cache_for_self_heal() { - // Fix 2b mechanism: on a primary-android rejection the extractor calls - // reset_visitor_data_cache() so a poisoned value doesn't ride the - // primary reel call forever. - let _g = DOWNLOADER_TEST_LOCK.lock(); - set_force_visitor_stale(false); - reset_visitor_data_cache(); - seed_visitor_data_cache("POISONED_VD"); - assert_eq!(peek_visitor_data_cache().as_deref(), Some("POISONED_VD")); - reset_visitor_data_cache(); - assert!(peek_visitor_data_cache().is_none()); - } } diff --git a/src/youtube/xtags.rs b/src/youtube/xtags.rs deleted file mode 100644 index 54c7d0b..0000000 --- a/src/youtube/xtags.rs +++ /dev/null @@ -1,297 +0,0 @@ -// Minimal reader for an itag format's `xtags` field → AudioTrackType. -// -// Mirrors NPE YoutubeParsingHelper.extractAudioTrackType (commits b0bca7e7 + -// e3479a7c, 2026-06-01), which replaced the old `audioTrack.audioIsDefault` -// heuristic with the authoritative `acont` value carried in the format's -// `xtags` blob. -// -// `xtags` is a base64url-encoded `youtube.video.XTags` protobuf (upstream -// proto/youtube/video/xtags.proto): -// -// message KeyValuePair { optional string key = 1; optional string value = 2; } -// message XTags { repeated KeyValuePair xtags = 1; } -// -// The structure is trivial (two nested length-delimited string fields), so we -// read it with a ~40-line varint/field walker instead of pulling in a protobuf -// crate — matching the codebase's minimal-dependency approach. Everything is -// best-effort: any malformed input (bad base64, truncated protobuf, unknown -// `acont` value) yields `None`, and the caller falls back to the legacy -// `audioIsDefault` heuristic — never a panic, never a break. - -use crate::stream::AudioTrackType; - -/// Decode a format's `xtags` string and return its audio track type, mirroring -/// NPE `extractAudioTrackType`. Returns `None` when the blob is absent-shaped, -/// undecodable, carries no `acont` key, or maps to an unknown value. -pub fn extract_audio_track_type(xtags: &str) -> Option { - let bytes = base64url_decode(xtags)?; - let acont = xtags_find(&bytes, "acont")?; - match acont.as_str() { - "original" => Some(AudioTrackType::Original), - // NPE maps both "dubbed" and "dubbed-auto" → DUBBED. - "dubbed" | "dubbed-auto" => Some(AudioTrackType::Dubbed), - "descriptive" => Some(AudioTrackType::Descriptive), - "secondary" => Some(AudioTrackType::Secondary), - _ => None, - } -} - -/// Walk the top-level `XTags` message, returning the `value` of the first -/// `KeyValuePair` (field 1) whose `key` equals `wanted`. Unknown fields and -/// non-string wire types are skipped; any framing error bails to `None`. -fn xtags_find(buf: &[u8], wanted: &str) -> Option { - let mut i = 0; - while i < buf.len() { - let (tag, adv) = read_varint(buf, i)?; - i += adv; - let field = tag >> 3; - match wire_type(tag) { - // Length-delimited: field 1 is a KeyValuePair submessage. - 2 => { - let payload = read_len_delimited(buf, &mut i)?; - if field == 1 { - if let Some((k, v)) = parse_key_value_pair(payload) { - if k.as_deref() == Some(wanted) { - return v; - } - } - } - } - wt => skip_scalar(buf, &mut i, wt)?, - } - } - None -} - -/// Parse a `KeyValuePair` submessage into (key, value); either may be absent. -fn parse_key_value_pair(buf: &[u8]) -> Option<(Option, Option)> { - let mut key = None; - let mut val = None; - let mut i = 0; - while i < buf.len() { - let (tag, adv) = read_varint(buf, i)?; - i += adv; - let field = tag >> 3; - match wire_type(tag) { - 2 => { - let payload = read_len_delimited(buf, &mut i)?; - let s = std::str::from_utf8(payload).ok()?.to_string(); - match field { - 1 => key = Some(s), - 2 => val = Some(s), - _ => {} - } - } - wt => skip_scalar(buf, &mut i, wt)?, - } - } - Some((key, val)) -} - -#[inline] -fn wire_type(tag: u64) -> u8 { - (tag & 0x7) as u8 -} - -/// Read a length-delimited (wire type 2) payload, advancing `*i` past it. -fn read_len_delimited<'a>(buf: &'a [u8], i: &mut usize) -> Option<&'a [u8]> { - let (len, adv) = read_varint(buf, *i)?; - *i += adv; - // `usize::try_from` (not `as usize`) so a >4 GiB length can't wrap on a - // 32-bit target (armeabi-v7a is a real Straw ABI) — it fails closed instead. - let len = usize::try_from(len).ok()?; - let end = i.checked_add(len)?; - if end > buf.len() { - return None; - } - let payload = &buf[*i..end]; - *i = end; - Some(payload) -} - -/// Advance `*i` past a non-length-delimited field. Groups (3/4) and any -/// unrecognized wire type bail out — they never appear in a well-formed XTags. -fn skip_scalar(buf: &[u8], i: &mut usize, wire: u8) -> Option<()> { - match wire { - 0 => { - let (_, adv) = read_varint(buf, *i)?; - *i += adv; - } - 1 => { - *i = i.checked_add(8)?; - if *i > buf.len() { - return None; - } - } - 5 => { - *i = i.checked_add(4)?; - if *i > buf.len() { - return None; - } - } - _ => return None, - } - Some(()) -} - -/// Read a base-128 varint at `start`; returns (value, bytes_consumed). Caps at -/// 10 bytes (64 bits) so a malformed run can't spin. -fn read_varint(buf: &[u8], start: usize) -> Option<(u64, usize)> { - let mut result: u64 = 0; - let mut shift: u32 = 0; - let mut i = start; - loop { - if i >= buf.len() || shift >= 64 { - return None; - } - let byte = buf[i]; - result |= u64::from(byte & 0x7f) << shift; - i += 1; - if byte & 0x80 == 0 { - return Some((result, i - start)); - } - shift += 7; - } -} - -/// Strict base64url (RFC 4648 §5, no padding required) decoder. Rejects any -/// non-url-safe byte (`+`, `/`, whitespace) → `None`, matching upstream's use -/// of `Base64.getUrlDecoder()`. Stops at the first `=` padding byte. -fn base64url_decode(s: &str) -> Option> { - fn sextet(c: u8) -> Option { - match c { - b'A'..=b'Z' => Some(c - b'A'), - b'a'..=b'z' => Some(c - b'a' + 26), - b'0'..=b'9' => Some(c - b'0' + 52), - b'-' => Some(62), - b'_' => Some(63), - _ => None, - } - } - if s.is_empty() { - return None; - } - let mut out = Vec::with_capacity(s.len() * 3 / 4 + 3); - let mut acc: u32 = 0; - let mut bits: u32 = 0; - for &c in s.as_bytes() { - if c == b'=' { - break; - } - let v = u32::from(sextet(c)?); - acc = (acc << 6) | v; - bits += 6; - if bits >= 8 { - bits -= 8; - out.push((acc >> bits) as u8); - } - } - Some(out) -} - -#[cfg(test)] -mod tests { - use super::*; - - // Independently-generated (Python urlsafe_b64encode of a hand-built XTags - // protobuf) so these vectors are an external oracle, not a round-trip of - // our own encoder. See the Loop-2 build notes for the generator. - const XT_ORIGINAL: &str = "ChEKBWFjb250EghvcmlnaW5hbA"; - const XT_DUBBED: &str = "Cg8KBWFjb250EgZkdWJiZWQ"; - const XT_DUBBED_AUTO: &str = "ChQKBWFjb250EgtkdWJiZWQtYXV0bw"; - const XT_DESCRIPTIVE: &str = "ChQKBWFjb250EgtkZXNjcmlwdGl2ZQ"; - const XT_SECONDARY: &str = "ChIKBWFjb250EglzZWNvbmRhcnk"; - // Two pairs: {lang=en},{acont=descriptive} — proves we scan past a - // non-matching pair to find `acont`. - const XT_MULTI: &str = "CgoKBGxhbmcSAmVuChQKBWFjb250EgtkZXNjcmlwdGl2ZQ"; - // Single pair {lang=en} — no `acont` key. - const XT_NO_ACONT: &str = "CgoKBGxhbmcSAmVu"; - - #[test] - fn maps_each_known_acont_value() { - assert_eq!( - extract_audio_track_type(XT_ORIGINAL), - Some(AudioTrackType::Original) - ); - assert_eq!( - extract_audio_track_type(XT_DUBBED), - Some(AudioTrackType::Dubbed) - ); - assert_eq!( - extract_audio_track_type(XT_DUBBED_AUTO), - Some(AudioTrackType::Dubbed) - ); - assert_eq!( - extract_audio_track_type(XT_DESCRIPTIVE), - Some(AudioTrackType::Descriptive) - ); - assert_eq!( - extract_audio_track_type(XT_SECONDARY), - Some(AudioTrackType::Secondary) - ); - } - - #[test] - fn finds_acont_among_multiple_pairs() { - assert_eq!( - extract_audio_track_type(XT_MULTI), - Some(AudioTrackType::Descriptive) - ); - } - - #[test] - fn absent_acont_key_yields_none() { - assert_eq!(extract_audio_track_type(XT_NO_ACONT), None); - } - - #[test] - fn malformed_input_yields_none_never_panics() { - // Empty, non-base64url chars, valid base64url but not a protobuf, - // and a truncated length-delimited frame. - assert_eq!(extract_audio_track_type(""), None); - assert_eq!(extract_audio_track_type("!!!not base64!!!"), None); - assert_eq!(extract_audio_track_type("++//"), None); // standard-b64 chars rejected - assert_eq!(extract_audio_track_type("Zm9vYmFy"), None); // "foobar", valid b64url, junk proto - // 0x0A (field1,LEN) claiming length 0x7F with no body → truncated. - assert_eq!(extract_audio_track_type("Cn8"), None); - } - - #[test] - fn unknown_acont_value_yields_none() { - // {acont=weird} — well-formed protobuf, value not in the enum. - // ld(1, ld(1,"acont")+ld(2,"weird")) - let bytes = { - let kv = [ - &[0x0a, 0x05][..], - b"acont", - &[0x12, 0x05], - b"weird", - ] - .concat(); - let mut top = vec![0x0a, kv.len() as u8]; - top.extend_from_slice(&kv); - top - }; - // sanity: our decoder round-trips what we assert on - let b64 = { - // encode without padding, url-safe - const A: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; - let mut out = String::new(); - for chunk in bytes.chunks(3) { - let b = [ - chunk[0], - *chunk.get(1).unwrap_or(&0), - *chunk.get(2).unwrap_or(&0), - ]; - let n = (u32::from(b[0]) << 16) | (u32::from(b[1]) << 8) | u32::from(b[2]); - let take = chunk.len() + 1; - for k in 0..take { - out.push(A[((n >> (18 - 6 * k)) & 0x3f) as usize] as char); - } - } - out - }; - assert_eq!(base64url_decode(&b64).unwrap(), bytes); - assert_eq!(extract_audio_track_type(&b64), None); - } -} diff --git a/tests/foundation_smoke.rs b/tests/foundation_smoke.rs index 2544e87..1fb88f2 100644 --- a/tests/foundation_smoke.rs +++ b/tests/foundation_smoke.rs @@ -9,22 +9,16 @@ use std::sync::Arc; -use strawcore_core::downloader::request::Request; -use strawcore_core::downloader::ReqwestDownloader; -use strawcore_core::exceptions::NetworkError; -use strawcore_core::localization::{ContentCountry, Localization}; -use strawcore_core::{Downloader, NewPipe}; - -/// The Downloader trait dropped its `get` convenience (commit f917e4a); -/// mirror it here so the suite reads as before. -fn get(dl: &impl Downloader, url: &str) -> Result { - dl.execute(Request::get(url).build()) -} +use strawcore::downloader::request::Request; +use strawcore::downloader::ReqwestDownloader; +use strawcore::exceptions::NetworkError; +use strawcore::localization::{ContentCountry, Localization}; +use strawcore::{Downloader, NewPipe}; #[test] fn get_through_default_downloader() { let dl = ReqwestDownloader::new().expect("build downloader"); - let resp = get(&dl, "https://httpbin.org/get").expect("transport"); + let resp = dl.get("https://httpbin.org/get").expect("transport"); assert_eq!(resp.response_code(), 200); assert!(resp.response_body().contains("\"url\"")); } @@ -32,7 +26,9 @@ fn get_through_default_downloader() { #[test] fn latest_url_follows_redirects() { let dl = ReqwestDownloader::new().expect("build downloader"); - let resp = get(&dl, "https://httpbin.org/redirect/3").expect("transport"); + let resp = dl + .get("https://httpbin.org/redirect/3") + .expect("transport"); assert_eq!(resp.response_code(), 200); assert!( resp.latest_url().ends_with("/get"), @@ -44,14 +40,14 @@ fn latest_url_follows_redirects() { #[test] fn non_2xx_returns_ok_not_err() { let dl = ReqwestDownloader::new().expect("build downloader"); - let resp = get(&dl, "https://httpbin.org/status/404").expect("transport"); + let resp = dl.get("https://httpbin.org/status/404").expect("transport"); assert_eq!(resp.response_code(), 404); } #[test] fn http_429_surfaces_as_recaptcha_err() { let dl = ReqwestDownloader::new().expect("build downloader"); - let err = get(&dl, "https://httpbin.org/status/429").expect_err("429 must be NetworkError"); + let err = dl.get("https://httpbin.org/status/429").expect_err("429 must be NetworkError"); match err { NetworkError::Recaptcha { url } => assert!(url.contains("/status/429")), other => panic!("expected Recaptcha, got {other:?}"), @@ -76,8 +72,8 @@ fn localization_header_attached_when_enabled() { #[test] fn header_keys_lowercased_in_response() { let dl = ReqwestDownloader::new().expect("build downloader"); - let resp = get(&dl, "https://httpbin.org/get").expect("transport"); - for k in resp.response_headers().keys() { + let resp = dl.get("https://httpbin.org/get").expect("transport"); + for (k, _) in resp.response_headers() { assert_eq!(k, &k.to_ascii_lowercase(), "header key {k} not lowercased"); } } @@ -92,9 +88,7 @@ fn newpipe_singleton_wires_downloader() { ); let from_global = NewPipe::downloader().expect("downloader registered"); - let resp = from_global - .execute(Request::get("https://httpbin.org/get").build()) - .expect("transport"); + let resp = from_global.get("https://httpbin.org/get").expect("transport"); assert_eq!(resp.response_code(), 200); assert_eq!(NewPipe::preferred_localization().localization_code(), "en-GB"); } diff --git a/tests/js_phase2_offline.rs b/tests/js_phase2_offline.rs index 3e3a6ad..ea34f12 100644 --- a/tests/js_phase2_offline.rs +++ b/tests/js_phase2_offline.rs @@ -14,7 +14,7 @@ // * url_with_throttling_parameter_deobfuscated round-trip changes &n= // and caches the result -use strawcore_core::youtube::js::{signature, nsig, runtime, DeobfError}; +use strawcore::youtube::js::{signature, nsig, runtime, DeobfError}; // Synthetic minified player.js — replicates the shape of real YT player.js. //