From dd964407ccf1a7733da6863123a2812ac64d2333 Mon Sep 17 00:00:00 2001 From: Cobb Date: Thu, 6 Aug 2026 06:35:27 -0700 Subject: [PATCH] youtube: fall back to VISIONOS primary when ANDROID is bot-walled YouTube's 2026 poToken/BotGuard enforcement now rejects the pot-less ANDROID client outright (playabilityStatus "sign in to confirm you're not a bot"), which aborted the whole extraction since ANDROID was the required primary. stream_info_with() now cascades ANDROID -> VISIONOS for the primary player response: when ANDROID is unusable (fetch error, failed playability, or decoy), retry with VISIONOS (client 101) as the primary. VISIONOS is pot-free, visitorData-based, and needs no JS player -- the current survivor (yt-dlp default; NewPipeExtractor #1508). Only if VISIONOS also fails do we surface the ORIGINAL ANDROID error (keeps the real bot-wall reason visible). The fallback drops the (poisoned) visitorData cache first so VISIONOS mints a fresh one (the ingredient that passes attestation). On the VISIONOS-primary path the primary source carries no poToken (null) and uses the VISIONOS cpn, so no ANDROID pot/cpn is appended to a visionOS URL (would 403); the redundant add-on VISIONOS fetch is skipped. Output shape is unchanged (still the muxed itag-18 the app plays) -- no app or delivery-labeling change. 5 new unit tests. --- examples/emit_json.rs | 141 ++++++++++++++++++ examples/live_extract.rs | 50 +++++++ src/youtube/stream_extractor.rs | 253 +++++++++++++++++++++++++++++--- 3 files changed, 421 insertions(+), 23 deletions(-) create mode 100644 examples/emit_json.rs create mode 100644 examples/live_extract.rs diff --git a/examples/emit_json.rs b/examples/emit_json.rs new file mode 100644 index 0000000..6e35034 --- /dev/null +++ b/examples/emit_json.rs @@ -0,0 +1,141 @@ +// Differential-verification JSON emitter (Phase-1 harness, NOT shipped). +// +// Additive example only — adds no production code. Emits strawcore's +// extraction for one video id as the harness's normalized JSON schema on +// stdout, so it can be diffed against the upstream NewPipe-Extractor (Java) +// oracle. See /root/build/npe-diff/ for the driver + differ. +// +// cargo run --example emit_json -- [visionos-on] +// +// Default matches the shipped app path (ExtractOptions::default(), visionOS +// OFF). Pass `visionos-on` to fetch the visionOS client (client 101) — that +// is what upstream NPE does UNCONDITIONALLY, so `visionos-on` is the +// apples-to-apples mode for an NPE fidelity diff. +use std::sync::Arc; + +use serde_json::{json, Value}; +use strawcore_core::stream::{AudioStream, DeliveryMethod, StreamInfo, VideoStream}; +use strawcore_core::youtube::stream_extractor::{stream_info_with, ExtractOptions}; +use strawcore_core::NewPipe; + +/// Parse the innertube client tag (`&c=WEB` / `&c=ANDROID` / ...) off a +/// googlevideo URL — this is the "which client sourced this stream" signal +/// that made the visionOS outage visible. +fn client_source(url: &str) -> Value { + url.split(['&', '?']) + .find_map(|kv| kv.strip_prefix("c=")) + .map(|c| Value::String(c.to_string())) + .unwrap_or(Value::Null) +} + +fn delivery_str(d: DeliveryMethod) -> &'static str { + match d { + DeliveryMethod::Progressive => "progressive", + DeliveryMethod::Dash => "dash", + DeliveryMethod::Hls => "hls", + DeliveryMethod::Torrent => "torrent", + } +} + +fn video_json(s: &VideoStream, kind: &str) -> Value { + json!({ + "kind": kind, + "itag": s.itag, + "delivery": delivery_str(s.delivery), + "format": s.format.extension(), + "mime": s.format.mime(), + "codec": s.codec, + "resolution": if s.resolution.is_empty() { Value::Null } else { json!(s.resolution) }, + "fps": s.fps, + "bitrate": s.bandwidth, + "has_url": !s.url.is_empty(), + "client_source": client_source(&s.url), + }) +} + +fn audio_json(s: &AudioStream) -> Value { + json!({ + "kind": "audio", + "itag": s.itag, + "delivery": delivery_str(s.delivery), + "format": s.format.extension(), + "mime": s.format.mime(), + "codec": s.codec, + "resolution": Value::Null, + "fps": 0, + "bitrate": s.average_bitrate_kbps, + "has_url": !s.url.is_empty(), + "client_source": client_source(&s.url), + }) +} + +fn emit(info: &StreamInfo) -> Value { + let mut streams: Vec = Vec::new(); + for s in &info.video_streams { + streams.push(video_json(s, "muxed")); + } + for s in &info.video_only_streams { + streams.push(video_json(s, "video_only")); + } + for s in &info.audio_streams { + streams.push(audio_json(s)); + } + // Deterministic order so diffs are stable: (kind, itag, delivery). + streams.sort_by(|a, b| { + ( + a["kind"].as_str().unwrap_or(""), + a["itag"].as_u64().unwrap_or(0), + a["delivery"].as_str().unwrap_or(""), + ) + .cmp(&( + b["kind"].as_str().unwrap_or(""), + b["itag"].as_u64().unwrap_or(0), + b["delivery"].as_str().unwrap_or(""), + )) + }); + + json!({ + "source": "strawcore", + "video_id": info.video_id, + "meta": { + "name": info.name, + "duration_seconds": info.duration_seconds, + "stream_type": format!("{:?}", info.stream_type), + "uploader_id": info.uploader_id, + "uploader_name": info.uploader_name, + "view_count": info.view_count, + }, + "dash_present": info.dash_manifest_url.is_some(), + "hls_present": info.hls_manifest_url.is_some(), + "counts": { + "muxed": info.video_streams.len(), + "video_only": info.video_only_streams.len(), + "audio": info.audio_streams.len(), + }, + "streams": streams, + }) +} + +fn main() { + let mut args = std::env::args().skip(1); + let vid = args.next().unwrap_or_else(|| "dQw4w9WgXcQ".into()); + let visionos = args.next().as_deref() == Some("visionos-on"); + + let dl = + Arc::new(strawcore_core::downloader::ReqwestDownloader::new().expect("build downloader")); + NewPipe::init(dl); + + let opts = ExtractOptions { + fetch_visionos_client: visionos, + ..Default::default() + }; + match stream_info_with(&vid, opts) { + Ok(info) => { + println!("{}", serde_json::to_string_pretty(&emit(&info)).unwrap()); + } + Err(e) => { + eprintln!("strawcore extraction FAILED for {vid}: {e:?}"); + std::process::exit(2); + } + } +} diff --git a/examples/live_extract.rs b/examples/live_extract.rs new file mode 100644 index 0000000..2d802a9 --- /dev/null +++ b/examples/live_extract.rs @@ -0,0 +1,50 @@ +// Throwaway live diagnostic (playback-outage triage). Not shipped. +use std::sync::Arc; +use strawcore_core::downloader::ReqwestDownloader; +use strawcore_core::youtube::stream_extractor::{stream_info_with, ExtractOptions}; +use strawcore_core::NewPipe; + +fn client_of(url: &str) -> String { + url.split('&') + .find_map(|kv| kv.strip_prefix("c=")) + .unwrap_or("?") + .to_string() +} + +fn report(label: &str, vid: &str, visionos: bool) { + let opts = ExtractOptions { + fetch_visionos_client: visionos, + ..Default::default() + }; + match stream_info_with(vid, opts) { + Ok(i) => { + let a = i.audio_streams.first().map(|s| client_of(&s.url)); + let v = i + .video_streams + .first() + .or_else(|| i.video_only_streams.first()) + .map(|s| client_of(&s.url)); + println!( + "[{label}] OK audio={} video={} video_only={} dash={} hls={} | first audio c={:?} first video c={:?}", + i.audio_streams.len(), + i.video_streams.len(), + i.video_only_streams.len(), + i.dash_manifest_url.is_some(), + i.hls_manifest_url.is_some(), + a, + v, + ); + } + Err(e) => println!("[{label}] ERR {e:?}"), + } +} + +fn main() { + let dl = Arc::new(ReqwestDownloader::new().expect("downloader")); + NewPipe::init(dl); + let vid = std::env::args() + .nth(1) + .unwrap_or_else(|| "dQw4w9WgXcQ".into()); + report("visionOS ON (shipped)", &vid, true); + report("visionOS OFF (like 91) ", &vid, false); +} diff --git a/src/youtube/stream_extractor.rs b/src/youtube/stream_extractor.rs index 36c4336..ea0adcf 100644 --- a/src/youtube/stream_extractor.rs +++ b/src/youtube/stream_extractor.rs @@ -175,25 +175,49 @@ pub fn stream_info_with( // A panicked worker joins as Err → 0, same as an extraction failure. (android, sig_ts.join().unwrap_or(0)) }); - let player_response = android_result?; + // ── Primary player-response selection: ANDROID → VISIONOS cascade ──────── + // ANDROID was the REQUIRED primary, but the 2026 poToken/BotGuard bot-wall + // now rejects the pot-less ANDROID client outright (playabilityStatus + // "sign in to confirm you're not a bot"). When the ANDROID result is + // unusable — a fetch error, a failed playability check, OR a decoy — fall + // back to VISIONOS (client 101) as the PRIMARY: it's pot-free, + // visitorData-based, and needs no JS player, so it's the current survivor + // (yt-dlp default; NPE #1508). Only if VISIONOS ALSO fails do we surface the + // ORIGINAL ANDROID error. + // + // A primary-android rejection is exactly the symptom a poisoned visitorData + // would produce, so `select_primary` drops the cached visitorData (via + // `reset_visitor_data_cache`) before the VISIONOS call — VISIONOS then mints + // a FRESH visitorData, the ingredient that makes it pass attestation. The + // visitorData-first ordering inside `get_visionos_player_response` is + // preserved (it fetches visitorData before the /player call). + let visionos_primary_cpn = generate_content_playback_nonce(); + let (player_response, primary_is_visionos) = select_primary( + android_result, + video_id, + stream_helper::reset_visitor_data_cache, + || { + stream_helper::get_visionos_player_response( + video_id, + &localization, + &content_country, + &visionos_primary_cpn, + ) + }, + )?; + // GOTCHA 2: the primary FormatSource's cpn MUST match the client that + // produced the streams (process_url appends it to the URL): `android_cpn` + // normally, the visionOS fallback cpn when VISIONOS won. + let primary_cpn: &str = if primary_is_visionos { + &visionos_primary_cpn + } else { + &android_cpn + }; - // A primary-android rejection (playability failure or decoy) is exactly the - // symptom a poisoned visitorData would produce on the primary reel call — - // and since it aborts extraction the value would otherwise never self-heal. - // Drop the cached visitorData so the next extraction fetches a fresh one. - if let Err(e) = check_playability_status(&player_response) { - stream_helper::reset_visitor_data_cache(); - return Err(e); - } - if is_player_response_not_valid(&player_response, video_id) { - stream_helper::reset_visitor_data_cache(); - return Err(ExtractionError::Other( - "ANDROID player response is not valid (decoy detected)".into(), - )); - } - - // Borrow streamingData out of the (owned, function-lived) player + // Borrow streamingData out of the (owned, function-lived) PRIMARY player // response rather than deep-cloning the largest subtree of the response. + // (Named `android_streaming_data` for historical continuity; on the + // VISIONOS-primary path it is VISIONOS's streamingData.) let android_streaming_data: &Value = player_response.get("streamingData").unwrap_or(&NULL_VALUE); @@ -241,8 +265,10 @@ pub fn stream_info_with( // SABR-only / "made for kids" videos. Carries NO poToken and (like iOS) is // decoy-checked before its streamingData is used. A failure leaves the // output exactly as it is today. + // GOTCHA 3: when VISIONOS is already the PRIMARY, skip this redundant add-on + // fetch — else we'd hit /player for visionOS twice. let (visionos_response, visionos_cpn): (Option, Option) = - if options.fetch_visionos_client { + if options.fetch_visionos_client && !primary_is_visionos { let cpn = generate_content_playback_nonce(); match stream_helper::get_visionos_player_response( video_id, @@ -261,10 +287,19 @@ pub fn stream_info_with( .and_then(|r| r.get("streamingData")) .unwrap_or(&NULL_VALUE); - let android_streaming_pot = android_token - .as_ref() - .map(|t| t.streaming_data_po_token.clone()) - .or_else(|| options.android_streaming_pot.clone()); + // GOTCHA 1: on the VISIONOS-primary path the primary streamingData is + // VISIONOS's — it carries NO poToken, and appending an ANDROID streaming pot + // to a visionOS URL corrupts it → 403. Force the pot to None so neither the + // primary FormatSource nor the (now visionOS-sourced) DASH/HLS manifests get + // an ANDROID pot appended. + let android_streaming_pot = if primary_is_visionos { + None + } else { + android_token + .as_ref() + .map(|t| t.streaming_data_po_token.clone()) + .or_else(|| options.android_streaming_pot.clone()) + }; let ios_streaming_pot = ios_token .as_ref() .map(|t| t.streaming_data_po_token.clone()) @@ -294,7 +329,12 @@ pub fn stream_info_with( FormatSource { streaming_data: android_streaming_data, client: "ANDROID", - cpn: &android_cpn, + // GOTCHA 2: cpn matches the winning client (android_cpn, or the + // visionOS fallback cpn when VISIONOS is primary). + cpn: primary_cpn, + // GOTCHA 1: android_streaming_pot is already forced to None on the + // VISIONOS-primary path, so the primary source never appends an + // ANDROID pot to a visionOS URL. pot: android_streaming_pot.as_deref(), }, FormatSource { @@ -578,6 +618,52 @@ fn is_player_response_not_valid(player_response: &Value, video_id: &str) -> bool returned.map(|r| r != video_id).unwrap_or(false) } +/// Is `resp` usable as the PRIMARY player response? Collapses the two checks the +/// primary gate applies: playability must be OK (not the bot-wall / geo / +/// paywall statuses) AND it must not be a decoy (its `videoDetails.videoId` must +/// match). Returns the mapped playability error, or a generic decoy error. +fn primary_response_usable(resp: &Value, video_id: &str) -> Result<(), ExtractionError> { + check_playability_status(resp)?; + if is_player_response_not_valid(resp, video_id) { + return Err(ExtractionError::Other( + "player response is not valid (decoy detected)".into(), + )); + } + Ok(()) +} + +/// Pick the PRIMARY player response, cascading ANDROID → VISIONOS. +/// +/// `android_result` is the ANDROID `/player` outcome. When it is unusable — a +/// fetch error, a failed playability check, OR a decoy — `on_fallback` runs +/// once (drops the poisoned visitorData cache) and then `fetch_visionos` +/// performs the VISIONOS `/player` call. Returns `(player_response, +/// primary_is_visionos)` for the winning client, or the ORIGINAL ANDROID error +/// when VISIONOS also fails. `fetch_visionos` is only invoked on the fallback +/// path (so a usable ANDROID never triggers a visionOS round-trip). +fn select_primary( + android_result: Result, + video_id: &str, + on_fallback: impl FnOnce(), + fetch_visionos: impl FnOnce() -> Result, +) -> Result<(Value, bool), ExtractionError> { + let android_error = match android_result { + Ok(resp) => match primary_response_usable(&resp, video_id) { + Ok(()) => return Ok((resp, false)), + Err(e) => e, + }, + Err(e) => e, + }; + // ANDROID unusable → drop the (likely poisoned) visitorData, then try + // VISIONOS as the primary. On any VISIONOS failure, surface the ORIGINAL + // ANDROID error rather than the visionOS one. + on_fallback(); + match fetch_visionos() { + Ok(resp) if primary_response_usable(&resp, video_id).is_ok() => Ok((resp, true)), + _ => Err(android_error), + } +} + fn populate_video_details(info: &mut StreamInfo, player_response: &Value) { let Some(vd) = player_response.get("videoDetails") else { return; @@ -1333,6 +1419,127 @@ mod tests { assert!(!ExtractOptions::default().fetch_visionos_client); } + // ---- ANDROID → VISIONOS primary cascade (2026 bot-wall fix) ------------ + + fn walled_response(video_id: &str) -> Value { + // The 2026 poToken bot-wall shape on the pot-less ANDROID client. + json!({ + "playabilityStatus": { + "status": "LOGIN_REQUIRED", + "reason": "Sign in to confirm you're not a bot" + }, + "videoDetails": {"videoId": video_id} + }) + } + + fn ok_response(video_id: &str, itag: u64) -> Value { + json!({ + "playabilityStatus": {"status": "OK"}, + "videoDetails": {"videoId": video_id}, + "streamingData": {"formats": [{"itag": itag}]} + }) + } + + #[test] + fn cascade_falls_back_to_visionos_when_android_walled() { + // android-playability-fail + visionos-ok → visionOS becomes primary. + let fallback_ran = std::cell::Cell::new(false); + let (resp, is_visionos) = select_primary( + Ok(walled_response("vid")), + "vid", + || fallback_ran.set(true), + || Ok(ok_response("vid", 18)), + ) + .expect("visionOS is usable → cascade must succeed"); + assert!( + is_visionos, + "VISIONOS must become the primary when ANDROID is walled" + ); + assert!( + fallback_ran.get(), + "the visitorData-cache reset hook must run before the visionOS fetch" + ); + // Proves we returned the VISIONOS response (itag 18 lives only there). + assert_eq!( + resp.pointer("/streamingData/formats/0/itag") + .and_then(|v| v.as_u64()), + Some(18) + ); + } + + #[test] + fn cascade_falls_back_on_android_fetch_error() { + // A hard ANDROID fetch error (not just a playability fail) also cascades. + let (_resp, is_visionos) = select_primary( + Err(ExtractionError::Other("android network boom".into())), + "vid", + || {}, + || Ok(ok_response("vid", 18)), + ) + .expect("visionOS is usable → cascade must succeed"); + assert!(is_visionos); + } + + #[test] + fn cascade_returns_original_android_error_when_both_fail() { + // both-fail → Err, and it's the ORIGINAL android error, not the visionOS + // one. + let out = select_primary( + Err(ExtractionError::Other("ORIGINAL android error".into())), + "vid", + || {}, + || Ok(walled_response("vid")), + ); + match out { + Err(ExtractionError::Other(msg)) => assert_eq!(msg, "ORIGINAL android error"), + other => panic!("expected the original android error, got {other:?}"), + } + } + + #[test] + fn cascade_keeps_android_when_usable_and_skips_visionos() { + // A usable ANDROID stays primary and MUST NOT trigger a visionOS fetch + // or the cache reset. + let visionos_called = std::cell::Cell::new(false); + let fallback_ran = std::cell::Cell::new(false); + let (_resp, is_visionos) = select_primary( + Ok(ok_response("vid", 22)), + "vid", + || fallback_ran.set(true), + || { + visionos_called.set(true); + Ok(json!({})) + }, + ) + .expect("usable android → Ok"); + assert!(!is_visionos, "usable ANDROID must remain the primary"); + assert!( + !visionos_called.get(), + "usable ANDROID must NOT trigger the visionOS fallback fetch" + ); + assert!( + !fallback_ran.get(), + "no visitorData reset when ANDROID is usable" + ); + } + + #[test] + fn cascade_treats_android_decoy_as_unusable() { + // A decoy ANDROID response (videoId mismatch) is unusable → cascade to + // visionOS. + let decoy = json!({ + "playabilityStatus": {"status": "OK"}, + "videoDetails": {"videoId": "WRONG"} + }); + let (_resp, is_visionos) = + select_primary(Ok(decoy), "vid", || {}, || Ok(ok_response("vid", 18))) + .expect("visionOS usable → Ok"); + assert!( + is_visionos, + "an ANDROID decoy must fall through to visionOS" + ); + } + // ---- I-3 xtags audioTrackType ----------------------------------------- // itag 140 (M4A audio) + a direct url with no `n=` param, so process_url