npe-sync: visitorData + visionOS(101) + xtags audio typing (best-effort, additive)
Some checks failed
gitleaks / scan (push) Failing after 1s
Some checks failed
gitleaks / scan (push) Failing after 1s
Brings the port up to current NewPipe Extractor coverage: - visitorData: fetched + attached to the anonymous reel/web/ios/visionos InnerTube contexts (upstream "always pass a valid visitorData"). Best-effort — a failure returns None and extraction proceeds exactly as before. Cached process-wide with a 15-min TTL, and invalidated when the primary android response is rejected (decoy/playability) so a bad session self-heals. - visionOS client (101): a 4th /player fetch, ON by default, recovering >360p streams for SABR-only / made-for-kids videos where android/ios cap at 360p. Formats merged android-first (android wins itag ties); a failure changes nothing. HLS + DASH manifest branches now skip empty urls (an empty visionOS url no longer shadows a real ios/android one). - xtags: derive audio track type (original/dubbed/descriptive/secondary) from the format xtags protobuf, falling back to the audioIsDefault heuristic when absent. Hand-rolled protobuf walker, fuzzed (4M inputs, no panic); 32-bit length guarded via try_from. FFI surface unchanged (AudioStream gains an additive track_type the wrapper may later surface). 155 lib + 7 integration tests, clippy -D clean.
This commit is contained in:
parent
988e67414d
commit
2452f1785d
10 changed files with 821 additions and 70 deletions
|
|
@ -66,3 +66,10 @@ impl NewPipe {
|
|||
*Self::instance().preferred_content_country.write() = content_country;
|
||||
}
|
||||
}
|
||||
|
||||
/// Serializes every test that installs/uses a Downloader in the process-global
|
||||
/// `NewPipe` singleton, so they don't clobber each other's registered
|
||||
/// downloader (or corrupt request counts) when cargo runs tests in parallel.
|
||||
/// Shared by the `player_manager` and `stream_helper` test modules.
|
||||
#[cfg(test)]
|
||||
pub(crate) static DOWNLOADER_TEST_LOCK: parking_lot::Mutex<()> = parking_lot::Mutex::new(());
|
||||
|
|
|
|||
|
|
@ -3,6 +3,21 @@
|
|||
use crate::stream::DeliveryMethod;
|
||||
use crate::youtube::itag::MediaFormat;
|
||||
|
||||
/// The track type of an [`AudioStream`]. Mirrors NPE
|
||||
/// `stream/AudioTrackType.java`; derived from the format's `xtags` protobuf
|
||||
/// (`acont` key) — see `youtube::xtags`.
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)]
|
||||
pub enum AudioTrackType {
|
||||
/// The original audio track of a video.
|
||||
Original,
|
||||
/// The original voices replaced, typically in a different language.
|
||||
Dubbed,
|
||||
/// A descriptive (audio-description) track for accessibility.
|
||||
Descriptive,
|
||||
/// A secondary track (e.g. an alternate/commentary track).
|
||||
Secondary,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct AudioStream {
|
||||
pub itag: u32,
|
||||
|
|
@ -15,6 +30,12 @@ pub struct AudioStream {
|
|||
pub audio_track_id: Option<String>,
|
||||
pub audio_track_name: Option<String>,
|
||||
pub audio_locale: Option<String>,
|
||||
/// True iff this is a descriptive (audio-description) track. Preferentially
|
||||
/// derived from `xtags` (`acont == descriptive`); falls back to the legacy
|
||||
/// `audioTrack.audioIsDefault` heuristic when `xtags` is absent/unparseable.
|
||||
pub is_descriptive: bool,
|
||||
/// The authoritative audio track type from the format's `xtags` blob, or
|
||||
/// `None` for single-track audio / when `xtags` is absent or unparseable.
|
||||
pub track_type: Option<AudioTrackType>,
|
||||
pub itag_url_format: Option<String>,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ pub mod delivery;
|
|||
pub mod subtitles;
|
||||
pub mod video;
|
||||
|
||||
pub use audio::AudioStream;
|
||||
pub use audio::{AudioStream, AudioTrackType};
|
||||
pub use delivery::DeliveryMethod;
|
||||
pub use subtitles::SubtitlesStream;
|
||||
pub use video::VideoStream;
|
||||
|
|
|
|||
|
|
@ -111,6 +111,29 @@ impl InnertubeClientRequestInfo {
|
|||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// visionOS client (client 101). Mirrors NPE
|
||||
/// `InnertubeClientRequestInfo.ofVisionOsClient()`. Same MOBILE/Apple shape
|
||||
/// as iOS but with the RealityDevice model + visionOS os fields.
|
||||
pub fn of_visionos_client() -> Self {
|
||||
Self {
|
||||
client_info: ClientInfo {
|
||||
client_name: VISIONOS_CLIENT_NAME.into(),
|
||||
client_version: VISIONOS_CLIENT_VERSION.into(),
|
||||
client_id: VISIONOS_CLIENT_ID.into(),
|
||||
client_screen: Some(WATCH_CLIENT_SCREEN.into()),
|
||||
visitor_data: None,
|
||||
},
|
||||
device_info: DeviceInfo {
|
||||
platform: Some(MOBILE_CLIENT_PLATFORM.into()),
|
||||
device_make: Some("Apple".into()),
|
||||
device_model: Some(VISIONOS_DEVICE_MODEL.into()),
|
||||
os_name: Some("visionOS".into()),
|
||||
os_version: Some(VISIONOS_OS_VERSION.into()),
|
||||
android_sdk_version: -1,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Builds the InnerTube request envelope mirroring NPE prepareJsonBuilder.
|
||||
|
|
@ -275,6 +298,30 @@ mod tests {
|
|||
assert!(client.get("androidSdkVersion").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn visionos_client_envelope_shape() {
|
||||
let info = InnertubeClientRequestInfo::of_visionos_client();
|
||||
let env = build_envelope(
|
||||
&info,
|
||||
&Localization::default(),
|
||||
&ContentCountry::default(),
|
||||
None,
|
||||
);
|
||||
let client = &env["context"]["client"];
|
||||
assert_eq!(client["clientName"], "VISIONOS");
|
||||
assert_eq!(client["clientVersion"], "1.02");
|
||||
assert_eq!(client["platform"], "MOBILE");
|
||||
assert_eq!(client["deviceMake"], "Apple");
|
||||
assert_eq!(client["deviceModel"], "RealityDevice14,1");
|
||||
assert_eq!(client["osName"], "visionOS");
|
||||
assert_eq!(client["osVersion"], "25.6.0.23O471");
|
||||
assert_eq!(client["clientScreen"], "WATCH");
|
||||
// no androidSdkVersion for an Apple client
|
||||
assert!(client.get("androidSdkVersion").is_none());
|
||||
// best-effort visitorData not set at envelope-build time
|
||||
assert!(client.get("visitorData").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn embed_url_lands_in_third_party_block() {
|
||||
let info = InnertubeClientRequestInfo::of_web_embedded_player_client();
|
||||
|
|
|
|||
|
|
@ -36,6 +36,18 @@ pub const ANDROID_CLIENT_VERSION: &str = "21.03.36";
|
|||
pub const ANDROID_SDK_VERSION: u32 = 36;
|
||||
pub const ANDROID_OS_VERSION: &str = "16";
|
||||
|
||||
// visionOS client (client 101). A best-effort 4th /player fetch that recovers
|
||||
// >360p streams for SABR-only / "made for kids" videos where the ANDROID/IOS
|
||||
// clients cap at a single 360p muxed stream. Values from NPE
|
||||
// ClientsConstants.java (commit 82b7e410, "Workaround again SABR-only
|
||||
// responses"). Carries NO poToken.
|
||||
pub const VISIONOS_CLIENT_ID: &str = "101";
|
||||
pub const VISIONOS_CLIENT_NAME: &str = "VISIONOS";
|
||||
pub const VISIONOS_CLIENT_VERSION: &str = "1.02";
|
||||
pub const VISIONOS_DEVICE_MODEL: &str = "RealityDevice14,1";
|
||||
pub const VISIONOS_OS_VERSION: &str = "25.6.0.23O471";
|
||||
pub const VISIONOS_USER_AGENT_VERSION: &str = "25_6_0";
|
||||
|
||||
// Base URLs (NPE YoutubeParsingHelper.java:91,96).
|
||||
pub const YOUTUBEI_V1_URL: &str = "https://www.youtube.com/youtubei/v1/";
|
||||
pub const YOUTUBEI_V1_GAPIS_URL: &str = "https://youtubei.googleapis.com/youtubei/v1/";
|
||||
|
|
|
|||
|
|
@ -589,7 +589,9 @@ mod tests {
|
|||
// test touches the downloader). Each uses its own PlayerManager
|
||||
// instance, so manager state never crosses tests.
|
||||
|
||||
static GLOBAL_DOWNLOADER_LOCK: Mutex<()> = Mutex::new(());
|
||||
// Shared with the stream_helper test module so downloader-touching tests
|
||||
// across both modules serialize (they all mutate the same NewPipe global).
|
||||
use crate::newpipe::DOWNLOADER_TEST_LOCK as GLOBAL_DOWNLOADER_LOCK;
|
||||
|
||||
/// Both channel endpoints wrapped in Mutex for the Downloader Sync bound.
|
||||
type FetchBlocker = (Mutex<mpsc::Sender<()>>, Mutex<mpsc::Receiver<()>>);
|
||||
|
|
|
|||
|
|
@ -20,4 +20,5 @@ pub mod potoken;
|
|||
pub mod search_extractor;
|
||||
pub mod stream_extractor;
|
||||
pub mod stream_helper;
|
||||
pub mod xtags;
|
||||
|
||||
|
|
|
|||
|
|
@ -78,6 +78,14 @@ pub fn ios_user_agent(country: &ContentCountry) -> String {
|
|||
)
|
||||
}
|
||||
|
||||
/// visionOS user-agent — mirrors NPE `getVisionOsUserAgent`.
|
||||
pub fn visionos_user_agent(country: &ContentCountry) -> String {
|
||||
format!(
|
||||
"com.google.visionos.youtube/{VISIONOS_CLIENT_VERSION}({VISIONOS_DEVICE_MODEL}; U; CPU visionOS {VISIONOS_USER_AGENT_VERSION} like Mac OS X; {})",
|
||||
country.country_code()
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
|
@ -134,4 +142,13 @@ mod tests {
|
|||
assert!(ua.contains("CPU iOS 18_7_2"));
|
||||
assert!(ua.contains("; US)"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn visionos_ua_template() {
|
||||
let ua = visionos_user_agent(&ContentCountry::new("US"));
|
||||
assert!(ua.contains("com.google.visionos.youtube/1.02"));
|
||||
assert!(ua.contains("RealityDevice14,1"));
|
||||
assert!(ua.contains("CPU visionOS 25_6_0 like Mac OS X"));
|
||||
assert!(ua.contains("; US)"));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -28,7 +28,8 @@ use crate::image::{Image, ResolutionLevel};
|
|||
use crate::localization::{ContentCountry, Localization};
|
||||
use crate::newpipe::NewPipe;
|
||||
use crate::stream::{
|
||||
AudioStream, DeliveryMethod, StreamInfo, StreamType, SubtitlesStream, VideoStream,
|
||||
AudioStream, AudioTrackType, DeliveryMethod, StreamInfo, StreamType, SubtitlesStream,
|
||||
VideoStream,
|
||||
};
|
||||
use crate::youtube::itag::{lookup as itag_lookup, ItagType};
|
||||
#[cfg(test)]
|
||||
|
|
@ -36,6 +37,7 @@ use crate::youtube::itag::MediaFormat;
|
|||
use crate::youtube::js::PlayerManager;
|
||||
use crate::youtube::potoken::{po_token_provider, PoTokenResult};
|
||||
use crate::youtube::stream_helper::{self, generate_content_playback_nonce};
|
||||
use crate::youtube::xtags;
|
||||
|
||||
/// Shared `Value::Null` sentinel so a missing `streamingData` can be handed
|
||||
/// back as a borrowed `&Value` instead of cloning the (large) present
|
||||
|
|
@ -49,9 +51,21 @@ pub enum FetchPolicy {
|
|||
AndroidWithPoToken,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default)]
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct ExtractOptions {
|
||||
pub fetch_ios_client: bool,
|
||||
/// Fetch the visionOS client (client 101) as an additional best-effort
|
||||
/// /player call. Recovers >360p streams for SABR-only / "made for kids"
|
||||
/// videos where ANDROID/IOS cap at 360p. Its formats are merged in
|
||||
/// (android-preferred on itag ties); a failure changes nothing.
|
||||
///
|
||||
/// Defaults to `true` (see `impl Default`), matching upstream NPE, which
|
||||
/// runs the visionOS fetch UNCONDITIONALLY in `onFetchPage`. The shipped app
|
||||
/// path (`stream_info` → `stream_info_with(.., ExtractOptions::default())`)
|
||||
/// therefore fetches visionOS on every open. Set to `false` per-call to skip
|
||||
/// it (e.g. a latency-sensitive path). The cheap `stream_metadata` path does
|
||||
/// not build streams and never fetches visionOS regardless.
|
||||
pub fetch_visionos_client: bool,
|
||||
pub android_streaming_pot: Option<String>,
|
||||
pub ios_streaming_pot: Option<String>,
|
||||
pub android_visitor_data: Option<String>,
|
||||
|
|
@ -60,6 +74,23 @@ pub struct ExtractOptions {
|
|||
pub ios_player_request_pot: Option<String>,
|
||||
}
|
||||
|
||||
impl Default for ExtractOptions {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
// visionOS ON by default — the real app path must fetch it on every
|
||||
// open (Cobb, Loop-2 audit), matching upstream.
|
||||
fetch_visionos_client: true,
|
||||
fetch_ios_client: false,
|
||||
android_streaming_pot: None,
|
||||
ios_streaming_pot: None,
|
||||
android_visitor_data: None,
|
||||
ios_visitor_data: None,
|
||||
android_player_request_pot: None,
|
||||
ios_player_request_pot: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One-shot StreamInfo build for a video. Walks NPE's Android-primary
|
||||
/// fetch path, applies URL post-processing, returns the final shape.
|
||||
pub fn stream_info(video_id: &str) -> Result<StreamInfo, ExtractionError> {
|
||||
|
|
@ -101,8 +132,16 @@ pub fn stream_info_with(
|
|||
android_token.as_ref().map(|t| t.visitor_data.as_str()),
|
||||
)?;
|
||||
|
||||
check_playability_status(&player_response)?;
|
||||
// A primary-android rejection (playability failure or decoy) is exactly the
|
||||
// symptom a poisoned visitorData would produce on the primary reel call —
|
||||
// and since it aborts extraction the value would otherwise never self-heal.
|
||||
// Drop the cached visitorData so the next extraction fetches a fresh one.
|
||||
if let Err(e) = check_playability_status(&player_response) {
|
||||
stream_helper::reset_visitor_data_cache();
|
||||
return Err(e);
|
||||
}
|
||||
if is_player_response_not_valid(&player_response, video_id) {
|
||||
stream_helper::reset_visitor_data_cache();
|
||||
return Err(ExtractionError::Other(
|
||||
"ANDROID player response is not valid (decoy detected)".into(),
|
||||
));
|
||||
|
|
@ -152,6 +191,31 @@ pub fn stream_info_with(
|
|||
.and_then(|r| r.get("streamingData"))
|
||||
.unwrap_or(&NULL_VALUE);
|
||||
|
||||
// Optional visionOS — best-effort, gated (default OFF; see
|
||||
// ExtractOptions::fetch_visionos_client). Recovers >360p streams for
|
||||
// SABR-only / "made for kids" videos. Carries NO poToken and (like iOS) is
|
||||
// decoy-checked before its streamingData is used. A failure leaves the
|
||||
// output exactly as it is today.
|
||||
let (visionos_response, visionos_cpn): (Option<Value>, Option<String>) =
|
||||
if options.fetch_visionos_client {
|
||||
let cpn = generate_content_playback_nonce();
|
||||
match stream_helper::get_visionos_player_response(
|
||||
video_id,
|
||||
&localization,
|
||||
&content_country,
|
||||
&cpn,
|
||||
) {
|
||||
Ok(r) if !is_player_response_not_valid(&r, video_id) => (Some(r), Some(cpn)),
|
||||
_ => (None, None),
|
||||
}
|
||||
} else {
|
||||
(None, None)
|
||||
};
|
||||
let visionos_streaming_data: &Value = visionos_response
|
||||
.as_ref()
|
||||
.and_then(|r| r.get("streamingData"))
|
||||
.unwrap_or(&NULL_VALUE);
|
||||
|
||||
let android_streaming_pot = android_token
|
||||
.as_ref()
|
||||
.map(|t| t.streaming_data_po_token.clone())
|
||||
|
|
@ -176,19 +240,37 @@ pub fn stream_info_with(
|
|||
|
||||
populate_video_details(&mut info, &player_response);
|
||||
populate_microformat(&mut info, &web_metadata);
|
||||
populate_streams(
|
||||
&mut info,
|
||||
android_streaming_data,
|
||||
ios_streaming_data,
|
||||
video_id,
|
||||
&android_cpn,
|
||||
ios_cpn.as_deref(),
|
||||
android_streaming_pot.as_deref(),
|
||||
ios_streaming_pot.as_deref(),
|
||||
)?;
|
||||
|
||||
// Merge order mirrors NPE getItags: ANDROID, then VISIONOS, then IOS. The
|
||||
// per-(itag,delivery) dedup is first-wins, so android formats win ties and
|
||||
// visionOS/iOS only ADD formats android lacked (e.g. the >360p renditions
|
||||
// for SABR-only videos). visionOS carries no poToken.
|
||||
let sources = [
|
||||
FormatSource {
|
||||
streaming_data: android_streaming_data,
|
||||
client: "ANDROID",
|
||||
cpn: &android_cpn,
|
||||
pot: android_streaming_pot.as_deref(),
|
||||
},
|
||||
FormatSource {
|
||||
streaming_data: visionos_streaming_data,
|
||||
client: "VISIONOS",
|
||||
cpn: visionos_cpn.as_deref().unwrap_or(""),
|
||||
pot: None,
|
||||
},
|
||||
FormatSource {
|
||||
streaming_data: ios_streaming_data,
|
||||
client: "IOS",
|
||||
cpn: ios_cpn.as_deref().unwrap_or(""),
|
||||
pot: ios_streaming_pot.as_deref(),
|
||||
},
|
||||
];
|
||||
|
||||
populate_streams(&mut info, &sources, video_id)?;
|
||||
populate_manifests(
|
||||
&mut info,
|
||||
android_streaming_data,
|
||||
visionos_streaming_data,
|
||||
ios_streaming_data,
|
||||
android_streaming_pot.as_deref(),
|
||||
ios_streaming_pot.as_deref(),
|
||||
|
|
@ -242,8 +324,16 @@ pub fn stream_metadata(video_id: &str) -> Result<StreamInfo, ExtractionError> {
|
|||
android_token.as_ref().map(|t| t.visitor_data.as_str()),
|
||||
)?;
|
||||
|
||||
check_playability_status(&player_response)?;
|
||||
// A primary-android rejection (playability failure or decoy) is exactly the
|
||||
// symptom a poisoned visitorData would produce on the primary reel call —
|
||||
// and since it aborts extraction the value would otherwise never self-heal.
|
||||
// Drop the cached visitorData so the next extraction fetches a fresh one.
|
||||
if let Err(e) = check_playability_status(&player_response) {
|
||||
stream_helper::reset_visitor_data_cache();
|
||||
return Err(e);
|
||||
}
|
||||
if is_player_response_not_valid(&player_response, video_id) {
|
||||
stream_helper::reset_visitor_data_cache();
|
||||
return Err(ExtractionError::Other(
|
||||
"ANDROID player response is not valid (decoy detected)".into(),
|
||||
));
|
||||
|
|
@ -461,66 +551,50 @@ fn populate_microformat(info: &mut StreamInfo, web_metadata: &Value) {
|
|||
}
|
||||
}
|
||||
|
||||
/// Merge the Android + iOS format objects under one streamingData array
|
||||
/// key (`formats` / `adaptiveFormats`), each tagged with its client + cpn +
|
||||
/// pot. Returns borrowed `&Value`s into the player responses: the format
|
||||
/// objects are only ever read (`.get(...)`) downstream, so cloning the whole
|
||||
/// array (~20-40 objects per video) was pure waste.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
/// One client's streamingData plus the tags every format from it carries.
|
||||
/// Holds borrowed `&Value`s into the (function-lived) player responses.
|
||||
struct FormatSource<'a> {
|
||||
streaming_data: &'a Value,
|
||||
client: &'static str,
|
||||
cpn: &'a str,
|
||||
pot: Option<&'a str>,
|
||||
}
|
||||
|
||||
/// Merge the per-client format objects under one streamingData array key
|
||||
/// (`formats` / `adaptiveFormats`), each tagged with its client + cpn + pot,
|
||||
/// in source order (ANDROID, VISIONOS, IOS). Returns borrowed `&Value`s into
|
||||
/// the player responses: the format objects are only ever read (`.get(...)`)
|
||||
/// downstream, so cloning the whole array (~20-40 objects per video) was pure
|
||||
/// waste.
|
||||
fn merge_formats<'a>(
|
||||
android: &'a Value,
|
||||
ios: &'a Value,
|
||||
sources: &[FormatSource<'a>],
|
||||
fmt_array_key: &str,
|
||||
android_cpn: &'a str,
|
||||
ios_cpn: Option<&'a str>,
|
||||
android_pot: Option<&'a str>,
|
||||
ios_pot: Option<&'a str>,
|
||||
) -> Vec<(&'a Value, &'static str, &'a str, Option<&'a str>)> {
|
||||
let mut out = Vec::new();
|
||||
if let Some(arr) = android.get(fmt_array_key).and_then(|v| v.as_array()) {
|
||||
for f in arr {
|
||||
out.push((f, "ANDROID", android_cpn, android_pot));
|
||||
}
|
||||
}
|
||||
if let Some(arr) = ios.get(fmt_array_key).and_then(|v| v.as_array()) {
|
||||
let cpn = ios_cpn.unwrap_or("");
|
||||
for f in arr {
|
||||
out.push((f, "IOS", cpn, ios_pot));
|
||||
for src in sources {
|
||||
if let Some(arr) = src.streaming_data.get(fmt_array_key).and_then(|v| v.as_array()) {
|
||||
for f in arr {
|
||||
out.push((f, src.client, src.cpn, src.pot));
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn populate_streams(
|
||||
info: &mut StreamInfo,
|
||||
android: &Value,
|
||||
ios: &Value,
|
||||
sources: &[FormatSource],
|
||||
video_id: &str,
|
||||
android_cpn: &str,
|
||||
ios_cpn: Option<&str>,
|
||||
android_pot: Option<&str>,
|
||||
ios_pot: Option<&str>,
|
||||
) -> Result<(), ExtractionError> {
|
||||
// Progressive: streamingData.formats[]
|
||||
for (fmt, _client, cpn, pot) in
|
||||
merge_formats(android, ios, "formats", android_cpn, ios_cpn, android_pot, ios_pot)
|
||||
{
|
||||
for (fmt, _client, cpn, pot) in merge_formats(sources, "formats") {
|
||||
if let Some(stream) = build_video_progressive(fmt, video_id, cpn, pot)? {
|
||||
push_video_dedup(&mut info.video_streams, stream);
|
||||
}
|
||||
}
|
||||
|
||||
// Adaptive: streamingData.adaptiveFormats[]
|
||||
for (fmt, _client, cpn, pot) in merge_formats(
|
||||
android,
|
||||
ios,
|
||||
"adaptiveFormats",
|
||||
android_cpn,
|
||||
ios_cpn,
|
||||
android_pot,
|
||||
ios_pot,
|
||||
) {
|
||||
for (fmt, _client, cpn, pot) in merge_formats(sources, "adaptiveFormats") {
|
||||
let mime = fmt
|
||||
.get("mimeType")
|
||||
.and_then(|v| v.as_str())
|
||||
|
|
@ -541,18 +615,43 @@ fn populate_streams(
|
|||
fn populate_manifests(
|
||||
info: &mut StreamInfo,
|
||||
android: &Value,
|
||||
visionos: &Value,
|
||||
ios: &Value,
|
||||
android_pot: Option<&str>,
|
||||
ios_pot: Option<&str>,
|
||||
) {
|
||||
// NPE getManifestUrl skips null-OR-EMPTY manifest URLs; the `!is_empty`
|
||||
// filters matter because an EMPTY visionOS `hlsManifestUrl` would otherwise
|
||||
// shadow a real iOS/Android one (`Some("")`), and the wrapper drops
|
||||
// `Some("")` → the app loses HLS entirely.
|
||||
// DASH is Android-only.
|
||||
if let Some(url) = android.get("dashManifestUrl").and_then(|v| v.as_str()) {
|
||||
if let Some(url) = android
|
||||
.get("dashManifestUrl")
|
||||
.and_then(|v| v.as_str())
|
||||
.filter(|s| !s.is_empty())
|
||||
{
|
||||
info.dash_manifest_url = Some(append_pot_to_manifest(url, android_pot));
|
||||
}
|
||||
// HLS prefers iOS, falls back to Android.
|
||||
if let Some(url) = ios.get("hlsManifestUrl").and_then(|v| v.as_str()) {
|
||||
// HLS prefers an Apple client — visionOS, then iOS — because their HLS
|
||||
// manifests carry separated audio/video for livestreams (NPE getHlsUrl:
|
||||
// visionOS → iOS → android). visionOS carries no poToken.
|
||||
if let Some(url) = visionos
|
||||
.get("hlsManifestUrl")
|
||||
.and_then(|v| v.as_str())
|
||||
.filter(|s| !s.is_empty())
|
||||
{
|
||||
info.hls_manifest_url = Some(append_pot_to_manifest(url, None));
|
||||
} else if let Some(url) = ios
|
||||
.get("hlsManifestUrl")
|
||||
.and_then(|v| v.as_str())
|
||||
.filter(|s| !s.is_empty())
|
||||
{
|
||||
info.hls_manifest_url = Some(append_pot_to_manifest(url, ios_pot));
|
||||
} else if let Some(url) = android.get("hlsManifestUrl").and_then(|v| v.as_str()) {
|
||||
} else if let Some(url) = android
|
||||
.get("hlsManifestUrl")
|
||||
.and_then(|v| v.as_str())
|
||||
.filter(|s| !s.is_empty())
|
||||
{
|
||||
info.hls_manifest_url = Some(append_pot_to_manifest(url, android_pot));
|
||||
}
|
||||
}
|
||||
|
|
@ -759,6 +858,35 @@ fn build_audio(
|
|||
return Ok(None);
|
||||
};
|
||||
let audio_track = fmt.get("audioTrack");
|
||||
let has_track = audio_track
|
||||
.and_then(|t| t.get("id"))
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| !s.is_empty())
|
||||
.unwrap_or(false);
|
||||
|
||||
// audioTrackType from the format's `xtags` protobuf (NPE b0bca7e7/e3479a7c).
|
||||
// Only meaningful for a real alternate track (matches NPE, which sets it
|
||||
// only inside `if (!isNullOrEmpty(audioTrackId))`).
|
||||
let track_type = if has_track {
|
||||
fmt.get("xtags")
|
||||
.and_then(|v| v.as_str())
|
||||
.and_then(xtags::extract_audio_track_type)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
// `is_descriptive`: prefer the authoritative xtags-derived type; fall back
|
||||
// to the legacy `audioTrack.audioIsDefault` heuristic when xtags is
|
||||
// absent/unparseable (track_type == None).
|
||||
let is_descriptive = match track_type {
|
||||
Some(t) => t == AudioTrackType::Descriptive,
|
||||
None => audio_track
|
||||
.and_then(|t| t.get("audioIsDefault"))
|
||||
.and_then(|v| v.as_bool())
|
||||
.map(|b| !b)
|
||||
.unwrap_or(false),
|
||||
};
|
||||
|
||||
Ok(Some(AudioStream {
|
||||
itag: itag.id,
|
||||
url,
|
||||
|
|
@ -787,11 +915,8 @@ fn build_audio(
|
|||
.and_then(|v| v.as_str())
|
||||
.and_then(|s| s.split('.').next())
|
||||
.map(String::from),
|
||||
is_descriptive: audio_track
|
||||
.and_then(|t| t.get("audioIsDefault"))
|
||||
.and_then(|v| v.as_bool())
|
||||
.map(|b| !b)
|
||||
.unwrap_or(false),
|
||||
is_descriptive,
|
||||
track_type,
|
||||
itag_url_format: None,
|
||||
}))
|
||||
}
|
||||
|
|
@ -955,4 +1080,180 @@ mod tests {
|
|||
push_video_dedup(&mut list, s2);
|
||||
assert_eq!(list.len(), 2);
|
||||
}
|
||||
|
||||
// ---- I-1 visionOS merge ------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn absent_visionos_source_contributes_nothing() {
|
||||
// Best-effort proof: a Null visionOS streamingData (fetch failed or the
|
||||
// client disabled) adds ZERO formats — the merged set is android+ios
|
||||
// only, exactly today's behavior.
|
||||
let android = json!({ "formats": [ {"itag": 18} ] });
|
||||
let ios = json!({ "formats": [ {"itag": 22} ] });
|
||||
let null = Value::Null;
|
||||
let sources = [
|
||||
FormatSource { streaming_data: &android, client: "ANDROID", cpn: "a", pot: None },
|
||||
FormatSource { streaming_data: &null, client: "VISIONOS", cpn: "", pot: None },
|
||||
FormatSource { streaming_data: &ios, client: "IOS", cpn: "i", pot: None },
|
||||
];
|
||||
let merged = merge_formats(&sources, "formats");
|
||||
let itags: Vec<u64> = merged
|
||||
.iter()
|
||||
.map(|(f, _, _, _)| f["itag"].as_u64().unwrap())
|
||||
.collect();
|
||||
assert_eq!(itags, vec![18, 22]);
|
||||
assert!(merged.iter().all(|(_, client, _, _)| *client != "VISIONOS"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn populate_streams_merges_visionos_extra_itags_android_wins_ties() {
|
||||
// android has only itag 18 (360p); visionOS returns a dup 18 plus an
|
||||
// extra 22 (720p) android lacked — the SABR-only coverage recovery.
|
||||
let android = json!({ "formats": [
|
||||
{"itag": 18, "url": "https://r/vp?a=1", "mimeType": "video/mp4; codecs=\"avc1.42\""}
|
||||
]});
|
||||
let visionos = json!({ "formats": [
|
||||
{"itag": 18, "url": "https://r/vp?v=1", "mimeType": "video/mp4"},
|
||||
{"itag": 22, "url": "https://r/vp?v=2", "mimeType": "video/mp4"}
|
||||
]});
|
||||
let null = Value::Null;
|
||||
let sources = [
|
||||
FormatSource { streaming_data: &android, client: "ANDROID", cpn: "acpn", pot: None },
|
||||
FormatSource { streaming_data: &visionos, client: "VISIONOS", cpn: "vcpn", pot: None },
|
||||
FormatSource { streaming_data: &null, client: "IOS", cpn: "", pot: None },
|
||||
];
|
||||
let mut info = StreamInfo::default();
|
||||
populate_streams(&mut info, &sources, "vid").unwrap();
|
||||
let mut tags: Vec<u32> = info.video_streams.iter().map(|s| s.itag).collect();
|
||||
tags.sort_unstable();
|
||||
assert_eq!(tags, vec![18, 22]); // dup 18 deduped, 22 recovered
|
||||
// itag 18 kept from ANDROID (first-wins): its URL carries android's cpn.
|
||||
let s18 = info.video_streams.iter().find(|s| s.itag == 18).unwrap();
|
||||
assert!(s18.url.contains("a=1"), "android's itag-18 must win: {}", s18.url);
|
||||
assert!(s18.url.contains("cpn=acpn"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hls_prefers_visionos_then_ios_then_android() {
|
||||
let android = json!({ "hlsManifestUrl": "https://a/hls" });
|
||||
let visionos = json!({ "hlsManifestUrl": "https://v/hls" });
|
||||
let ios = json!({ "hlsManifestUrl": "https://i/hls" });
|
||||
let null = Value::Null;
|
||||
|
||||
let mut info = StreamInfo::default();
|
||||
populate_manifests(&mut info, &android, &visionos, &ios, None, None);
|
||||
assert_eq!(info.hls_manifest_url.as_deref(), Some("https://v/hls"));
|
||||
|
||||
// visionOS absent → iOS wins.
|
||||
let mut info = StreamInfo::default();
|
||||
populate_manifests(&mut info, &android, &null, &ios, None, None);
|
||||
assert_eq!(info.hls_manifest_url.as_deref(), Some("https://i/hls"));
|
||||
|
||||
// Only android → android (unchanged from today).
|
||||
let mut info = StreamInfo::default();
|
||||
populate_manifests(&mut info, &android, &null, &null, None, None);
|
||||
assert_eq!(info.hls_manifest_url.as_deref(), Some("https://a/hls"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_hls_url_is_skipped_not_shadowing_a_real_one() {
|
||||
// Fix 3: an EMPTY visionOS hlsManifestUrl must NOT shadow iOS's real one
|
||||
// (else the wrapper drops Some("") → the app loses HLS).
|
||||
let android = json!({ "hlsManifestUrl": "https://a/hls" });
|
||||
let visionos_empty = json!({ "hlsManifestUrl": "" });
|
||||
let ios = json!({ "hlsManifestUrl": "https://i/hls" });
|
||||
|
||||
let mut info = StreamInfo::default();
|
||||
populate_manifests(&mut info, &android, &visionos_empty, &ios, None, None);
|
||||
assert_eq!(info.hls_manifest_url.as_deref(), Some("https://i/hls"));
|
||||
|
||||
// Empty visionOS AND empty iOS → falls through to android.
|
||||
let ios_empty = json!({ "hlsManifestUrl": "" });
|
||||
let mut info = StreamInfo::default();
|
||||
populate_manifests(&mut info, &android, &visionos_empty, &ios_empty, None, None);
|
||||
assert_eq!(info.hls_manifest_url.as_deref(), Some("https://a/hls"));
|
||||
|
||||
// All empty → None (not Some("")).
|
||||
let android_empty = json!({ "hlsManifestUrl": "", "dashManifestUrl": "" });
|
||||
let mut info = StreamInfo::default();
|
||||
populate_manifests(&mut info, &android_empty, &visionos_empty, &ios_empty, None, None);
|
||||
assert!(info.hls_manifest_url.is_none());
|
||||
assert!(info.dash_manifest_url.is_none()); // empty DASH also skipped
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn default_options_fetch_visionos() {
|
||||
// Fix 1: the shipped app path (stream_info → ExtractOptions::default())
|
||||
// must fetch visionOS on every open.
|
||||
assert!(ExtractOptions::default().fetch_visionos_client);
|
||||
}
|
||||
|
||||
// ---- I-3 xtags audioTrackType -----------------------------------------
|
||||
|
||||
// itag 140 (M4A audio) + a direct url with no `n=` param, so process_url
|
||||
// never touches the network (nsig quick-exits).
|
||||
fn audio_fmt(xtags: Option<&str>, audio_is_default: bool) -> Value {
|
||||
let mut fmt = json!({
|
||||
"itag": 140,
|
||||
"url": "https://r.googlevideo.com/videoplayback?foo=1",
|
||||
"mimeType": "audio/mp4; codecs=\"mp4a.40.2\"",
|
||||
"audioTrack": { "id": "en.4", "displayName": "English", "audioIsDefault": audio_is_default }
|
||||
});
|
||||
if let Some(x) = xtags {
|
||||
fmt["xtags"] = json!(x);
|
||||
}
|
||||
fmt
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn xtags_descriptive_sets_track_type_and_is_descriptive() {
|
||||
let fmt = audio_fmt(Some("ChQKBWFjb250EgtkZXNjcmlwdGl2ZQ"), true);
|
||||
let a = build_audio(&fmt, "vid", "cpn", None).unwrap().unwrap();
|
||||
assert_eq!(a.track_type, Some(AudioTrackType::Descriptive));
|
||||
assert!(a.is_descriptive);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn xtags_overrides_audio_is_default_heuristic() {
|
||||
// acont=dubbed → Dubbed + is_descriptive false, even though
|
||||
// audioIsDefault=false would make the legacy heuristic say "descriptive".
|
||||
let fmt = audio_fmt(Some("Cg8KBWFjb250EgZkdWJiZWQ"), false);
|
||||
let a = build_audio(&fmt, "vid", "cpn", None).unwrap().unwrap();
|
||||
assert_eq!(a.track_type, Some(AudioTrackType::Dubbed));
|
||||
assert!(!a.is_descriptive);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn xtags_original_maps_to_original() {
|
||||
let fmt = audio_fmt(Some("ChEKBWFjb250EghvcmlnaW5hbA"), true);
|
||||
let a = build_audio(&fmt, "vid", "cpn", None).unwrap().unwrap();
|
||||
assert_eq!(a.track_type, Some(AudioTrackType::Original));
|
||||
assert!(!a.is_descriptive);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn absent_xtags_falls_back_to_audio_is_default_heuristic() {
|
||||
// No xtags → track_type None → is_descriptive == !audioIsDefault.
|
||||
let a = build_audio(&audio_fmt(None, false), "vid", "cpn", None)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(a.track_type, None);
|
||||
assert!(a.is_descriptive); // !false
|
||||
|
||||
let a = build_audio(&audio_fmt(None, true), "vid", "cpn", None)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(a.track_type, None);
|
||||
assert!(!a.is_descriptive); // !true
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unparseable_xtags_falls_back_to_heuristic() {
|
||||
// Garbage xtags → extract_audio_track_type None → heuristic used.
|
||||
let a = build_audio(&audio_fmt(Some("!!!not-b64!!!"), false), "vid", "cpn", None)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(a.track_type, None);
|
||||
assert!(a.is_descriptive);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,6 +6,8 @@
|
|||
// serviceIntegrityDimensions for poToken), POSTs to the right URL with
|
||||
// the right headers, returns the parsed JSON.
|
||||
|
||||
use once_cell::sync::Lazy;
|
||||
use parking_lot::RwLock;
|
||||
use serde_json::{json, Map, Value};
|
||||
|
||||
use crate::downloader::request::Request;
|
||||
|
|
@ -15,7 +17,8 @@ use crate::newpipe::NewPipe;
|
|||
use crate::youtube::client_request::{build_envelope, InnertubeClientRequestInfo};
|
||||
use crate::youtube::constants::*;
|
||||
use crate::youtube::parsing::{
|
||||
android_user_agent, ios_user_agent, mobile_post_headers, youtube_post_headers,
|
||||
android_user_agent, ios_user_agent, mobile_post_headers, visionos_user_agent,
|
||||
youtube_post_headers,
|
||||
};
|
||||
|
||||
/// Builds a 12-char alphanumeric `cpn` (content playback nonce). NPE uses
|
||||
|
|
@ -72,6 +75,145 @@ fn envelope_to_body(envelope: Value) -> Map<String, Value> {
|
|||
}
|
||||
}
|
||||
|
||||
/// A cached `visitorData` plus the instant it was fetched (for TTL expiry).
|
||||
struct CachedVisitorData {
|
||||
value: String,
|
||||
at: std::time::Instant,
|
||||
}
|
||||
|
||||
impl CachedVisitorData {
|
||||
fn is_fresh(&self) -> bool {
|
||||
self.at.elapsed() < VISITOR_DATA_TTL
|
||||
}
|
||||
}
|
||||
|
||||
/// How long a cached `visitorData` is reused before a fresh one is fetched.
|
||||
/// Upstream fetches a brand-new visitorData on EVERY call; we cache to avoid a
|
||||
/// round-trip per video, but bound the reuse so (a) traffic isn't pinned to a
|
||||
/// single visitor identity indefinitely (rate-flag risk) and (b) a
|
||||
/// subtly-bad-but-non-empty value can't ride the PRIMARY android reel call —
|
||||
/// whose failure aborts extraction — forever. 15 min is a small fraction of
|
||||
/// YouTube's own visitorData lifetime while still rotating regularly.
|
||||
const VISITOR_DATA_TTL: std::time::Duration = std::time::Duration::from_secs(15 * 60);
|
||||
|
||||
/// Process-wide cached `visitorData`. See [`CachedVisitorData`] / TTL above.
|
||||
static VISITOR_DATA_CACHE: Lazy<RwLock<Option<CachedVisitorData>>> =
|
||||
Lazy::new(|| RwLock::new(None));
|
||||
|
||||
/// Best-effort `visitorData` for the InnerTube client context.
|
||||
///
|
||||
/// Upstream (`YoutubeStreamHelper`) attaches a freshly-fetched visitorData to
|
||||
/// every non-token reel/web/ios/visionos call — *"We must always pass a valid
|
||||
/// visitorData to get valid player responses"* — via
|
||||
/// `YoutubeParsingHelper.getVisitorDataFromInnertube` (POST to the
|
||||
/// `visitor_id` endpoint, read `responseContext.visitorData`).
|
||||
///
|
||||
/// DELIBERATE DEVIATION (Loop-2 best-effort mandate): upstream THROWS when
|
||||
/// visitorData can't be obtained, which would fail the whole extraction.
|
||||
/// strawcore's anonymous happy-path has always worked WITHOUT visitorData, so
|
||||
/// any failure here returns `None` and the caller proceeds exactly as it does
|
||||
/// today. This is a durability improvement that can never regress the current
|
||||
/// path — sending visitorData when we can get it, silently skipping it when we
|
||||
/// can't.
|
||||
///
|
||||
/// The cache is bounded by [`VISITOR_DATA_TTL`] and can be dropped early via
|
||||
/// [`reset_visitor_data_cache`] (called when the PRIMARY android response is
|
||||
/// rejected, so a poisoned value self-heals). Locking mirrors the Loop-1
|
||||
/// `player_manager` discipline: the `RwLock` is NEVER held across the network.
|
||||
/// read (fresh?) → (miss/expired) fetch unlocked → write. A rare concurrent
|
||||
/// double-fetch simply stores one of two interchangeable values.
|
||||
fn get_visitor_data(
|
||||
info: &InnertubeClientRequestInfo,
|
||||
localization: &Localization,
|
||||
content_country: &ContentCountry,
|
||||
headers: Vec<(String, String)>,
|
||||
domain: &str,
|
||||
) -> Option<String> {
|
||||
// Serve a still-fresh cached value; an expired one falls through to refetch.
|
||||
if let Some(cached) = VISITOR_DATA_CACHE.read().as_ref() {
|
||||
if cached.is_fresh() && !visitor_force_stale() {
|
||||
return Some(cached.value.clone());
|
||||
}
|
||||
}
|
||||
let env = build_envelope(info, localization, content_country, None);
|
||||
let body = Value::Object(envelope_to_body(env));
|
||||
let url = format!("{domain}visitor_id{DISABLE_PRETTY_PRINT_PARAM}");
|
||||
// post_youtube returns Err on any non-200 / transport / parse failure →
|
||||
// best-effort None (the caller then proceeds without visitorData). An
|
||||
// expired-but-present cache entry is deliberately NOT served on a failed
|
||||
// refetch: a stale visitorData is the thing we're rotating away from.
|
||||
let parsed = post_youtube(&url, &body, headers).ok()?;
|
||||
let visitor = visitor_data_from_response(&parsed)?;
|
||||
*VISITOR_DATA_CACHE.write() = Some(CachedVisitorData {
|
||||
value: visitor.clone(),
|
||||
at: std::time::Instant::now(),
|
||||
});
|
||||
Some(visitor)
|
||||
}
|
||||
|
||||
/// Drop the cached `visitorData` so the next extraction fetches a fresh one.
|
||||
/// Called when the PRIMARY android response is rejected (decoy / playability
|
||||
/// failure): those are exactly the symptoms a bad visitorData would produce on
|
||||
/// the primary path, and since that failure aborts extraction the value would
|
||||
/// otherwise never self-heal.
|
||||
pub(crate) fn reset_visitor_data_cache() {
|
||||
*VISITOR_DATA_CACHE.write() = None;
|
||||
}
|
||||
|
||||
/// Pull `responseContext.visitorData` out of a visitor_id response, treating an
|
||||
/// absent or empty value as failure (`None`) — the trigger for the best-effort
|
||||
/// fallback to "no visitorData" (today's behavior).
|
||||
fn visitor_data_from_response(parsed: &Value) -> Option<String> {
|
||||
parsed
|
||||
.get("responseContext")
|
||||
.and_then(|v| v.get("visitorData"))
|
||||
.and_then(|v| v.as_str())
|
||||
.filter(|s| !s.is_empty())
|
||||
.map(str::to_string)
|
||||
}
|
||||
|
||||
/// Test-only override that forces a cached entry to be treated as stale,
|
||||
/// exercising the TTL-expiry refetch branch deterministically (fabricating an
|
||||
/// old `Instant` via `checked_sub` is process/host-uptime dependent). Compiled
|
||||
/// out entirely in non-test builds.
|
||||
#[cfg(test)]
|
||||
static FORCE_VISITOR_STALE: std::sync::atomic::AtomicBool =
|
||||
std::sync::atomic::AtomicBool::new(false);
|
||||
|
||||
#[inline]
|
||||
fn visitor_force_stale() -> bool {
|
||||
#[cfg(test)]
|
||||
{
|
||||
FORCE_VISITOR_STALE.load(std::sync::atomic::Ordering::Relaxed)
|
||||
}
|
||||
#[cfg(not(test))]
|
||||
{
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
/// Test hook: pre-seed the visitorData cache with a fresh entry (at = now) so
|
||||
/// the cache-hit read path can be exercised without a network round-trip.
|
||||
#[cfg(test)]
|
||||
fn seed_visitor_data_cache(v: &str) {
|
||||
*VISITOR_DATA_CACHE.write() = Some(CachedVisitorData {
|
||||
value: v.to_string(),
|
||||
at: std::time::Instant::now(),
|
||||
});
|
||||
}
|
||||
|
||||
/// Test hook: read the current cached visitorData value (if any).
|
||||
#[cfg(test)]
|
||||
fn peek_visitor_data_cache() -> Option<String> {
|
||||
VISITOR_DATA_CACHE.read().as_ref().map(|c| c.value.clone())
|
||||
}
|
||||
|
||||
/// Test hook: force/clear the stale override (see [`FORCE_VISITOR_STALE`]).
|
||||
#[cfg(test)]
|
||||
fn set_force_visitor_stale(on: bool) {
|
||||
FORCE_VISITOR_STALE.store(on, std::sync::atomic::Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// WEB-client metadata-only /player call — used for microformat +
|
||||
/// thumbnails only; never used as a stream URL source.
|
||||
pub fn get_web_metadata_player_response(
|
||||
|
|
@ -80,7 +222,18 @@ pub fn get_web_metadata_player_response(
|
|||
content_country: &ContentCountry,
|
||||
signature_timestamp: i32,
|
||||
) -> Result<Value, ExtractionError> {
|
||||
let info = InnertubeClientRequestInfo::of_web_client();
|
||||
let mut info = InnertubeClientRequestInfo::of_web_client();
|
||||
// Best-effort visitorData (upstream sends it on every non-token /player
|
||||
// call). WEB uses the youtube.com domain + web headers for visitor_id.
|
||||
if let Some(v) = get_visitor_data(
|
||||
&info,
|
||||
localization,
|
||||
content_country,
|
||||
youtube_post_headers(),
|
||||
YOUTUBEI_V1_URL,
|
||||
) {
|
||||
info.client_info.visitor_data = Some(v);
|
||||
}
|
||||
let env = build_envelope(&info, localization, content_country, None);
|
||||
let mut body = envelope_to_body(env);
|
||||
add_player_body_fields(&mut body, video_id, &generate_content_playback_nonce());
|
||||
|
|
@ -130,7 +283,22 @@ pub fn get_android_reel_player_response(
|
|||
content_country: &ContentCountry,
|
||||
cpn: &str,
|
||||
) -> Result<Value, ExtractionError> {
|
||||
let info = InnertubeClientRequestInfo::of_android_client();
|
||||
let mut info = InnertubeClientRequestInfo::of_android_client();
|
||||
let ua = android_user_agent(content_country);
|
||||
// Best-effort visitorData (upstream sends it on the reel call too). The reel
|
||||
// path uses the gapis domain + android mobile headers for visitor_id.
|
||||
// NOTE: the reel body shape is otherwise UNCHANGED from today (the upstream
|
||||
// playerRequest/disablePlayerResponse realignment — recon W-1 — is out of
|
||||
// scope for this pass; only touch it if a decoy/empty regression appears).
|
||||
if let Some(v) = get_visitor_data(
|
||||
&info,
|
||||
localization,
|
||||
content_country,
|
||||
mobile_post_headers(&ua),
|
||||
YOUTUBEI_V1_GAPIS_URL,
|
||||
) {
|
||||
info.client_info.visitor_data = Some(v);
|
||||
}
|
||||
let env = build_envelope(&info, localization, content_country, None);
|
||||
let mut body = envelope_to_body(env);
|
||||
body.insert(
|
||||
|
|
@ -145,7 +313,6 @@ pub fn get_android_reel_player_response(
|
|||
"{YOUTUBEI_V1_GAPIS_URL}reel/reel_item_watch{DISABLE_PRETTY_PRINT_PARAM}&t={t}&id={video_id}&$fields=playerResponse",
|
||||
t = generate_content_playback_nonce()
|
||||
);
|
||||
let ua = android_user_agent(content_country);
|
||||
post_youtube(&url, &Value::Object(body), mobile_post_headers(&ua))
|
||||
}
|
||||
|
||||
|
|
@ -163,8 +330,20 @@ pub fn get_ios_player_response(
|
|||
visitor_data: Option<&str>,
|
||||
) -> Result<Value, ExtractionError> {
|
||||
let mut info = InnertubeClientRequestInfo::of_ios_client();
|
||||
let ua = ios_user_agent(content_country);
|
||||
// Caller-supplied visitorData (paired with a poToken) wins; otherwise fetch
|
||||
// it best-effort. iOS uses the youtube.com domain for visitor_id (upstream
|
||||
// getIosPlayerResponse) even though the /player call goes to gapis.
|
||||
if let Some(v) = visitor_data {
|
||||
info.client_info.visitor_data = Some(v.into());
|
||||
} else if let Some(v) = get_visitor_data(
|
||||
&info,
|
||||
localization,
|
||||
content_country,
|
||||
mobile_post_headers(&ua),
|
||||
YOUTUBEI_V1_URL,
|
||||
) {
|
||||
info.client_info.visitor_data = Some(v);
|
||||
}
|
||||
let env = build_envelope(&info, localization, content_country, None);
|
||||
let mut body = envelope_to_body(env);
|
||||
|
|
@ -176,7 +355,42 @@ pub fn get_ios_player_response(
|
|||
"{YOUTUBEI_V1_GAPIS_URL}player{DISABLE_PRETTY_PRINT_PARAM}&t={t}&id={video_id}",
|
||||
t = generate_content_playback_nonce()
|
||||
);
|
||||
let ua = ios_user_agent(content_country);
|
||||
post_youtube(&url, &Value::Object(body), mobile_post_headers(&ua))
|
||||
}
|
||||
|
||||
/// visionOS /player call (client 101). Best-effort 4th client that recovers
|
||||
/// higher-than-360p streams for SABR-only / "made for kids" videos. Mirrors
|
||||
/// NPE `getVisionOsPlayerResponse`: hits the gapis /player endpoint with the
|
||||
/// visionOS mobile header set, carries a best-effort visitorData, and NO
|
||||
/// poToken.
|
||||
pub fn get_visionos_player_response(
|
||||
video_id: &str,
|
||||
localization: &Localization,
|
||||
content_country: &ContentCountry,
|
||||
cpn: &str,
|
||||
) -> Result<Value, ExtractionError> {
|
||||
let mut info = InnertubeClientRequestInfo::of_visionos_client();
|
||||
let ua = visionos_user_agent(content_country);
|
||||
// visionOS returns valid responses ONLY with a visitorData — but still
|
||||
// best-effort: on failure we proceed without it (the whole visionOS fetch
|
||||
// is itself swallowed best-effort by the caller). Uses the youtube.com
|
||||
// domain for visitor_id (upstream), gapis for /player.
|
||||
if let Some(v) = get_visitor_data(
|
||||
&info,
|
||||
localization,
|
||||
content_country,
|
||||
mobile_post_headers(&ua),
|
||||
YOUTUBEI_V1_URL,
|
||||
) {
|
||||
info.client_info.visitor_data = Some(v);
|
||||
}
|
||||
let env = build_envelope(&info, localization, content_country, None);
|
||||
let mut body = envelope_to_body(env);
|
||||
add_player_body_fields(&mut body, video_id, cpn);
|
||||
let url = format!(
|
||||
"{YOUTUBEI_V1_GAPIS_URL}player{DISABLE_PRETTY_PRINT_PARAM}&t={t}&id={video_id}",
|
||||
t = generate_content_playback_nonce()
|
||||
);
|
||||
post_youtube(&url, &Value::Object(body), mobile_post_headers(&ua))
|
||||
}
|
||||
|
||||
|
|
@ -231,4 +445,133 @@ mod tests {
|
|||
let b = generate_content_playback_nonce();
|
||||
assert_ne!(a, b);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn visitor_data_parsed_from_response() {
|
||||
let resp = json!({
|
||||
"responseContext": { "visitorData": "Cgs1ZG1abc==" },
|
||||
"other": 1
|
||||
});
|
||||
assert_eq!(
|
||||
visitor_data_from_response(&resp).as_deref(),
|
||||
Some("Cgs1ZG1abc==")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn visitor_data_absent_or_empty_is_none_triggering_fallback() {
|
||||
// Missing responseContext → None (→ caller proceeds without visitorData,
|
||||
// i.e. exactly today's behavior).
|
||||
assert!(visitor_data_from_response(&json!({})).is_none());
|
||||
// Present but empty string → None (empty is not a valid visitorData).
|
||||
let resp = json!({ "responseContext": { "visitorData": "" } });
|
||||
assert!(visitor_data_from_response(&resp).is_none());
|
||||
// responseContext without the field → None.
|
||||
let resp = json!({ "responseContext": { "foo": "bar" } });
|
||||
assert!(visitor_data_from_response(&resp).is_none());
|
||||
}
|
||||
|
||||
// ---- visitorData cache: fresh-hit / TTL-expiry / reset ----------------
|
||||
//
|
||||
// These touch the process-global visitorData cache and (for the refetch
|
||||
// test) the global Downloader, so they serialize on DOWNLOADER_TEST_LOCK,
|
||||
// shared with the player_manager test module.
|
||||
|
||||
use crate::downloader::request::Request;
|
||||
use crate::downloader::response::Response;
|
||||
use crate::downloader::Downloader;
|
||||
use crate::newpipe::DOWNLOADER_TEST_LOCK;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Serves any `visitor_id` POST with a fixed visitorData; counts requests.
|
||||
struct VisitorStub {
|
||||
visitor: String,
|
||||
requests: AtomicUsize,
|
||||
}
|
||||
|
||||
impl Downloader for VisitorStub {
|
||||
fn execute(&self, request: Request) -> Result<Response, NetworkError> {
|
||||
self.requests.fetch_add(1, Ordering::SeqCst);
|
||||
let url = request.url().to_string();
|
||||
if url.contains("visitor_id") {
|
||||
let body = format!(
|
||||
r#"{{"responseContext":{{"visitorData":"{}"}}}}"#,
|
||||
self.visitor
|
||||
);
|
||||
Ok(Response::new(200, "OK", Default::default(), body, url))
|
||||
} else {
|
||||
Err(NetworkError::Transport("visitor-stub: unexpected request".into()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fresh_cached_visitor_data_served_without_network() {
|
||||
let _g = DOWNLOADER_TEST_LOCK.lock();
|
||||
set_force_visitor_stale(false);
|
||||
reset_visitor_data_cache();
|
||||
seed_visitor_data_cache("FRESH_VD");
|
||||
// No downloader is needed: a fresh cache hit returns before any fetch.
|
||||
let info = InnertubeClientRequestInfo::of_android_client();
|
||||
let got = get_visitor_data(
|
||||
&info,
|
||||
&Localization::default(),
|
||||
&ContentCountry::default(),
|
||||
mobile_post_headers("ua/1.0"),
|
||||
YOUTUBEI_V1_GAPIS_URL,
|
||||
);
|
||||
assert_eq!(got.as_deref(), Some("FRESH_VD"));
|
||||
reset_visitor_data_cache();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn expired_visitor_data_triggers_refetch() {
|
||||
let _g = DOWNLOADER_TEST_LOCK.lock();
|
||||
let stub = Arc::new(VisitorStub {
|
||||
visitor: "REFRESHED_VD".into(),
|
||||
requests: AtomicUsize::new(0),
|
||||
});
|
||||
NewPipe::init(stub.clone() as Arc<dyn Downloader>);
|
||||
|
||||
reset_visitor_data_cache();
|
||||
seed_visitor_data_cache("STALE_VD"); // fresh instant…
|
||||
set_force_visitor_stale(true); // …but forced past its TTL
|
||||
|
||||
let info = InnertubeClientRequestInfo::of_android_client();
|
||||
let got = get_visitor_data(
|
||||
&info,
|
||||
&Localization::default(),
|
||||
&ContentCountry::default(),
|
||||
mobile_post_headers("ua/1.0"),
|
||||
YOUTUBEI_V1_GAPIS_URL,
|
||||
);
|
||||
|
||||
assert_eq!(got.as_deref(), Some("REFRESHED_VD"), "expired entry must refetch");
|
||||
assert_eq!(
|
||||
stub.requests.load(Ordering::SeqCst),
|
||||
1,
|
||||
"exactly one visitor_id refetch"
|
||||
);
|
||||
// The refetched value replaced the stale one, stamped fresh.
|
||||
set_force_visitor_stale(false);
|
||||
assert_eq!(peek_visitor_data_cache().as_deref(), Some("REFRESHED_VD"));
|
||||
|
||||
set_force_visitor_stale(false);
|
||||
reset_visitor_data_cache();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reset_clears_visitor_data_cache_for_self_heal() {
|
||||
// Fix 2b mechanism: on a primary-android rejection the extractor calls
|
||||
// reset_visitor_data_cache() so a poisoned value doesn't ride the
|
||||
// primary reel call forever.
|
||||
let _g = DOWNLOADER_TEST_LOCK.lock();
|
||||
set_force_visitor_stale(false);
|
||||
reset_visitor_data_cache();
|
||||
seed_visitor_data_cache("POISONED_VD");
|
||||
assert_eq!(peek_visitor_data_cache().as_deref(), Some("POISONED_VD"));
|
||||
reset_visitor_data_cache();
|
||||
assert!(peek_visitor_data_cache().is_none());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue