Compare commits
1 commit
main
...
renovate/r
| Author | SHA1 | Date | |
|---|---|---|---|
| 2702e8fe3a |
31 changed files with 309 additions and 3508 deletions
56
Cargo.lock
generated
56
Cargo.lock
generated
|
|
@ -121,7 +121,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"proc-macro2",
|
"proc-macro2",
|
||||||
"quote",
|
"quote",
|
||||||
"syn 2.0.117",
|
"syn",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
|
@ -711,9 +711,9 @@ dependencies = [
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "regex"
|
name = "regex"
|
||||||
version = "1.12.4"
|
version = "1.13.1"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba"
|
checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aho-corasick",
|
"aho-corasick",
|
||||||
"memchr",
|
"memchr",
|
||||||
|
|
@ -723,9 +723,9 @@ dependencies = [
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "regex-automata"
|
name = "regex-automata"
|
||||||
version = "0.4.14"
|
version = "0.4.16"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f"
|
checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aho-corasick",
|
"aho-corasick",
|
||||||
"memchr",
|
"memchr",
|
||||||
|
|
@ -891,9 +891,9 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "serde"
|
name = "serde"
|
||||||
version = "1.0.229"
|
version = "1.0.228"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
|
checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"serde_core",
|
"serde_core",
|
||||||
"serde_derive",
|
"serde_derive",
|
||||||
|
|
@ -901,29 +901,29 @@ dependencies = [
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "serde_core"
|
name = "serde_core"
|
||||||
version = "1.0.229"
|
version = "1.0.228"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
|
checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"serde_derive",
|
"serde_derive",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "serde_derive"
|
name = "serde_derive"
|
||||||
version = "1.0.229"
|
version = "1.0.228"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
|
checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"proc-macro2",
|
"proc-macro2",
|
||||||
"quote",
|
"quote",
|
||||||
"syn 3.0.3",
|
"syn",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "serde_json"
|
name = "serde_json"
|
||||||
version = "1.0.151"
|
version = "1.0.150"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
|
checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"itoa",
|
"itoa",
|
||||||
"memchr",
|
"memchr",
|
||||||
|
|
@ -988,7 +988,6 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596"
|
||||||
name = "strawcore-core"
|
name = "strawcore-core"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"log",
|
|
||||||
"once_cell",
|
"once_cell",
|
||||||
"parking_lot",
|
"parking_lot",
|
||||||
"regex",
|
"regex",
|
||||||
|
|
@ -1018,17 +1017,6 @@ dependencies = [
|
||||||
"unicode-ident",
|
"unicode-ident",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "syn"
|
|
||||||
version = "3.0.3"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3"
|
|
||||||
dependencies = [
|
|
||||||
"proc-macro2",
|
|
||||||
"quote",
|
|
||||||
"unicode-ident",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "sync_wrapper"
|
name = "sync_wrapper"
|
||||||
version = "1.0.2"
|
version = "1.0.2"
|
||||||
|
|
@ -1046,7 +1034,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"proc-macro2",
|
"proc-macro2",
|
||||||
"quote",
|
"quote",
|
||||||
"syn 2.0.117",
|
"syn",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
|
@ -1075,7 +1063,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"proc-macro2",
|
"proc-macro2",
|
||||||
"quote",
|
"quote",
|
||||||
"syn 2.0.117",
|
"syn",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
|
@ -1086,7 +1074,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"proc-macro2",
|
"proc-macro2",
|
||||||
"quote",
|
"quote",
|
||||||
"syn 2.0.117",
|
"syn",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
|
@ -1328,7 +1316,7 @@ dependencies = [
|
||||||
"bumpalo",
|
"bumpalo",
|
||||||
"proc-macro2",
|
"proc-macro2",
|
||||||
"quote",
|
"quote",
|
||||||
"syn 2.0.117",
|
"syn",
|
||||||
"wasm-bindgen-shared",
|
"wasm-bindgen-shared",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
@ -1563,7 +1551,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"proc-macro2",
|
"proc-macro2",
|
||||||
"quote",
|
"quote",
|
||||||
"syn 2.0.117",
|
"syn",
|
||||||
"synstructure",
|
"synstructure",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
@ -1584,7 +1572,7 @@ checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"proc-macro2",
|
"proc-macro2",
|
||||||
"quote",
|
"quote",
|
||||||
"syn 2.0.117",
|
"syn",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
|
@ -1604,7 +1592,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"proc-macro2",
|
"proc-macro2",
|
||||||
"quote",
|
"quote",
|
||||||
"syn 2.0.117",
|
"syn",
|
||||||
"synstructure",
|
"synstructure",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
@ -1644,7 +1632,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"proc-macro2",
|
"proc-macro2",
|
||||||
"quote",
|
"quote",
|
||||||
"syn 2.0.117",
|
"syn",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
|
|
||||||
|
|
@ -19,7 +19,6 @@ reqwest = { version = "0.12", default-features = false, features = ["rustls-tls-
|
||||||
serde = { version = "1", features = ["derive"] }
|
serde = { version = "1", features = ["derive"] }
|
||||||
serde_json = "1"
|
serde_json = "1"
|
||||||
thiserror = "1"
|
thiserror = "1"
|
||||||
log = "0.4"
|
|
||||||
parking_lot = "0.12"
|
parking_lot = "0.12"
|
||||||
url = "2"
|
url = "2"
|
||||||
once_cell = "1"
|
once_cell = "1"
|
||||||
|
|
|
||||||
|
|
@ -1,141 +0,0 @@
|
||||||
// Differential-verification JSON emitter (Phase-1 harness, NOT shipped).
|
|
||||||
//
|
|
||||||
// Additive example only — adds no production code. Emits strawcore's
|
|
||||||
// extraction for one video id as the harness's normalized JSON schema on
|
|
||||||
// stdout, so it can be diffed against the upstream NewPipe-Extractor (Java)
|
|
||||||
// oracle. See /root/build/npe-diff/ for the driver + differ.
|
|
||||||
//
|
|
||||||
// cargo run --example emit_json -- <VIDEO_ID> [visionos-on]
|
|
||||||
//
|
|
||||||
// Default matches the shipped app path (ExtractOptions::default(), visionOS
|
|
||||||
// OFF). Pass `visionos-on` to fetch the visionOS client (client 101) — that
|
|
||||||
// is what upstream NPE does UNCONDITIONALLY, so `visionos-on` is the
|
|
||||||
// apples-to-apples mode for an NPE fidelity diff.
|
|
||||||
use std::sync::Arc;
|
|
||||||
|
|
||||||
use serde_json::{json, Value};
|
|
||||||
use strawcore_core::stream::{AudioStream, DeliveryMethod, StreamInfo, VideoStream};
|
|
||||||
use strawcore_core::youtube::stream_extractor::{stream_info_with, ExtractOptions};
|
|
||||||
use strawcore_core::NewPipe;
|
|
||||||
|
|
||||||
/// Parse the innertube client tag (`&c=WEB` / `&c=ANDROID` / ...) off a
|
|
||||||
/// googlevideo URL — this is the "which client sourced this stream" signal
|
|
||||||
/// that made the visionOS outage visible.
|
|
||||||
fn client_source(url: &str) -> Value {
|
|
||||||
url.split(['&', '?'])
|
|
||||||
.find_map(|kv| kv.strip_prefix("c="))
|
|
||||||
.map(|c| Value::String(c.to_string()))
|
|
||||||
.unwrap_or(Value::Null)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn delivery_str(d: DeliveryMethod) -> &'static str {
|
|
||||||
match d {
|
|
||||||
DeliveryMethod::Progressive => "progressive",
|
|
||||||
DeliveryMethod::Dash => "dash",
|
|
||||||
DeliveryMethod::Hls => "hls",
|
|
||||||
DeliveryMethod::Torrent => "torrent",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn video_json(s: &VideoStream, kind: &str) -> Value {
|
|
||||||
json!({
|
|
||||||
"kind": kind,
|
|
||||||
"itag": s.itag,
|
|
||||||
"delivery": delivery_str(s.delivery),
|
|
||||||
"format": s.format.extension(),
|
|
||||||
"mime": s.format.mime(),
|
|
||||||
"codec": s.codec,
|
|
||||||
"resolution": if s.resolution.is_empty() { Value::Null } else { json!(s.resolution) },
|
|
||||||
"fps": s.fps,
|
|
||||||
"bitrate": s.bandwidth,
|
|
||||||
"has_url": !s.url.is_empty(),
|
|
||||||
"client_source": client_source(&s.url),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
fn audio_json(s: &AudioStream) -> Value {
|
|
||||||
json!({
|
|
||||||
"kind": "audio",
|
|
||||||
"itag": s.itag,
|
|
||||||
"delivery": delivery_str(s.delivery),
|
|
||||||
"format": s.format.extension(),
|
|
||||||
"mime": s.format.mime(),
|
|
||||||
"codec": s.codec,
|
|
||||||
"resolution": Value::Null,
|
|
||||||
"fps": 0,
|
|
||||||
"bitrate": s.average_bitrate_kbps,
|
|
||||||
"has_url": !s.url.is_empty(),
|
|
||||||
"client_source": client_source(&s.url),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
fn emit(info: &StreamInfo) -> Value {
|
|
||||||
let mut streams: Vec<Value> = Vec::new();
|
|
||||||
for s in &info.video_streams {
|
|
||||||
streams.push(video_json(s, "muxed"));
|
|
||||||
}
|
|
||||||
for s in &info.video_only_streams {
|
|
||||||
streams.push(video_json(s, "video_only"));
|
|
||||||
}
|
|
||||||
for s in &info.audio_streams {
|
|
||||||
streams.push(audio_json(s));
|
|
||||||
}
|
|
||||||
// Deterministic order so diffs are stable: (kind, itag, delivery).
|
|
||||||
streams.sort_by(|a, b| {
|
|
||||||
(
|
|
||||||
a["kind"].as_str().unwrap_or(""),
|
|
||||||
a["itag"].as_u64().unwrap_or(0),
|
|
||||||
a["delivery"].as_str().unwrap_or(""),
|
|
||||||
)
|
|
||||||
.cmp(&(
|
|
||||||
b["kind"].as_str().unwrap_or(""),
|
|
||||||
b["itag"].as_u64().unwrap_or(0),
|
|
||||||
b["delivery"].as_str().unwrap_or(""),
|
|
||||||
))
|
|
||||||
});
|
|
||||||
|
|
||||||
json!({
|
|
||||||
"source": "strawcore",
|
|
||||||
"video_id": info.video_id,
|
|
||||||
"meta": {
|
|
||||||
"name": info.name,
|
|
||||||
"duration_seconds": info.duration_seconds,
|
|
||||||
"stream_type": format!("{:?}", info.stream_type),
|
|
||||||
"uploader_id": info.uploader_id,
|
|
||||||
"uploader_name": info.uploader_name,
|
|
||||||
"view_count": info.view_count,
|
|
||||||
},
|
|
||||||
"dash_present": info.dash_manifest_url.is_some(),
|
|
||||||
"hls_present": info.hls_manifest_url.is_some(),
|
|
||||||
"counts": {
|
|
||||||
"muxed": info.video_streams.len(),
|
|
||||||
"video_only": info.video_only_streams.len(),
|
|
||||||
"audio": info.audio_streams.len(),
|
|
||||||
},
|
|
||||||
"streams": streams,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
fn main() {
|
|
||||||
let mut args = std::env::args().skip(1);
|
|
||||||
let vid = args.next().unwrap_or_else(|| "dQw4w9WgXcQ".into());
|
|
||||||
let visionos = args.next().as_deref() == Some("visionos-on");
|
|
||||||
|
|
||||||
let dl =
|
|
||||||
Arc::new(strawcore_core::downloader::ReqwestDownloader::new().expect("build downloader"));
|
|
||||||
NewPipe::init(dl);
|
|
||||||
|
|
||||||
let opts = ExtractOptions {
|
|
||||||
fetch_visionos_client: visionos,
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
match stream_info_with(&vid, opts) {
|
|
||||||
Ok(info) => {
|
|
||||||
println!("{}", serde_json::to_string_pretty(&emit(&info)).unwrap());
|
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
eprintln!("strawcore extraction FAILED for {vid}: {e:?}");
|
|
||||||
std::process::exit(2);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,50 +0,0 @@
|
||||||
// Throwaway live diagnostic (playback-outage triage). Not shipped.
|
|
||||||
use std::sync::Arc;
|
|
||||||
use strawcore_core::downloader::ReqwestDownloader;
|
|
||||||
use strawcore_core::youtube::stream_extractor::{stream_info_with, ExtractOptions};
|
|
||||||
use strawcore_core::NewPipe;
|
|
||||||
|
|
||||||
fn client_of(url: &str) -> String {
|
|
||||||
url.split('&')
|
|
||||||
.find_map(|kv| kv.strip_prefix("c="))
|
|
||||||
.unwrap_or("?")
|
|
||||||
.to_string()
|
|
||||||
}
|
|
||||||
|
|
||||||
fn report(label: &str, vid: &str, visionos: bool) {
|
|
||||||
let opts = ExtractOptions {
|
|
||||||
fetch_visionos_client: visionos,
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
match stream_info_with(vid, opts) {
|
|
||||||
Ok(i) => {
|
|
||||||
let a = i.audio_streams.first().map(|s| client_of(&s.url));
|
|
||||||
let v = i
|
|
||||||
.video_streams
|
|
||||||
.first()
|
|
||||||
.or_else(|| i.video_only_streams.first())
|
|
||||||
.map(|s| client_of(&s.url));
|
|
||||||
println!(
|
|
||||||
"[{label}] OK audio={} video={} video_only={} dash={} hls={} | first audio c={:?} first video c={:?}",
|
|
||||||
i.audio_streams.len(),
|
|
||||||
i.video_streams.len(),
|
|
||||||
i.video_only_streams.len(),
|
|
||||||
i.dash_manifest_url.is_some(),
|
|
||||||
i.hls_manifest_url.is_some(),
|
|
||||||
a,
|
|
||||||
v,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
Err(e) => println!("[{label}] ERR {e:?}"),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn main() {
|
|
||||||
let dl = Arc::new(ReqwestDownloader::new().expect("downloader"));
|
|
||||||
NewPipe::init(dl);
|
|
||||||
let vid = std::env::args()
|
|
||||||
.nth(1)
|
|
||||||
.unwrap_or_else(|| "dQw4w9WgXcQ".into());
|
|
||||||
report("visionOS ON (shipped)", &vid, true);
|
|
||||||
report("visionOS OFF (like 91) ", &vid, false);
|
|
||||||
}
|
|
||||||
|
|
@ -9,7 +9,7 @@
|
||||||
// as Ok(Response)
|
// as Ok(Response)
|
||||||
|
|
||||||
use std::io::Read;
|
use std::io::Read;
|
||||||
use std::time::{Duration, Instant};
|
use std::time::Duration;
|
||||||
|
|
||||||
use reqwest::blocking::Client;
|
use reqwest::blocking::Client;
|
||||||
use reqwest::redirect::Policy;
|
use reqwest::redirect::Policy;
|
||||||
|
|
@ -52,11 +52,6 @@ impl ReqwestDownloader {
|
||||||
|
|
||||||
impl Downloader for ReqwestDownloader {
|
impl Downloader for ReqwestDownloader {
|
||||||
fn execute(&self, request: Request) -> Result<Response, NetworkError> {
|
fn execute(&self, request: Request) -> Result<Response, NetworkError> {
|
||||||
let started = Instant::now();
|
|
||||||
let method_str = match request.method() {
|
|
||||||
Method::Get => "GET",
|
|
||||||
Method::Post => "POST",
|
|
||||||
};
|
|
||||||
let method = match request.method() {
|
let method = match request.method() {
|
||||||
Method::Get => reqwest::Method::GET,
|
Method::Get => reqwest::Method::GET,
|
||||||
Method::Post => reqwest::Method::POST,
|
Method::Post => reqwest::Method::POST,
|
||||||
|
|
@ -84,27 +79,9 @@ impl Downloader for ReqwestDownloader {
|
||||||
|
|
||||||
let status = resp.status();
|
let status = resp.status();
|
||||||
let url_after_redirects = resp.url().to_string();
|
let url_after_redirects = resp.url().to_string();
|
||||||
// Query/fragment carry n/sig/pot/cpn/id — strip for every log line.
|
|
||||||
let endpoint = url_after_redirects
|
|
||||||
.split(['?', '#'])
|
|
||||||
.next()
|
|
||||||
.unwrap_or(&url_after_redirects)
|
|
||||||
.to_string();
|
|
||||||
|
|
||||||
if status.as_u16() == 429 {
|
if status.as_u16() == 429 {
|
||||||
// Privacy: this URL ends up in error strings (and from there in
|
return Err(NetworkError::Recaptcha { url: url_after_redirects });
|
||||||
// Kotlin exception messages / logs, and the deobf failure memo).
|
|
||||||
// YouTube URLs carry the video id in the query (`?v=` / `&id=`)
|
|
||||||
// AND in some paths (`/embed/<id>`, `/shorts/<id>`), so keep only
|
|
||||||
// scheme+host. The Response's latest_url below stays full — it's
|
|
||||||
// data consumed by redirect-tracking logic, not a message.
|
|
||||||
let mut stripped = resp.url().clone();
|
|
||||||
stripped.set_query(None);
|
|
||||||
stripped.set_fragment(None);
|
|
||||||
stripped.set_path("/");
|
|
||||||
// Densest bot-flag signal — host only (path/query already stripped).
|
|
||||||
log::warn!("HTTP 429 reCAPTCHA / bot-wall from {stripped} — YouTube is rate-limiting");
|
|
||||||
return Err(NetworkError::Recaptcha { url: stripped.to_string() });
|
|
||||||
}
|
}
|
||||||
|
|
||||||
let code = status.as_u16();
|
let code = status.as_u16();
|
||||||
|
|
@ -120,9 +97,6 @@ impl Downloader for ReqwestDownloader {
|
||||||
// Fail fast when a known Content-Length already exceeds the cap.
|
// Fail fast when a known Content-Length already exceeds the cap.
|
||||||
if let Some(len) = resp.content_length() {
|
if let Some(len) = resp.content_length() {
|
||||||
if len > MAX_BODY_BYTES {
|
if len > MAX_BODY_BYTES {
|
||||||
log::warn!(
|
|
||||||
"{endpoint}: Content-Length {len}B exceeds cap {MAX_BODY_BYTES}B; aborting"
|
|
||||||
);
|
|
||||||
return Err(NetworkError::Transport(format!(
|
return Err(NetworkError::Transport(format!(
|
||||||
"response body {len} bytes exceeds cap {MAX_BODY_BYTES}"
|
"response body {len} bytes exceeds cap {MAX_BODY_BYTES}"
|
||||||
)));
|
)));
|
||||||
|
|
@ -139,7 +113,6 @@ impl Downloader for ReqwestDownloader {
|
||||||
.read_to_end(&mut buf)
|
.read_to_end(&mut buf)
|
||||||
.map_err(|e| NetworkError::Transport(format!("body read: {e}")))?;
|
.map_err(|e| NetworkError::Transport(format!("body read: {e}")))?;
|
||||||
if buf.len() as u64 > MAX_BODY_BYTES {
|
if buf.len() as u64 > MAX_BODY_BYTES {
|
||||||
log::warn!("{endpoint}: streamed body exceeded cap {MAX_BODY_BYTES}B; aborting");
|
|
||||||
return Err(NetworkError::Transport(format!(
|
return Err(NetworkError::Transport(format!(
|
||||||
"response body exceeded cap {MAX_BODY_BYTES}"
|
"response body exceeded cap {MAX_BODY_BYTES}"
|
||||||
)));
|
)));
|
||||||
|
|
@ -148,19 +121,11 @@ impl Downloader for ReqwestDownloader {
|
||||||
// common case): String::from_utf8 reinterprets the Vec in place,
|
// common case): String::from_utf8 reinterprets the Vec in place,
|
||||||
// whereas from_utf8_lossy always allocates + copies. Fall back to
|
// whereas from_utf8_lossy always allocates + copies. Fall back to
|
||||||
// lossy only on genuinely invalid bytes, preserving U+FFFD behavior.
|
// lossy only on genuinely invalid bytes, preserving U+FFFD behavior.
|
||||||
let body_len = buf.len();
|
|
||||||
let body = match String::from_utf8(buf) {
|
let body = match String::from_utf8(buf) {
|
||||||
Ok(s) => s,
|
Ok(s) => s,
|
||||||
Err(e) => String::from_utf8_lossy(e.as_bytes()).into_owned(),
|
Err(e) => String::from_utf8_lossy(e.as_bytes()).into_owned(),
|
||||||
};
|
};
|
||||||
|
|
||||||
// One DEBUG per request (dogfood ring). Query stripped; body length +
|
|
||||||
// status only — never the body. Gated behind the Debug-level bump.
|
|
||||||
log::debug!(
|
|
||||||
"{method_str} {endpoint} → {code} {body_len}B {}ms",
|
|
||||||
started.elapsed().as_millis()
|
|
||||||
);
|
|
||||||
|
|
||||||
Ok(Response::new(code, message, headers, body, url_after_redirects))
|
Ok(Response::new(code, message, headers, body, url_after_redirects))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -54,15 +54,6 @@ impl Response {
|
||||||
&self.response_body
|
&self.response_body
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Consumes the response, handing back the owned body `String` with no
|
|
||||||
/// copy. Use at the last read of a body when the rest of the `Response`
|
|
||||||
/// is no longer needed — e.g. the ~1.5 MB player.js download, where
|
|
||||||
/// `response_body().to_string()` would clone the whole buffer only to
|
|
||||||
/// drop the original (S8 speed fix).
|
|
||||||
pub fn into_body(self) -> String {
|
|
||||||
self.response_body
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn latest_url(&self) -> &str {
|
pub fn latest_url(&self) -> &str {
|
||||||
&self.latest_url
|
&self.latest_url
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -59,13 +59,6 @@ pub enum ContentUnavailable {
|
||||||
SoundCloudGoPlus,
|
SoundCloudGoPlus,
|
||||||
#[error("account terminated")]
|
#[error("account terminated")]
|
||||||
AccountTerminated,
|
AccountTerminated,
|
||||||
/// The 2026 poToken/BotGuard "sign in to confirm you're not a bot" wall.
|
|
||||||
/// Its Display is kept byte-identical to the old
|
|
||||||
/// `Other("sign in to confirm you're not a bot")` so string-matching
|
|
||||||
/// callers keep working; the dedicated variant makes the wall greppable in
|
|
||||||
/// logs and distinctly classifiable in app state.
|
|
||||||
#[error("sign in to confirm you're not a bot")]
|
|
||||||
BotDetected,
|
|
||||||
#[error("unavailable: {0}")]
|
#[error("unavailable: {0}")]
|
||||||
Other(String),
|
Other(String),
|
||||||
}
|
}
|
||||||
|
|
@ -90,12 +83,7 @@ pub enum ExtractionError {
|
||||||
|
|
||||||
impl From<reqwest::Error> for NetworkError {
|
impl From<reqwest::Error> for NetworkError {
|
||||||
fn from(e: reqwest::Error) -> Self {
|
fn from(e: reqwest::Error) -> Self {
|
||||||
// Privacy: reqwest's Display appends " for url (…)" whenever the
|
NetworkError::Transport(e.to_string())
|
||||||
// error carries a URL — for player requests that URL contains
|
|
||||||
// `id=<videoId>`, i.e. what the user was watching. Error strings
|
|
||||||
// become Kotlin exception messages and land in logcat / exported
|
|
||||||
// logs, so strip the URL at this single choke point.
|
|
||||||
NetworkError::Transport(e.without_url().to_string())
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -66,10 +66,3 @@ impl NewPipe {
|
||||||
*Self::instance().preferred_content_country.write() = content_country;
|
*Self::instance().preferred_content_country.write() = content_country;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Serializes every test that installs/uses a Downloader in the process-global
|
|
||||||
/// `NewPipe` singleton, so they don't clobber each other's registered
|
|
||||||
/// downloader (or corrupt request counts) when cargo runs tests in parallel.
|
|
||||||
/// Shared by the `player_manager` and `stream_helper` test modules.
|
|
||||||
#[cfg(test)]
|
|
||||||
pub(crate) static DOWNLOADER_TEST_LOCK: parking_lot::Mutex<()> = parking_lot::Mutex::new(());
|
|
||||||
|
|
|
||||||
|
|
@ -3,21 +3,6 @@
|
||||||
use crate::stream::DeliveryMethod;
|
use crate::stream::DeliveryMethod;
|
||||||
use crate::youtube::itag::MediaFormat;
|
use crate::youtube::itag::MediaFormat;
|
||||||
|
|
||||||
/// The track type of an [`AudioStream`]. Mirrors NPE
|
|
||||||
/// `stream/AudioTrackType.java`; derived from the format's `xtags` protobuf
|
|
||||||
/// (`acont` key) — see `youtube::xtags`.
|
|
||||||
#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)]
|
|
||||||
pub enum AudioTrackType {
|
|
||||||
/// The original audio track of a video.
|
|
||||||
Original,
|
|
||||||
/// The original voices replaced, typically in a different language.
|
|
||||||
Dubbed,
|
|
||||||
/// A descriptive (audio-description) track for accessibility.
|
|
||||||
Descriptive,
|
|
||||||
/// A secondary track (e.g. an alternate/commentary track).
|
|
||||||
Secondary,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug)]
|
#[derive(Clone, Debug)]
|
||||||
pub struct AudioStream {
|
pub struct AudioStream {
|
||||||
pub itag: u32,
|
pub itag: u32,
|
||||||
|
|
@ -30,12 +15,6 @@ pub struct AudioStream {
|
||||||
pub audio_track_id: Option<String>,
|
pub audio_track_id: Option<String>,
|
||||||
pub audio_track_name: Option<String>,
|
pub audio_track_name: Option<String>,
|
||||||
pub audio_locale: Option<String>,
|
pub audio_locale: Option<String>,
|
||||||
/// True iff this is a descriptive (audio-description) track. Preferentially
|
|
||||||
/// derived from `xtags` (`acont == descriptive`); falls back to the legacy
|
|
||||||
/// `audioTrack.audioIsDefault` heuristic when `xtags` is absent/unparseable.
|
|
||||||
pub is_descriptive: bool,
|
pub is_descriptive: bool,
|
||||||
/// The authoritative audio track type from the format's `xtags` blob, or
|
|
||||||
/// `None` for single-track audio / when `xtags` is absent or unparseable.
|
|
||||||
pub track_type: Option<AudioTrackType>,
|
|
||||||
pub itag_url_format: Option<String>,
|
pub itag_url_format: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -10,7 +10,7 @@ pub mod delivery;
|
||||||
pub mod subtitles;
|
pub mod subtitles;
|
||||||
pub mod video;
|
pub mod video;
|
||||||
|
|
||||||
pub use audio::{AudioStream, AudioTrackType};
|
pub use audio::AudioStream;
|
||||||
pub use delivery::DeliveryMethod;
|
pub use delivery::DeliveryMethod;
|
||||||
pub use subtitles::SubtitlesStream;
|
pub use subtitles::SubtitlesStream;
|
||||||
pub use video::VideoStream;
|
pub use video::VideoStream;
|
||||||
|
|
|
||||||
|
|
@ -46,14 +46,7 @@ pub fn channel_info(identifier: ChannelIdentifier) -> Result<ChannelInfo, Extrac
|
||||||
ChannelIdentifier::Custom(c) => resolve_handle_to_channel_id(&format!("c/{c}"))?,
|
ChannelIdentifier::Custom(c) => resolve_handle_to_channel_id(&format!("c/{c}"))?,
|
||||||
ChannelIdentifier::LegacyUser(u) => resolve_handle_to_channel_id(&format!("user/{u}"))?,
|
ChannelIdentifier::LegacyUser(u) => resolve_handle_to_channel_id(&format!("user/{u}"))?,
|
||||||
};
|
};
|
||||||
let info = fetch_channel_browse(&resolved)?;
|
fetch_channel_browse(&resolved)
|
||||||
log::info!(
|
|
||||||
"[channel {}] channel_info ok: videos={} continuation={}",
|
|
||||||
info.channel_id,
|
|
||||||
info.recent_videos.len(),
|
|
||||||
info.videos_continuation.is_some()
|
|
||||||
);
|
|
||||||
Ok(info)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn resolve_handle_to_channel_id(url_fragment: &str) -> Result<String, ExtractionError> {
|
pub fn resolve_handle_to_channel_id(url_fragment: &str) -> Result<String, ExtractionError> {
|
||||||
|
|
@ -120,21 +113,10 @@ pub fn fetch_channel_browse(channel_id: &str) -> Result<ChannelInfo, ExtractionE
|
||||||
|
|
||||||
// Videos tab is best-effort: a fetch error OR a panicked worker thread
|
// Videos tab is best-effort: a fetch error OR a panicked worker thread
|
||||||
// just leaves recent_videos empty (header still populated above).
|
// just leaves recent_videos empty (header still populated above).
|
||||||
match videos_result {
|
if let Ok(Ok(videos_response)) = videos_result {
|
||||||
Ok(Ok(videos_response)) => {
|
info.recent_videos = parse_videos_tab(&videos_response);
|
||||||
info.recent_videos = parse_videos_tab(&videos_response, channel_id);
|
if let Some(token) = parse_videos_continuation(&videos_response) {
|
||||||
if let Some(token) = parse_videos_continuation(&videos_response) {
|
info.videos_continuation = Some(token);
|
||||||
info.videos_continuation = Some(token);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Ok(Err(e)) => {
|
|
||||||
log::warn!("[channel {channel_id}] Videos-tab fetch failed ({e}); recent_videos empty (header still populated)");
|
|
||||||
}
|
|
||||||
Err(_) => {
|
|
||||||
// The worker closure can't panic by construction → invariant breach.
|
|
||||||
log::error!(
|
|
||||||
"[channel {channel_id}] Videos-tab worker thread panicked; recent_videos empty"
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Ok(info)
|
Ok(info)
|
||||||
|
|
@ -147,13 +129,7 @@ pub fn fetch_channel_browse(channel_id: &str) -> Result<ChannelInfo, ExtractionE
|
||||||
/// exhausted).
|
/// exhausted).
|
||||||
pub fn channel_videos_continuation(token: &str) -> Result<ContinuationPage, ExtractionError> {
|
pub fn channel_videos_continuation(token: &str) -> Result<ContinuationPage, ExtractionError> {
|
||||||
let body = fetch_continuation_browse(token)?;
|
let body = fetch_continuation_browse(token)?;
|
||||||
let page = parse_channel_continuation(&body);
|
Ok(parse_channel_continuation(&body))
|
||||||
log::info!(
|
|
||||||
"channel_videos_continuation ok: items={} more={}",
|
|
||||||
page.items.len(),
|
|
||||||
page.continuation.is_some()
|
|
||||||
);
|
|
||||||
Ok(page)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// POST `browse` with `{"continuation": token}` (no browseId/params).
|
/// POST `browse` with `{"continuation": token}` (no browseId/params).
|
||||||
|
|
@ -194,24 +170,12 @@ fn fetch_continuation_browse(token: &str) -> Result<Value, ExtractionError> {
|
||||||
/// `continuationItems` array.
|
/// `continuationItems` array.
|
||||||
pub fn parse_channel_continuation(body: &Value) -> ContinuationPage {
|
pub fn parse_channel_continuation(body: &Value) -> ContinuationPage {
|
||||||
let Some(items) = continuation_items(body, "onResponseReceivedActions") else {
|
let Some(items) = continuation_items(body, "onResponseReceivedActions") else {
|
||||||
// A non-empty body with no continuationItems = the pagination shape
|
|
||||||
// moved; the page returns empty and pagination silently ends.
|
|
||||||
if body.as_object().map(|m| !m.is_empty()).unwrap_or(false) {
|
|
||||||
log::warn!("channel continuation: no continuationItems found (pagination shape changed?); ending pagination");
|
|
||||||
}
|
|
||||||
return ContinuationPage::default();
|
return ContinuationPage::default();
|
||||||
};
|
};
|
||||||
let videos: Vec<StreamInfoItem> = items.iter().filter_map(parse_rich_grid_item).collect();
|
let videos = items.iter().filter_map(parse_rich_grid_item).collect();
|
||||||
// Aggregate renderer-migration signal: cells present but none parsed.
|
|
||||||
if !items.is_empty() && videos.is_empty() {
|
|
||||||
log::warn!(
|
|
||||||
"channel continuation: {} cell(s) but 0 parsed (renderer migration?)",
|
|
||||||
items.len()
|
|
||||||
);
|
|
||||||
}
|
|
||||||
ContinuationPage {
|
ContinuationPage {
|
||||||
continuation: token_from_items(items),
|
|
||||||
items: videos,
|
items: videos,
|
||||||
|
continuation: token_from_items(items),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -325,23 +289,11 @@ pub fn parse_channel_browse(channel_id: &str, body: &Value) -> ChannelInfo {
|
||||||
/// Handles BOTH old-style `videoRenderer` items and new-style
|
/// Handles BOTH old-style `videoRenderer` items and new-style
|
||||||
/// `lockupViewModel` items (YT migrated channel-videos UI to
|
/// `lockupViewModel` items (YT migrated channel-videos UI to
|
||||||
/// lockupViewModel around 2024).
|
/// lockupViewModel around 2024).
|
||||||
fn parse_videos_tab(body: &Value, channel_id: &str) -> Vec<StreamInfoItem> {
|
fn parse_videos_tab(body: &Value) -> Vec<StreamInfoItem> {
|
||||||
let Some(items) = selected_tab_grid_contents(body) else {
|
match selected_tab_grid_contents(body) {
|
||||||
// A real browse response (has `contents`) but no grid = layout change.
|
Some(items) => items.iter().filter_map(parse_rich_grid_item).collect(),
|
||||||
if body.get("contents").is_some() {
|
None => Vec::new(),
|
||||||
log::warn!("[channel {channel_id}] Videos-tab grid not found (layout change?); recent_videos empty");
|
|
||||||
}
|
|
||||||
return Vec::new();
|
|
||||||
};
|
|
||||||
let parsed: Vec<StreamInfoItem> = items.iter().filter_map(parse_rich_grid_item).collect();
|
|
||||||
// Aggregate renderer-migration signal: cells present but none parsed.
|
|
||||||
if !items.is_empty() && parsed.is_empty() {
|
|
||||||
log::warn!(
|
|
||||||
"[channel {channel_id}] Videos-tab grid had {} cell(s) but 0 parsed (renderer migration?)",
|
|
||||||
items.len()
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
parsed
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The `contents[]` array of the selected channel tab's
|
/// The `contents[]` array of the selected channel tab's
|
||||||
|
|
|
||||||
|
|
@ -111,29 +111,6 @@ impl InnertubeClientRequestInfo {
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// visionOS client (client 101). Mirrors NPE
|
|
||||||
/// `InnertubeClientRequestInfo.ofVisionOsClient()`. Same MOBILE/Apple shape
|
|
||||||
/// as iOS but with the RealityDevice model + visionOS os fields.
|
|
||||||
pub fn of_visionos_client() -> Self {
|
|
||||||
Self {
|
|
||||||
client_info: ClientInfo {
|
|
||||||
client_name: VISIONOS_CLIENT_NAME.into(),
|
|
||||||
client_version: VISIONOS_CLIENT_VERSION.into(),
|
|
||||||
client_id: VISIONOS_CLIENT_ID.into(),
|
|
||||||
client_screen: Some(WATCH_CLIENT_SCREEN.into()),
|
|
||||||
visitor_data: None,
|
|
||||||
},
|
|
||||||
device_info: DeviceInfo {
|
|
||||||
platform: Some(MOBILE_CLIENT_PLATFORM.into()),
|
|
||||||
device_make: Some("Apple".into()),
|
|
||||||
device_model: Some(VISIONOS_DEVICE_MODEL.into()),
|
|
||||||
os_name: Some("visionOS".into()),
|
|
||||||
os_version: Some(VISIONOS_OS_VERSION.into()),
|
|
||||||
android_sdk_version: -1,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Builds the InnerTube request envelope mirroring NPE prepareJsonBuilder.
|
/// Builds the InnerTube request envelope mirroring NPE prepareJsonBuilder.
|
||||||
|
|
@ -298,30 +275,6 @@ mod tests {
|
||||||
assert!(client.get("androidSdkVersion").is_none());
|
assert!(client.get("androidSdkVersion").is_none());
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn visionos_client_envelope_shape() {
|
|
||||||
let info = InnertubeClientRequestInfo::of_visionos_client();
|
|
||||||
let env = build_envelope(
|
|
||||||
&info,
|
|
||||||
&Localization::default(),
|
|
||||||
&ContentCountry::default(),
|
|
||||||
None,
|
|
||||||
);
|
|
||||||
let client = &env["context"]["client"];
|
|
||||||
assert_eq!(client["clientName"], "VISIONOS");
|
|
||||||
assert_eq!(client["clientVersion"], "1.02");
|
|
||||||
assert_eq!(client["platform"], "MOBILE");
|
|
||||||
assert_eq!(client["deviceMake"], "Apple");
|
|
||||||
assert_eq!(client["deviceModel"], "RealityDevice14,1");
|
|
||||||
assert_eq!(client["osName"], "visionOS");
|
|
||||||
assert_eq!(client["osVersion"], "25.6.0.23O471");
|
|
||||||
assert_eq!(client["clientScreen"], "WATCH");
|
|
||||||
// no androidSdkVersion for an Apple client
|
|
||||||
assert!(client.get("androidSdkVersion").is_none());
|
|
||||||
// best-effort visitorData not set at envelope-build time
|
|
||||||
assert!(client.get("visitorData").is_none());
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn embed_url_lands_in_third_party_block() {
|
fn embed_url_lands_in_third_party_block() {
|
||||||
let info = InnertubeClientRequestInfo::of_web_embedded_player_client();
|
let info = InnertubeClientRequestInfo::of_web_embedded_player_client();
|
||||||
|
|
|
||||||
|
|
@ -36,18 +36,6 @@ pub const ANDROID_CLIENT_VERSION: &str = "21.03.36";
|
||||||
pub const ANDROID_SDK_VERSION: u32 = 36;
|
pub const ANDROID_SDK_VERSION: u32 = 36;
|
||||||
pub const ANDROID_OS_VERSION: &str = "16";
|
pub const ANDROID_OS_VERSION: &str = "16";
|
||||||
|
|
||||||
// visionOS client (client 101). A best-effort 4th /player fetch that recovers
|
|
||||||
// >360p streams for SABR-only / "made for kids" videos where the ANDROID/IOS
|
|
||||||
// clients cap at a single 360p muxed stream. Values from NPE
|
|
||||||
// ClientsConstants.java (commit 82b7e410, "Workaround again SABR-only
|
|
||||||
// responses"). Carries NO poToken.
|
|
||||||
pub const VISIONOS_CLIENT_ID: &str = "101";
|
|
||||||
pub const VISIONOS_CLIENT_NAME: &str = "VISIONOS";
|
|
||||||
pub const VISIONOS_CLIENT_VERSION: &str = "1.02";
|
|
||||||
pub const VISIONOS_DEVICE_MODEL: &str = "RealityDevice14,1";
|
|
||||||
pub const VISIONOS_OS_VERSION: &str = "25.6.0.23O471";
|
|
||||||
pub const VISIONOS_USER_AGENT_VERSION: &str = "25_6_0";
|
|
||||||
|
|
||||||
// Base URLs (NPE YoutubeParsingHelper.java:91,96).
|
// Base URLs (NPE YoutubeParsingHelper.java:91,96).
|
||||||
pub const YOUTUBEI_V1_URL: &str = "https://www.youtube.com/youtubei/v1/";
|
pub const YOUTUBEI_V1_URL: &str = "https://www.youtube.com/youtubei/v1/";
|
||||||
pub const YOUTUBEI_V1_GAPIS_URL: &str = "https://youtubei.googleapis.com/youtubei/v1/";
|
pub const YOUTUBEI_V1_GAPIS_URL: &str = "https://youtubei.googleapis.com/youtubei/v1/";
|
||||||
|
|
|
||||||
|
|
@ -40,28 +40,15 @@ static SCRIPT_TAG: Lazy<Regex> =
|
||||||
/// Extracts the player.js URL + body. Tries iframe_api first, falls back
|
/// Extracts the player.js URL + body. Tries iframe_api first, falls back
|
||||||
/// to the embed page on any failure (matches NPE's try/catch flow).
|
/// to the embed page on any failure (matches NPE's try/catch flow).
|
||||||
pub fn extract_javascript_player_code(video_id: &str) -> Result<(String, String), DeobfError> {
|
pub fn extract_javascript_player_code(video_id: &str) -> Result<(String, String), DeobfError> {
|
||||||
let url = extract_javascript_player_url(video_id)?;
|
|
||||||
let body = download_player_code(&url)?;
|
|
||||||
Ok((url, body))
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Discovers the CURRENT player.js URL only (a few-KB iframe_api fetch —
|
|
||||||
/// cheap next to the ~1.7 MB body). Split out so PlayerManager's failure
|
|
||||||
/// memo can probe "did YouTube rotate player.js?" without paying for a
|
|
||||||
/// body download that would deterministically re-fail extraction.
|
|
||||||
pub fn extract_javascript_player_url(video_id: &str) -> Result<String, DeobfError> {
|
|
||||||
let downloader = NewPipe::downloader().ok_or(DeobfError::DownloaderMissing)?;
|
let downloader = NewPipe::downloader().ok_or(DeobfError::DownloaderMissing)?;
|
||||||
|
|
||||||
let url = match extract_from_iframe(&*downloader) {
|
let url = match extract_from_iframe(&*downloader) {
|
||||||
Ok(u) => u,
|
Ok(u) => u,
|
||||||
Err(_iframe_err) => extract_from_embed(&*downloader, video_id)?,
|
Err(_iframe_err) => extract_from_embed(&*downloader, video_id)?,
|
||||||
};
|
};
|
||||||
clean_javascript_url(&url)
|
let cleaned = clean_javascript_url(&url)?;
|
||||||
}
|
let body = download_javascript_code(&*downloader, &cleaned)?;
|
||||||
|
Ok((cleaned, body))
|
||||||
/// Downloads the player.js body for an already-discovered URL.
|
|
||||||
pub fn download_player_code(url: &str) -> Result<String, DeobfError> {
|
|
||||||
let downloader = NewPipe::downloader().ok_or(DeobfError::DownloaderMissing)?;
|
|
||||||
download_javascript_code(&*downloader, url)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn extract_from_iframe(downloader: &dyn Downloader) -> Result<String, DeobfError> {
|
fn extract_from_iframe(downloader: &dyn Downloader) -> Result<String, DeobfError> {
|
||||||
|
|
@ -136,9 +123,7 @@ fn download_javascript_code(downloader: &dyn Downloader, url: &str) -> Result<St
|
||||||
resp.response_code()
|
resp.response_code()
|
||||||
)));
|
)));
|
||||||
}
|
}
|
||||||
// Consume the owned body — no second ~1.5 MB copy (S8). `resp` is dead
|
Ok(resp.response_body().to_string())
|
||||||
// after the response-code check above.
|
|
||||||
Ok(resp.into_body())
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|
|
||||||
|
|
@ -23,10 +23,7 @@ pub mod signature;
|
||||||
|
|
||||||
use thiserror::Error;
|
use thiserror::Error;
|
||||||
|
|
||||||
// Clone: PlayerManager's failure memo replays the original extraction
|
#[derive(Debug, Error)]
|
||||||
// error on calls suppressed by the cooldown (mirrors NPE's cached
|
|
||||||
// ParsingException fields).
|
|
||||||
#[derive(Debug, Clone, Error)]
|
|
||||||
pub enum DeobfError {
|
pub enum DeobfError {
|
||||||
#[error("could not fetch iframe_api: {0}")]
|
#[error("could not fetch iframe_api: {0}")]
|
||||||
FetchIframe(String),
|
FetchIframe(String),
|
||||||
|
|
@ -60,8 +57,6 @@ pub enum DeobfError {
|
||||||
JsRuntimeFailed(String),
|
JsRuntimeFailed(String),
|
||||||
#[error("nsig output was empty (function neutered?)")]
|
#[error("nsig output was empty (function neutered?)")]
|
||||||
NsigEmpty,
|
NsigEmpty,
|
||||||
#[error("nsig output equalled its input (identity — deobfuscator neutered?)")]
|
|
||||||
NsigIdentity,
|
|
||||||
#[error("downloader not initialized")]
|
#[error("downloader not initialized")]
|
||||||
DownloaderMissing,
|
DownloaderMissing,
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -112,15 +112,6 @@ fn deobfuscation_function_name(player_code: &str) -> Result<String, DeobfError>
|
||||||
// NPE's `groupCount()` excludes group 0, so:
|
// NPE's `groupCount()` excludes group 0, so:
|
||||||
// len() == 2 → 1 capture → direct name
|
// len() == 2 → 1 capture → direct name
|
||||||
// len() == 3 → 2 captures → array indirection
|
// len() == 3 → 2 captures → array indirection
|
||||||
//
|
|
||||||
// BUT: `caps.len()` is pattern-static — it counts DECLARED groups,
|
|
||||||
// not the ones that PARTICIPATED in the match. Regexes 6/7 declare
|
|
||||||
// the array-access group inside an optional `(?:@ARRAY@)?`, so a
|
|
||||||
// direct-call player.js shape matches with group 2 absent. Route on
|
|
||||||
// participation: group 2 unmatched → the group-1 name IS the answer.
|
|
||||||
// (NPE's Java would NPE on `Integer.parseInt(matcher.group(2))` here
|
|
||||||
// — a latent upstream bug; the optional group's evident intent, and
|
|
||||||
// yt-dlp's guarded equivalent, is the direct name.)
|
|
||||||
match caps.len() {
|
match caps.len() {
|
||||||
2 => {
|
2 => {
|
||||||
if let Some(m) = caps.get(1) {
|
if let Some(m) = caps.get(1) {
|
||||||
|
|
@ -128,14 +119,8 @@ fn deobfuscation_function_name(player_code: &str) -> Result<String, DeobfError>
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
3 => {
|
3 => {
|
||||||
let Some(index_m) = caps.get(2) else {
|
|
||||||
if let Some(m) = caps.get(1) {
|
|
||||||
return Ok(m.as_str().to_string());
|
|
||||||
}
|
|
||||||
continue;
|
|
||||||
};
|
|
||||||
let array_name = caps.get(1).map(|m| m.as_str()).unwrap_or_default();
|
let array_name = caps.get(1).map(|m| m.as_str()).unwrap_or_default();
|
||||||
let index_str = index_m.as_str();
|
let index_str = caps.get(2).map(|m| m.as_str()).unwrap_or_default();
|
||||||
let index: usize = index_str.parse().map_err(|_| {
|
let index: usize = index_str.parse().map_err(|_| {
|
||||||
DeobfError::NsigArrayLookupFailed(format!("bad index: {index_str}"))
|
DeobfError::NsigArrayLookupFailed(format!("bad index: {index_str}"))
|
||||||
})?;
|
})?;
|
||||||
|
|
@ -177,10 +162,7 @@ fn deobfuscation_function_body(
|
||||||
let function_base = format!("{function_name}=function");
|
let function_base = format!("{function_name}=function");
|
||||||
match match_to_closing_brace(player_code, &function_base) {
|
match match_to_closing_brace(player_code, &function_base) {
|
||||||
Ok(body) => Ok(format!("{function_base}{body};")),
|
Ok(body) => Ok(format!("{function_base}{body};")),
|
||||||
Err(_) => {
|
Err(_) => deobfuscation_function_body_regex(player_code, function_name),
|
||||||
log::debug!("nsig body: lexer brace-match failed, falling back to regex extraction");
|
|
||||||
deobfuscation_function_body_regex(player_code, function_name)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -203,18 +185,8 @@ fn deobfuscation_function_body_regex(
|
||||||
|
|
||||||
/// Strips `if(typeof X==="undefined")return <firstArg>;` so the function
|
/// Strips `if(typeof X==="undefined")return <firstArg>;` so the function
|
||||||
/// actually runs standalone. NPE adds this 2024-12-29 (`56595bd9d`).
|
/// actually runs standalone. NPE adds this 2024-12-29 (`56595bd9d`).
|
||||||
///
|
|
||||||
/// Must handle BOTH body shapes this crate produces:
|
|
||||||
/// * lexer path: `name=function(a){…};`
|
|
||||||
/// * regex-fallback path: `function name(a){…}`
|
|
||||||
///
|
|
||||||
/// NPE's FUNCTION_ARGUMENTS_REGEX requires the `=` (its regex-fallback
|
|
||||||
/// shape would throw from matchGroup1); ours accepts both so the fallback
|
|
||||||
/// path doesn't silently no-op — a surviving guard makes the function
|
|
||||||
/// return its input unchanged, which then gets CACHED as the deobfuscated
|
|
||||||
/// n-param → permanent silent throttling.
|
|
||||||
pub fn fixup_function(function: &str) -> Result<String, DeobfError> {
|
pub fn fixup_function(function: &str) -> Result<String, DeobfError> {
|
||||||
let args_re = Regex::new(r"(?:=\s*)?function\s*[a-zA-Z0-9$_]*\s*\(\s*([^)]*)\s*\)")
|
let args_re = Regex::new(r"=\s*function\s*\(\s*([^)]*)\s*\)")
|
||||||
.map_err(|e| DeobfError::NsigBodyParseFailed(e.to_string()))?;
|
.map_err(|e| DeobfError::NsigBodyParseFailed(e.to_string()))?;
|
||||||
let first_arg = args_re
|
let first_arg = args_re
|
||||||
.captures(function)
|
.captures(function)
|
||||||
|
|
@ -223,11 +195,6 @@ pub fn fixup_function(function: &str) -> Result<String, DeobfError> {
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
|
|
||||||
if first_arg.is_empty() {
|
if first_arg.is_empty() {
|
||||||
// The args regex didn't capture a first argument, so the
|
|
||||||
// `if(typeof X==="undefined")return <arg>;` guard can't be stripped —
|
|
||||||
// the deobfuscator will return its input unchanged (throttled URLs).
|
|
||||||
// Fires ~once per player.js generation (build is cached).
|
|
||||||
log::warn!("nsig fixup: could not extract the function's first argument; the early-return guard cannot be stripped → deobfuscator may return its input unchanged");
|
|
||||||
return Ok(function.to_string());
|
return Ok(function.to_string());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -236,18 +203,13 @@ pub fn fixup_function(function: &str) -> Result<String, DeobfError> {
|
||||||
// Substitute with an alternation of fully-quoted `"undefined"` /
|
// Substitute with an alternation of fully-quoted `"undefined"` /
|
||||||
// `'undefined'` forms. Loosens slightly (allows `"undefined'`) but
|
// `'undefined'` forms. Loosens slightly (allows `"undefined'`) but
|
||||||
// real player.js always uses balanced quotes; harmless.
|
// real player.js always uses balanced quotes; harmless.
|
||||||
//
|
|
||||||
// NPE's EARLY_RETURN_REGEX anchors on a preceding `;` only. When the
|
|
||||||
// guard is the FIRST statement of the body (regex-fallback shape:
|
|
||||||
// `function name(a){if(typeof X==="undefined")return a;…}`) the
|
|
||||||
// preceding token is `{` — accept both and re-emit whichever matched.
|
|
||||||
let early_return_re_src = format!(
|
let early_return_re_src = format!(
|
||||||
r#"(?s)([;{{])\s*if\s*\(\s*typeof\s+[a-zA-Z0-9$_]+\s*===?\s*(?:"undefined"|'undefined')\s*\)\s*return\s+{};"#,
|
r#"(?s);\s*if\s*\(\s*typeof\s+[a-zA-Z0-9$_]+\s*===?\s*(?:"undefined"|'undefined')\s*\)\s*return\s+{};"#,
|
||||||
regex::escape(&first_arg)
|
regex::escape(&first_arg)
|
||||||
);
|
);
|
||||||
let er_re = Regex::new(&early_return_re_src)
|
let er_re = Regex::new(&early_return_re_src)
|
||||||
.map_err(|e| DeobfError::NsigBodyParseFailed(e.to_string()))?;
|
.map_err(|e| DeobfError::NsigBodyParseFailed(e.to_string()))?;
|
||||||
Ok(er_re.replace(function, "$1").to_string())
|
Ok(er_re.replace(function, ";").to_string())
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|
@ -324,63 +286,4 @@ mod tests {
|
||||||
Err(e) => panic!("expected name match, got {e:?}"),
|
Err(e) => panic!("expected name match, got {e:?}"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn regex_6_direct_call_without_array_access() {
|
|
||||||
// Regex 6 (String.fromCharCode(110)) with a DIRECT call — the
|
|
||||||
// optional `(?:\[(\d+)])?` group does not participate. Must route
|
|
||||||
// to the group-1 name, not the array-indirection branch (which
|
|
||||||
// would hard-fail with NsigArrayLookupFailed on the empty index).
|
|
||||||
let src = r#"WL=function(a){a.j=1};(b=String.fromCharCode(110),c=a.get(b))&&(c=mfn(c),a.set(b,c))"#;
|
|
||||||
match deobfuscation_function_name(src) {
|
|
||||||
Ok(n) => assert_eq!(n, "mfn"),
|
|
||||||
Err(e) => panic!("direct-call shape must yield the direct name, got {e:?}"),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn regex_7_direct_call_without_array_access() {
|
|
||||||
// Regex 7 (.get("n")) with a direct call — same non-participating
|
|
||||||
// optional group.
|
|
||||||
let src = r#"(c=d.get("n"))&&(e=Nfn(e),d.set("n",e))"#;
|
|
||||||
match deobfuscation_function_name(src) {
|
|
||||||
Ok(n) => assert_eq!(n, "Nfn"),
|
|
||||||
Err(e) => panic!("direct-call shape must yield the direct name, got {e:?}"),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn fixup_strips_guard_on_function_declaration_shape() {
|
|
||||||
// The regex-fallback body shape: `function name(a){…}` — no `=`
|
|
||||||
// before `function`, and the guard sits right after `{` with no
|
|
||||||
// preceding `;`. The old fixup no-op'd on both counts, letting the
|
|
||||||
// early return survive → nsig returns its input unchanged → the
|
|
||||||
// identity result gets cached → permanent silent throttling.
|
|
||||||
let body = r#"function m85(p){if(typeof RUQ==="undefined")return p;var a=p.split("");a.reverse();return a.join("");}"#;
|
|
||||||
let fixed = fixup_function(body).unwrap();
|
|
||||||
assert!(
|
|
||||||
!fixed.contains("typeof RUQ"),
|
|
||||||
"guard must be stripped on the function-declaration shape, got: {fixed}"
|
|
||||||
);
|
|
||||||
assert!(fixed.contains(r#"var a=p.split("");"#));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn fixup_function_declaration_shape_runs_non_identity() {
|
|
||||||
// End-to-end: the fixed fallback-shape body must actually transform
|
|
||||||
// its input when run (i.e. the guard is gone, not just renamed).
|
|
||||||
let body = r#"function m85(p){if(typeof RUQ==="undefined")return p;var a=p.split("");a.reverse();return a.join("");}"#;
|
|
||||||
let fixed = fixup_function(body).unwrap();
|
|
||||||
let out = crate::youtube::js::runtime::run(&fixed, "m85", "abc123").unwrap();
|
|
||||||
assert_eq!(out, "321cba");
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn fixup_lexer_shape_guard_at_start_of_body() {
|
|
||||||
// Lexer shape (`name=function(...)`) whose guard is the FIRST
|
|
||||||
// statement — preceded by `{`, not `;`. Also must strip.
|
|
||||||
let body = r#"m85=function(p){if(typeof RUQ==="undefined")return p;var a=p.split("");return a.join("");}"#;
|
|
||||||
let fixed = fixup_function(body).unwrap();
|
|
||||||
assert!(!fixed.contains("typeof RUQ"));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
File diff suppressed because it is too large
Load diff
|
|
@ -64,11 +64,10 @@ pub fn compile_or_throw(snippet: &str) -> Result<(), DeobfError> {
|
||||||
/// with one string argument, returns the toString of the result.
|
/// with one string argument, returns the toString of the result.
|
||||||
/// Mirrors NPE `JavaScript.run(snippet, functionName, parameters)`.
|
/// Mirrors NPE `JavaScript.run(snippet, functionName, parameters)`.
|
||||||
pub fn run(snippet: &str, function_name: &str, parameter: &str) -> Result<String, DeobfError> {
|
pub fn run(snippet: &str, function_name: &str, parameter: &str) -> Result<String, DeobfError> {
|
||||||
let started = Instant::now();
|
|
||||||
let runtime = guarded_runtime().map_err(|e| DeobfError::JsRuntimeFailed(e.to_string()))?;
|
let runtime = guarded_runtime().map_err(|e| DeobfError::JsRuntimeFailed(e.to_string()))?;
|
||||||
let context =
|
let context =
|
||||||
Context::full(&runtime).map_err(|e| DeobfError::JsRuntimeFailed(e.to_string()))?;
|
Context::full(&runtime).map_err(|e| DeobfError::JsRuntimeFailed(e.to_string()))?;
|
||||||
let out = context.with(|ctx| -> Result<String, DeobfError> {
|
context.with(|ctx| -> Result<String, DeobfError> {
|
||||||
ctx.eval::<(), _>(snippet)
|
ctx.eval::<(), _>(snippet)
|
||||||
.map_err(|e| DeobfError::JsRuntimeFailed(format!("eval: {e}")))?;
|
.map_err(|e| DeobfError::JsRuntimeFailed(format!("eval: {e}")))?;
|
||||||
let func: Function = ctx
|
let func: Function = ctx
|
||||||
|
|
@ -81,18 +80,7 @@ pub fn run(snippet: &str, function_name: &str, parameter: &str) -> Result<String
|
||||||
.call((parameter,))
|
.call((parameter,))
|
||||||
.map_err(|e| DeobfError::JsRuntimeFailed(format!("call {function_name}: {e}")))?;
|
.map_err(|e| DeobfError::JsRuntimeFailed(format!("call {function_name}: {e}")))?;
|
||||||
Ok(result)
|
Ok(result)
|
||||||
});
|
})
|
||||||
// Distinguish the interrupt-handler deadline abort (a pathological /
|
|
||||||
// looping player.js) from an ordinary eval error — both surface as a
|
|
||||||
// generic QuickJS error at the eval site. `function_name` is a deobf
|
|
||||||
// function id (e.g. "deobfuscate"), not user data.
|
|
||||||
if out.is_err() && started.elapsed() >= JS_DEADLINE {
|
|
||||||
log::warn!(
|
|
||||||
"JS deadline ({}s) hit during `{function_name}` — execution aborted (pathological player.js?)",
|
|
||||||
JS_DEADLINE.as_secs()
|
|
||||||
);
|
|
||||||
}
|
|
||||||
out
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|
|
||||||
|
|
@ -101,10 +101,7 @@ pub fn deobfuscate_function_body(
|
||||||
let function_base = format!("{function_name}=function");
|
let function_base = format!("{function_name}=function");
|
||||||
match match_to_closing_brace(player_code, &function_base) {
|
match match_to_closing_brace(player_code, &function_base) {
|
||||||
Ok(body) => Ok(format!("{function_base}{body}")),
|
Ok(body) => Ok(format!("{function_base}{body}")),
|
||||||
Err(_) => {
|
Err(_) => deobfuscate_with_regex(player_code, function_name),
|
||||||
log::debug!("sig body: lexer brace-match failed, falling back to regex extraction");
|
|
||||||
deobfuscate_with_regex(player_code, function_name)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -10,7 +10,7 @@
|
||||||
|
|
||||||
use url::Url;
|
use url::Url;
|
||||||
|
|
||||||
use crate::youtube::linkhandler::{host_is_youtube, redact_url, LinkError};
|
use crate::youtube::linkhandler::{host_is_youtube, LinkError};
|
||||||
|
|
||||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||||
pub enum ChannelIdentifier {
|
pub enum ChannelIdentifier {
|
||||||
|
|
@ -25,10 +25,8 @@ pub enum ChannelIdentifier {
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn parse(url_str: &str) -> Result<ChannelIdentifier, LinkError> {
|
pub fn parse(url_str: &str) -> Result<ChannelIdentifier, LinkError> {
|
||||||
// Privacy: never embed the raw input in the error (see stream.rs) —
|
|
||||||
// the parse reason alone suffices.
|
|
||||||
let url = Url::parse(url_str)
|
let url = Url::parse(url_str)
|
||||||
.map_err(|e| LinkError::InvalidUrl(e.to_string()))?;
|
.map_err(|e| LinkError::InvalidUrl(format!("{url_str}: {e}")))?;
|
||||||
let host = url
|
let host = url
|
||||||
.host_str()
|
.host_str()
|
||||||
.ok_or_else(|| LinkError::InvalidUrl("no host".into()))?;
|
.ok_or_else(|| LinkError::InvalidUrl("no host".into()))?;
|
||||||
|
|
@ -39,32 +37,32 @@ pub fn parse(url_str: &str) -> Result<ChannelIdentifier, LinkError> {
|
||||||
if let Some(rest) = path.strip_prefix("/channel/") {
|
if let Some(rest) = path.strip_prefix("/channel/") {
|
||||||
let id = rest.split('/').next().unwrap_or("");
|
let id = rest.split('/').next().unwrap_or("");
|
||||||
if id.is_empty() {
|
if id.is_empty() {
|
||||||
return Err(LinkError::MissingId(redact_url(&url)));
|
return Err(LinkError::MissingId(url_str.into()));
|
||||||
}
|
}
|
||||||
return Ok(ChannelIdentifier::DirectId(id.into()));
|
return Ok(ChannelIdentifier::DirectId(id.into()));
|
||||||
}
|
}
|
||||||
if let Some(rest) = path.strip_prefix("/c/") {
|
if let Some(rest) = path.strip_prefix("/c/") {
|
||||||
let s = rest.split('/').next().unwrap_or("");
|
let s = rest.split('/').next().unwrap_or("");
|
||||||
if s.is_empty() {
|
if s.is_empty() {
|
||||||
return Err(LinkError::MissingId(redact_url(&url)));
|
return Err(LinkError::MissingId(url_str.into()));
|
||||||
}
|
}
|
||||||
return Ok(ChannelIdentifier::Custom(s.into()));
|
return Ok(ChannelIdentifier::Custom(s.into()));
|
||||||
}
|
}
|
||||||
if let Some(rest) = path.strip_prefix("/user/") {
|
if let Some(rest) = path.strip_prefix("/user/") {
|
||||||
let s = rest.split('/').next().unwrap_or("");
|
let s = rest.split('/').next().unwrap_or("");
|
||||||
if s.is_empty() {
|
if s.is_empty() {
|
||||||
return Err(LinkError::MissingId(redact_url(&url)));
|
return Err(LinkError::MissingId(url_str.into()));
|
||||||
}
|
}
|
||||||
return Ok(ChannelIdentifier::LegacyUser(s.into()));
|
return Ok(ChannelIdentifier::LegacyUser(s.into()));
|
||||||
}
|
}
|
||||||
if let Some(rest) = path.strip_prefix("/@") {
|
if let Some(rest) = path.strip_prefix("/@") {
|
||||||
let s = rest.split('/').next().unwrap_or("");
|
let s = rest.split('/').next().unwrap_or("");
|
||||||
if s.is_empty() {
|
if s.is_empty() {
|
||||||
return Err(LinkError::MissingId(redact_url(&url)));
|
return Err(LinkError::MissingId(url_str.into()));
|
||||||
}
|
}
|
||||||
return Ok(ChannelIdentifier::Handle(s.into()));
|
return Ok(ChannelIdentifier::Handle(s.into()));
|
||||||
}
|
}
|
||||||
Err(LinkError::MissingId(redact_url(&url)))
|
Err(LinkError::MissingId(url_str.into()))
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn channel_url(channel_id: &str) -> String {
|
pub fn channel_url(channel_id: &str) -> String {
|
||||||
|
|
|
||||||
|
|
@ -34,20 +34,6 @@ pub const ACCEPTED_HOSTS: &[&str] = &[
|
||||||
"www.youtube-nocookie.com",
|
"www.youtube-nocookie.com",
|
||||||
];
|
];
|
||||||
|
|
||||||
/// Renders a URL as scheme+host only, for embedding in error strings.
|
|
||||||
/// Privacy: link errors reach Kotlin exception messages / logs. Query params
|
|
||||||
/// carry user browsing data (`v=<videoId>`, `list=<playlistId>`) AND several
|
|
||||||
/// YouTube PATHS embed the id (`/embed/<id>`, `/shorts/<id>`, `/clip/<id>`,
|
|
||||||
/// `/live/<id>`, `youtu.be/<id>`) — so drop the path too; the host alone says
|
|
||||||
/// which endpoint failed without leaking what the user was watching.
|
|
||||||
pub(crate) fn redact_url(url: &url::Url) -> String {
|
|
||||||
let mut u = url.clone();
|
|
||||||
u.set_query(None);
|
|
||||||
u.set_fragment(None);
|
|
||||||
u.set_path("/");
|
|
||||||
u.to_string()
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn host_is_youtube(host: &str) -> bool {
|
pub fn host_is_youtube(host: &str) -> bool {
|
||||||
let h = host.to_ascii_lowercase();
|
let h = host.to_ascii_lowercase();
|
||||||
let h = h.strip_prefix("www.").unwrap_or(&h);
|
let h = h.strip_prefix("www.").unwrap_or(&h);
|
||||||
|
|
@ -80,24 +66,4 @@ mod tests {
|
||||||
assert!(!host_is_youtube("piped.video"));
|
assert!(!host_is_youtube("piped.video"));
|
||||||
assert!(!host_is_youtube("evil.com"));
|
assert!(!host_is_youtube("evil.com"));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn redact_url_drops_id_bearing_path_query_and_fragment() {
|
|
||||||
// Every one of these carries the video/playlist id somewhere the old
|
|
||||||
// query-only strip would have kept (the path, for embed/shorts/youtu.be).
|
|
||||||
let cases = [
|
|
||||||
"https://www.youtube.com/embed/dQw4w9WgXcQ?v=secret#frag",
|
|
||||||
"https://www.youtube.com/shorts/dQw4w9WgXcQ",
|
|
||||||
"https://youtu.be/dQw4w9WgXcQ",
|
|
||||||
"https://www.youtube.com/watch?v=dQw4w9WgXcQ&list=PLsecretlist",
|
|
||||||
];
|
|
||||||
for c in cases {
|
|
||||||
let u = url::Url::parse(c).unwrap();
|
|
||||||
let r = redact_url(&u);
|
|
||||||
assert!(!r.contains("dQw4w9WgXcQ"), "leaked video id: {r}");
|
|
||||||
assert!(!r.contains("PLsecretlist"), "leaked list id: {r}");
|
|
||||||
// scheme + host only (no port in any of these cases).
|
|
||||||
assert_eq!(r, format!("{}://{}/", u.scheme(), u.host_str().unwrap()));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -15,7 +15,7 @@ use once_cell::sync::Lazy;
|
||||||
use regex::Regex;
|
use regex::Regex;
|
||||||
use url::Url;
|
use url::Url;
|
||||||
|
|
||||||
use crate::youtube::linkhandler::{host_is_youtube, redact_url, LinkError};
|
use crate::youtube::linkhandler::{host_is_youtube, LinkError};
|
||||||
|
|
||||||
const VIDEO_ID_LEN: usize = 11;
|
const VIDEO_ID_LEN: usize = 11;
|
||||||
|
|
||||||
|
|
@ -42,11 +42,8 @@ fn extract_video_id_inner(input_url: &str, depth: u8) -> Result<String, LinkErro
|
||||||
// the JVM via UniFFI. One level is enough for the legitimate
|
// the JVM via UniFFI. One level is enough for the legitimate
|
||||||
// share-from-attribution-app case.
|
// share-from-attribution-app case.
|
||||||
const MAX_ATTRIBUTION_DEPTH: u8 = 1;
|
const MAX_ATTRIBUTION_DEPTH: u8 = 1;
|
||||||
// Privacy: never embed the raw input in the error — even a malformed
|
|
||||||
// paste can carry a video id / user browsing data, and link errors
|
|
||||||
// reach exception messages / logs. The parse reason alone suffices.
|
|
||||||
let url = Url::parse(input_url)
|
let url = Url::parse(input_url)
|
||||||
.map_err(|e| LinkError::InvalidUrl(e.to_string()))?;
|
.map_err(|e| LinkError::InvalidUrl(format!("{input_url}: {e}")))?;
|
||||||
let host = url
|
let host = url
|
||||||
.host_str()
|
.host_str()
|
||||||
.ok_or_else(|| LinkError::InvalidUrl("no host".into()))?;
|
.ok_or_else(|| LinkError::InvalidUrl("no host".into()))?;
|
||||||
|
|
@ -96,7 +93,7 @@ fn extract_video_id_inner(input_url: &str, depth: u8) -> Result<String, LinkErro
|
||||||
}
|
}
|
||||||
|
|
||||||
let id = candidate
|
let id = candidate
|
||||||
.ok_or_else(|| LinkError::MissingId(redact_url(&url)))?;
|
.ok_or_else(|| LinkError::MissingId(input_url.into()))?;
|
||||||
if !is_valid_video_id(&id) {
|
if !is_valid_video_id(&id) {
|
||||||
return Err(LinkError::MalformedId(id));
|
return Err(LinkError::MalformedId(id));
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -20,5 +20,4 @@ pub mod potoken;
|
||||||
pub mod search_extractor;
|
pub mod search_extractor;
|
||||||
pub mod stream_extractor;
|
pub mod stream_extractor;
|
||||||
pub mod stream_helper;
|
pub mod stream_helper;
|
||||||
pub mod xtags;
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -78,14 +78,6 @@ pub fn ios_user_agent(country: &ContentCountry) -> String {
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// visionOS user-agent — mirrors NPE `getVisionOsUserAgent`.
|
|
||||||
pub fn visionos_user_agent(country: &ContentCountry) -> String {
|
|
||||||
format!(
|
|
||||||
"com.google.visionos.youtube/{VISIONOS_CLIENT_VERSION}({VISIONOS_DEVICE_MODEL}; U; CPU visionOS {VISIONOS_USER_AGENT_VERSION} like Mac OS X; {})",
|
|
||||||
country.country_code()
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
@ -142,13 +134,4 @@ mod tests {
|
||||||
assert!(ua.contains("CPU iOS 18_7_2"));
|
assert!(ua.contains("CPU iOS 18_7_2"));
|
||||||
assert!(ua.contains("; US)"));
|
assert!(ua.contains("; US)"));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn visionos_ua_template() {
|
|
||||||
let ua = visionos_user_agent(&ContentCountry::new("US"));
|
|
||||||
assert!(ua.contains("com.google.visionos.youtube/1.02"));
|
|
||||||
assert!(ua.contains("RealityDevice14,1"));
|
|
||||||
assert!(ua.contains("CPU visionOS 25_6_0 like Mac OS X"));
|
|
||||||
assert!(ua.contains("; US)"));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -113,12 +113,10 @@ static REGISTERED_PROVIDER: Lazy<RwLock<Option<Arc<dyn PoTokenProvider>>>> =
|
||||||
Lazy::new(|| RwLock::new(None));
|
Lazy::new(|| RwLock::new(None));
|
||||||
|
|
||||||
pub fn set_po_token_provider(provider: Arc<dyn PoTokenProvider>) {
|
pub fn set_po_token_provider(provider: Arc<dyn PoTokenProvider>) {
|
||||||
log::info!("poToken provider registered");
|
|
||||||
*REGISTERED_PROVIDER.write() = Some(provider);
|
*REGISTERED_PROVIDER.write() = Some(provider);
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn clear_po_token_provider() {
|
pub fn clear_po_token_provider() {
|
||||||
log::info!("poToken provider cleared");
|
|
||||||
*REGISTERED_PROVIDER.write() = None;
|
*REGISTERED_PROVIDER.write() = None;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -69,15 +69,7 @@ pub fn search(query: &str, filter: SearchFilter) -> Result<SearchInfo, Extractio
|
||||||
}
|
}
|
||||||
let parsed: Value = serde_json::from_str(resp.response_body())
|
let parsed: Value = serde_json::from_str(resp.response_body())
|
||||||
.map_err(|e| ExtractionError::Parsing(ParsingError::JsonShape(e.to_string())))?;
|
.map_err(|e| ExtractionError::Parsing(ParsingError::JsonShape(e.to_string())))?;
|
||||||
let info = parse_search_response(query, &parsed);
|
Ok(parse_search_response(query, &parsed))
|
||||||
log::info!(
|
|
||||||
"search ok: query_len={} items={} corrected={} continuation={}",
|
|
||||||
query.len(),
|
|
||||||
info.videos.len(),
|
|
||||||
info.corrected_query.is_some(),
|
|
||||||
info.continuation_token.is_some()
|
|
||||||
);
|
|
||||||
Ok(info)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Fetch the NEXT page of search results via a continuation token.
|
/// Fetch the NEXT page of search results via a continuation token.
|
||||||
|
|
@ -113,13 +105,7 @@ pub fn search_continuation(token: &str) -> Result<ContinuationPage, ExtractionEr
|
||||||
}
|
}
|
||||||
let parsed: Value = serde_json::from_str(resp.response_body())
|
let parsed: Value = serde_json::from_str(resp.response_body())
|
||||||
.map_err(|e| ExtractionError::Parsing(ParsingError::JsonShape(e.to_string())))?;
|
.map_err(|e| ExtractionError::Parsing(ParsingError::JsonShape(e.to_string())))?;
|
||||||
let page = parse_search_continuation(&parsed);
|
Ok(parse_search_continuation(&parsed))
|
||||||
log::info!(
|
|
||||||
"search_continuation ok: items={} more={}",
|
|
||||||
page.items.len(),
|
|
||||||
page.continuation.is_some()
|
|
||||||
);
|
|
||||||
Ok(page)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Parse a search continuation response. Items arrive under
|
/// Parse a search continuation response. Items arrive under
|
||||||
|
|
@ -130,9 +116,6 @@ pub fn search_continuation(token: &str) -> Result<ContinuationPage, ExtractionEr
|
||||||
/// continuationItemRenderer carrying the next token.
|
/// continuationItemRenderer carrying the next token.
|
||||||
pub fn parse_search_continuation(body: &Value) -> ContinuationPage {
|
pub fn parse_search_continuation(body: &Value) -> ContinuationPage {
|
||||||
let Some(items) = continuation_items(body, "onResponseReceivedCommands") else {
|
let Some(items) = continuation_items(body, "onResponseReceivedCommands") else {
|
||||||
if body.as_object().map(|m| !m.is_empty()).unwrap_or(false) {
|
|
||||||
log::warn!("search continuation: no continuationItems found (shape changed?); ending pagination");
|
|
||||||
}
|
|
||||||
return ContinuationPage::default();
|
return ContinuationPage::default();
|
||||||
};
|
};
|
||||||
let mut info = SearchInfo::default();
|
let mut info = SearchInfo::default();
|
||||||
|
|
@ -172,16 +155,6 @@ pub fn parse_search_response(query: &str, body: &Value) -> SearchInfo {
|
||||||
.and_then(|c| c.get("sectionListRenderer"))
|
.and_then(|c| c.get("sectionListRenderer"))
|
||||||
.and_then(|c| c.get("contents"));
|
.and_then(|c| c.get("contents"));
|
||||||
|
|
||||||
// A non-empty response body whose primary results path is gone = a layout
|
|
||||||
// change; we'd otherwise silently return zero results. Log query_len, never
|
|
||||||
// the query.
|
|
||||||
if primary.is_none() && body.as_object().map(|m| !m.is_empty()).unwrap_or(false) {
|
|
||||||
log::warn!(
|
|
||||||
"search: primary results path missing (layout change?); 0 results for query_len={}",
|
|
||||||
query.len()
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
if let Some(sections) = primary.and_then(|v| v.as_array()) {
|
if let Some(sections) = primary.and_then(|v| v.as_array()) {
|
||||||
for section in sections {
|
for section in sections {
|
||||||
if let Some(items) = section
|
if let Some(items) = section
|
||||||
|
|
|
||||||
File diff suppressed because it is too large
Load diff
|
|
@ -6,8 +6,6 @@
|
||||||
// serviceIntegrityDimensions for poToken), POSTs to the right URL with
|
// serviceIntegrityDimensions for poToken), POSTs to the right URL with
|
||||||
// the right headers, returns the parsed JSON.
|
// the right headers, returns the parsed JSON.
|
||||||
|
|
||||||
use once_cell::sync::Lazy;
|
|
||||||
use parking_lot::RwLock;
|
|
||||||
use serde_json::{json, Map, Value};
|
use serde_json::{json, Map, Value};
|
||||||
|
|
||||||
use crate::downloader::request::Request;
|
use crate::downloader::request::Request;
|
||||||
|
|
@ -17,8 +15,7 @@ use crate::newpipe::NewPipe;
|
||||||
use crate::youtube::client_request::{build_envelope, InnertubeClientRequestInfo};
|
use crate::youtube::client_request::{build_envelope, InnertubeClientRequestInfo};
|
||||||
use crate::youtube::constants::*;
|
use crate::youtube::constants::*;
|
||||||
use crate::youtube::parsing::{
|
use crate::youtube::parsing::{
|
||||||
android_user_agent, ios_user_agent, mobile_post_headers, visionos_user_agent,
|
android_user_agent, ios_user_agent, mobile_post_headers, youtube_post_headers,
|
||||||
youtube_post_headers,
|
|
||||||
};
|
};
|
||||||
|
|
||||||
/// Builds a 12-char alphanumeric `cpn` (content playback nonce). NPE uses
|
/// Builds a 12-char alphanumeric `cpn` (content playback nonce). NPE uses
|
||||||
|
|
@ -75,161 +72,6 @@ fn envelope_to_body(envelope: Value) -> Map<String, Value> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A cached `visitorData` plus the instant it was fetched (for TTL expiry).
|
|
||||||
struct CachedVisitorData {
|
|
||||||
value: String,
|
|
||||||
at: std::time::Instant,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl CachedVisitorData {
|
|
||||||
fn is_fresh(&self) -> bool {
|
|
||||||
self.at.elapsed() < VISITOR_DATA_TTL
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// How long a cached `visitorData` is reused before a fresh one is fetched.
|
|
||||||
/// Upstream fetches a brand-new visitorData on EVERY call; we cache to avoid a
|
|
||||||
/// round-trip per video, but bound the reuse so (a) traffic isn't pinned to a
|
|
||||||
/// single visitor identity indefinitely (rate-flag risk) and (b) a
|
|
||||||
/// subtly-bad-but-non-empty value can't ride the PRIMARY android reel call —
|
|
||||||
/// whose failure aborts extraction — forever. 15 min is a small fraction of
|
|
||||||
/// YouTube's own visitorData lifetime while still rotating regularly.
|
|
||||||
const VISITOR_DATA_TTL: std::time::Duration = std::time::Duration::from_secs(15 * 60);
|
|
||||||
|
|
||||||
/// Process-wide cached `visitorData`. See [`CachedVisitorData`] / TTL above.
|
|
||||||
static VISITOR_DATA_CACHE: Lazy<RwLock<Option<CachedVisitorData>>> =
|
|
||||||
Lazy::new(|| RwLock::new(None));
|
|
||||||
|
|
||||||
/// Best-effort `visitorData` for the InnerTube client context.
|
|
||||||
///
|
|
||||||
/// Upstream (`YoutubeStreamHelper`) attaches a freshly-fetched visitorData to
|
|
||||||
/// every non-token reel/web/ios/visionos call — *"We must always pass a valid
|
|
||||||
/// visitorData to get valid player responses"* — via
|
|
||||||
/// `YoutubeParsingHelper.getVisitorDataFromInnertube` (POST to the
|
|
||||||
/// `visitor_id` endpoint, read `responseContext.visitorData`).
|
|
||||||
///
|
|
||||||
/// DELIBERATE DEVIATION (Loop-2 best-effort mandate): upstream THROWS when
|
|
||||||
/// visitorData can't be obtained, which would fail the whole extraction.
|
|
||||||
/// strawcore's anonymous happy-path has always worked WITHOUT visitorData, so
|
|
||||||
/// any failure here returns `None` and the caller proceeds exactly as it does
|
|
||||||
/// today. This is a durability improvement that can never regress the current
|
|
||||||
/// path — sending visitorData when we can get it, silently skipping it when we
|
|
||||||
/// can't.
|
|
||||||
///
|
|
||||||
/// The cache is bounded by [`VISITOR_DATA_TTL`] and can be dropped early via
|
|
||||||
/// [`reset_visitor_data_cache`] (called when the PRIMARY android response is
|
|
||||||
/// rejected, so a poisoned value self-heals). Locking mirrors the Loop-1
|
|
||||||
/// `player_manager` discipline: the `RwLock` is NEVER held across the network.
|
|
||||||
/// read (fresh?) → (miss/expired) fetch unlocked → write. A rare concurrent
|
|
||||||
/// double-fetch simply stores one of two interchangeable values.
|
|
||||||
fn get_visitor_data(
|
|
||||||
info: &InnertubeClientRequestInfo,
|
|
||||||
localization: &Localization,
|
|
||||||
content_country: &ContentCountry,
|
|
||||||
headers: Vec<(String, String)>,
|
|
||||||
domain: &str,
|
|
||||||
) -> Option<String> {
|
|
||||||
// Serve a still-fresh cached value; an expired one falls through to refetch.
|
|
||||||
if let Some(cached) = VISITOR_DATA_CACHE.read().as_ref() {
|
|
||||||
if cached.is_fresh() && !visitor_force_stale() {
|
|
||||||
return Some(cached.value.clone());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
let env = build_envelope(info, localization, content_country, None);
|
|
||||||
let body = Value::Object(envelope_to_body(env));
|
|
||||||
let url = format!("{domain}visitor_id{DISABLE_PRETTY_PRINT_PARAM}");
|
|
||||||
// post_youtube returns Err on any non-200 / transport / parse failure →
|
|
||||||
// best-effort None (the caller then proceeds without visitorData). An
|
|
||||||
// expired-but-present cache entry is deliberately NOT served on a failed
|
|
||||||
// refetch: a stale visitorData is the thing we're rotating away from.
|
|
||||||
// WARN because losing visitorData weakens attestation and kneecaps the
|
|
||||||
// visionOS bot-wall survivor. post_youtube already DEBUG-logged the
|
|
||||||
// HTTP-level detail; this is the semantic degradation. `domain` is a
|
|
||||||
// static endpoint constant (no secrets).
|
|
||||||
let parsed = match post_youtube(&url, &body, headers) {
|
|
||||||
Ok(p) => p,
|
|
||||||
Err(_e) => {
|
|
||||||
log::warn!("visitorData unavailable ({domain}visitor_id fetch failed); proceeding without it (attestation/visionOS weakened)");
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
let Some(visitor) = visitor_data_from_response(&parsed) else {
|
|
||||||
log::warn!("visitorData unavailable ({domain}visitor_id response had no visitorData); proceeding without it");
|
|
||||||
return None;
|
|
||||||
};
|
|
||||||
*VISITOR_DATA_CACHE.write() = Some(CachedVisitorData {
|
|
||||||
value: visitor.clone(),
|
|
||||||
at: std::time::Instant::now(),
|
|
||||||
});
|
|
||||||
Some(visitor)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Drop the cached `visitorData` so the next extraction fetches a fresh one.
|
|
||||||
/// Called when the PRIMARY android response is rejected (decoy / playability
|
|
||||||
/// failure): those are exactly the symptoms a bad visitorData would produce on
|
|
||||||
/// the primary path, and since that failure aborts extraction the value would
|
|
||||||
/// otherwise never self-heal.
|
|
||||||
pub(crate) fn reset_visitor_data_cache() {
|
|
||||||
// Lifecycle INFO: the self-heal trigger. Covers both the primary-cascade
|
|
||||||
// path and the stream_metadata twin path.
|
|
||||||
log::info!("visitorData cache reset (primary response rejected; next fetch mints fresh)");
|
|
||||||
*VISITOR_DATA_CACHE.write() = None;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Pull `responseContext.visitorData` out of a visitor_id response, treating an
|
|
||||||
/// absent or empty value as failure (`None`) — the trigger for the best-effort
|
|
||||||
/// fallback to "no visitorData" (today's behavior).
|
|
||||||
fn visitor_data_from_response(parsed: &Value) -> Option<String> {
|
|
||||||
parsed
|
|
||||||
.get("responseContext")
|
|
||||||
.and_then(|v| v.get("visitorData"))
|
|
||||||
.and_then(|v| v.as_str())
|
|
||||||
.filter(|s| !s.is_empty())
|
|
||||||
.map(str::to_string)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Test-only override that forces a cached entry to be treated as stale,
|
|
||||||
/// exercising the TTL-expiry refetch branch deterministically (fabricating an
|
|
||||||
/// old `Instant` via `checked_sub` is process/host-uptime dependent). Compiled
|
|
||||||
/// out entirely in non-test builds.
|
|
||||||
#[cfg(test)]
|
|
||||||
static FORCE_VISITOR_STALE: std::sync::atomic::AtomicBool =
|
|
||||||
std::sync::atomic::AtomicBool::new(false);
|
|
||||||
|
|
||||||
#[inline]
|
|
||||||
fn visitor_force_stale() -> bool {
|
|
||||||
#[cfg(test)]
|
|
||||||
{
|
|
||||||
FORCE_VISITOR_STALE.load(std::sync::atomic::Ordering::Relaxed)
|
|
||||||
}
|
|
||||||
#[cfg(not(test))]
|
|
||||||
{
|
|
||||||
false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Test hook: pre-seed the visitorData cache with a fresh entry (at = now) so
|
|
||||||
/// the cache-hit read path can be exercised without a network round-trip.
|
|
||||||
#[cfg(test)]
|
|
||||||
fn seed_visitor_data_cache(v: &str) {
|
|
||||||
*VISITOR_DATA_CACHE.write() = Some(CachedVisitorData {
|
|
||||||
value: v.to_string(),
|
|
||||||
at: std::time::Instant::now(),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Test hook: read the current cached visitorData value (if any).
|
|
||||||
#[cfg(test)]
|
|
||||||
fn peek_visitor_data_cache() -> Option<String> {
|
|
||||||
VISITOR_DATA_CACHE.read().as_ref().map(|c| c.value.clone())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Test hook: force/clear the stale override (see [`FORCE_VISITOR_STALE`]).
|
|
||||||
#[cfg(test)]
|
|
||||||
fn set_force_visitor_stale(on: bool) {
|
|
||||||
FORCE_VISITOR_STALE.store(on, std::sync::atomic::Ordering::Relaxed);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// WEB-client metadata-only /player call — used for microformat +
|
/// WEB-client metadata-only /player call — used for microformat +
|
||||||
/// thumbnails only; never used as a stream URL source.
|
/// thumbnails only; never used as a stream URL source.
|
||||||
pub fn get_web_metadata_player_response(
|
pub fn get_web_metadata_player_response(
|
||||||
|
|
@ -238,18 +80,7 @@ pub fn get_web_metadata_player_response(
|
||||||
content_country: &ContentCountry,
|
content_country: &ContentCountry,
|
||||||
signature_timestamp: i32,
|
signature_timestamp: i32,
|
||||||
) -> Result<Value, ExtractionError> {
|
) -> Result<Value, ExtractionError> {
|
||||||
let mut info = InnertubeClientRequestInfo::of_web_client();
|
let info = InnertubeClientRequestInfo::of_web_client();
|
||||||
// Best-effort visitorData (upstream sends it on every non-token /player
|
|
||||||
// call). WEB uses the youtube.com domain + web headers for visitor_id.
|
|
||||||
if let Some(v) = get_visitor_data(
|
|
||||||
&info,
|
|
||||||
localization,
|
|
||||||
content_country,
|
|
||||||
youtube_post_headers(),
|
|
||||||
YOUTUBEI_V1_URL,
|
|
||||||
) {
|
|
||||||
info.client_info.visitor_data = Some(v);
|
|
||||||
}
|
|
||||||
let env = build_envelope(&info, localization, content_country, None);
|
let env = build_envelope(&info, localization, content_country, None);
|
||||||
let mut body = envelope_to_body(env);
|
let mut body = envelope_to_body(env);
|
||||||
add_player_body_fields(&mut body, video_id, &generate_content_playback_nonce());
|
add_player_body_fields(&mut body, video_id, &generate_content_playback_nonce());
|
||||||
|
|
@ -292,75 +123,14 @@ pub fn get_android_player_response(
|
||||||
|
|
||||||
/// ANDROID `/reel/reel_item_watch` fallback — used when no poToken is
|
/// ANDROID `/reel/reel_item_watch` fallback — used when no poToken is
|
||||||
/// available. Returns a `playerResponse`-shaped JSON wrapped inside the
|
/// available. Returns a `playerResponse`-shaped JSON wrapped inside the
|
||||||
/// reel response. Requests the FULL `playerResponse` (`$fields=playerResponse`)
|
/// reel response.
|
||||||
/// — the stream-extraction path needs `streamingData` and everything else.
|
|
||||||
pub fn get_android_reel_player_response(
|
pub fn get_android_reel_player_response(
|
||||||
video_id: &str,
|
video_id: &str,
|
||||||
localization: &Localization,
|
localization: &Localization,
|
||||||
content_country: &ContentCountry,
|
content_country: &ContentCountry,
|
||||||
cpn: &str,
|
cpn: &str,
|
||||||
) -> Result<Value, ExtractionError> {
|
) -> Result<Value, ExtractionError> {
|
||||||
reel_player_response_with_fields(video_id, localization, content_country, cpn, "playerResponse")
|
let info = InnertubeClientRequestInfo::of_android_client();
|
||||||
}
|
|
||||||
|
|
||||||
/// Metadata-scoped variant of [`get_android_reel_player_response`] for the cheap
|
|
||||||
/// `stream_metadata` feed-enrich path. Requests a NARROWED `$fields` selector so
|
|
||||||
/// only `playabilityStatus` + `videoDetails` come back (~3-6 KB) instead of the
|
|
||||||
/// full ~150-500 KB reel body carrying every adaptive format's ~1.5 KB URL — the
|
|
||||||
/// feed path reads nothing outside those two subtrees (S3).
|
|
||||||
///
|
|
||||||
/// BEST-EFFORT, with a full-fetch fallback owned by the caller
|
|
||||||
/// (`stream_extractor::fetch_android_metadata`): the `reel_item_watch` endpoint's
|
|
||||||
/// acceptance of the NESTED `$fields` selector (`playerResponse.videoDetails`) is
|
|
||||||
/// not guaranteed. If it honors the selector we save the bytes; if it rejects it
|
|
||||||
/// (400 → `Err` here) or returns a shape without `videoDetails`, the caller falls
|
|
||||||
/// back to the full reel fetch, so feed enrichment can NEVER regress. The
|
|
||||||
/// dot-form nested selector matches the spelling upstream/`get_web_metadata_player_response`
|
|
||||||
/// already use against the /player endpoint (`videoDetails.thumbnail.thumbnails`),
|
|
||||||
/// so the syntax itself is proven — only this endpoint's honoring of it is not.
|
|
||||||
pub fn get_android_reel_player_response_metadata(
|
|
||||||
video_id: &str,
|
|
||||||
localization: &Localization,
|
|
||||||
content_country: &ContentCountry,
|
|
||||||
cpn: &str,
|
|
||||||
) -> Result<Value, ExtractionError> {
|
|
||||||
reel_player_response_with_fields(
|
|
||||||
video_id,
|
|
||||||
localization,
|
|
||||||
content_country,
|
|
||||||
cpn,
|
|
||||||
"playerResponse.playabilityStatus,playerResponse.videoDetails",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Shared body of the two reel helpers. `fields` is the `$fields` selector: the
|
|
||||||
/// full path passes `"playerResponse"` (byte-for-byte today's request), the
|
|
||||||
/// metadata path passes the narrowed nested selector. Everything else — the
|
|
||||||
/// best-effort visitorData, the envelope, the `playerRequest`/body shape, the
|
|
||||||
/// headers — is identical across both.
|
|
||||||
fn reel_player_response_with_fields(
|
|
||||||
video_id: &str,
|
|
||||||
localization: &Localization,
|
|
||||||
content_country: &ContentCountry,
|
|
||||||
cpn: &str,
|
|
||||||
fields: &str,
|
|
||||||
) -> Result<Value, ExtractionError> {
|
|
||||||
let mut info = InnertubeClientRequestInfo::of_android_client();
|
|
||||||
let ua = android_user_agent(content_country);
|
|
||||||
// Best-effort visitorData (upstream sends it on the reel call too). The reel
|
|
||||||
// path uses the gapis domain + android mobile headers for visitor_id.
|
|
||||||
// NOTE: the reel body shape is otherwise UNCHANGED from today (the upstream
|
|
||||||
// playerRequest/disablePlayerResponse realignment — recon W-1 — is out of
|
|
||||||
// scope for this pass; only touch it if a decoy/empty regression appears).
|
|
||||||
if let Some(v) = get_visitor_data(
|
|
||||||
&info,
|
|
||||||
localization,
|
|
||||||
content_country,
|
|
||||||
mobile_post_headers(&ua),
|
|
||||||
YOUTUBEI_V1_GAPIS_URL,
|
|
||||||
) {
|
|
||||||
info.client_info.visitor_data = Some(v);
|
|
||||||
}
|
|
||||||
let env = build_envelope(&info, localization, content_country, None);
|
let env = build_envelope(&info, localization, content_country, None);
|
||||||
let mut body = envelope_to_body(env);
|
let mut body = envelope_to_body(env);
|
||||||
body.insert(
|
body.insert(
|
||||||
|
|
@ -372,9 +142,10 @@ fn reel_player_response_with_fields(
|
||||||
);
|
);
|
||||||
add_player_body_fields(&mut body, video_id, cpn);
|
add_player_body_fields(&mut body, video_id, cpn);
|
||||||
let url = format!(
|
let url = format!(
|
||||||
"{YOUTUBEI_V1_GAPIS_URL}reel/reel_item_watch{DISABLE_PRETTY_PRINT_PARAM}&t={t}&id={video_id}&$fields={fields}",
|
"{YOUTUBEI_V1_GAPIS_URL}reel/reel_item_watch{DISABLE_PRETTY_PRINT_PARAM}&t={t}&id={video_id}&$fields=playerResponse",
|
||||||
t = generate_content_playback_nonce()
|
t = generate_content_playback_nonce()
|
||||||
);
|
);
|
||||||
|
let ua = android_user_agent(content_country);
|
||||||
post_youtube(&url, &Value::Object(body), mobile_post_headers(&ua))
|
post_youtube(&url, &Value::Object(body), mobile_post_headers(&ua))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -392,20 +163,8 @@ pub fn get_ios_player_response(
|
||||||
visitor_data: Option<&str>,
|
visitor_data: Option<&str>,
|
||||||
) -> Result<Value, ExtractionError> {
|
) -> Result<Value, ExtractionError> {
|
||||||
let mut info = InnertubeClientRequestInfo::of_ios_client();
|
let mut info = InnertubeClientRequestInfo::of_ios_client();
|
||||||
let ua = ios_user_agent(content_country);
|
|
||||||
// Caller-supplied visitorData (paired with a poToken) wins; otherwise fetch
|
|
||||||
// it best-effort. iOS uses the youtube.com domain for visitor_id (upstream
|
|
||||||
// getIosPlayerResponse) even though the /player call goes to gapis.
|
|
||||||
if let Some(v) = visitor_data {
|
if let Some(v) = visitor_data {
|
||||||
info.client_info.visitor_data = Some(v.into());
|
info.client_info.visitor_data = Some(v.into());
|
||||||
} else if let Some(v) = get_visitor_data(
|
|
||||||
&info,
|
|
||||||
localization,
|
|
||||||
content_country,
|
|
||||||
mobile_post_headers(&ua),
|
|
||||||
YOUTUBEI_V1_URL,
|
|
||||||
) {
|
|
||||||
info.client_info.visitor_data = Some(v);
|
|
||||||
}
|
}
|
||||||
let env = build_envelope(&info, localization, content_country, None);
|
let env = build_envelope(&info, localization, content_country, None);
|
||||||
let mut body = envelope_to_body(env);
|
let mut body = envelope_to_body(env);
|
||||||
|
|
@ -417,42 +176,7 @@ pub fn get_ios_player_response(
|
||||||
"{YOUTUBEI_V1_GAPIS_URL}player{DISABLE_PRETTY_PRINT_PARAM}&t={t}&id={video_id}",
|
"{YOUTUBEI_V1_GAPIS_URL}player{DISABLE_PRETTY_PRINT_PARAM}&t={t}&id={video_id}",
|
||||||
t = generate_content_playback_nonce()
|
t = generate_content_playback_nonce()
|
||||||
);
|
);
|
||||||
post_youtube(&url, &Value::Object(body), mobile_post_headers(&ua))
|
let ua = ios_user_agent(content_country);
|
||||||
}
|
|
||||||
|
|
||||||
/// visionOS /player call (client 101). Best-effort 4th client that recovers
|
|
||||||
/// higher-than-360p streams for SABR-only / "made for kids" videos. Mirrors
|
|
||||||
/// NPE `getVisionOsPlayerResponse`: hits the gapis /player endpoint with the
|
|
||||||
/// visionOS mobile header set, carries a best-effort visitorData, and NO
|
|
||||||
/// poToken.
|
|
||||||
pub fn get_visionos_player_response(
|
|
||||||
video_id: &str,
|
|
||||||
localization: &Localization,
|
|
||||||
content_country: &ContentCountry,
|
|
||||||
cpn: &str,
|
|
||||||
) -> Result<Value, ExtractionError> {
|
|
||||||
let mut info = InnertubeClientRequestInfo::of_visionos_client();
|
|
||||||
let ua = visionos_user_agent(content_country);
|
|
||||||
// visionOS returns valid responses ONLY with a visitorData — but still
|
|
||||||
// best-effort: on failure we proceed without it (the whole visionOS fetch
|
|
||||||
// is itself swallowed best-effort by the caller). Uses the youtube.com
|
|
||||||
// domain for visitor_id (upstream), gapis for /player.
|
|
||||||
if let Some(v) = get_visitor_data(
|
|
||||||
&info,
|
|
||||||
localization,
|
|
||||||
content_country,
|
|
||||||
mobile_post_headers(&ua),
|
|
||||||
YOUTUBEI_V1_URL,
|
|
||||||
) {
|
|
||||||
info.client_info.visitor_data = Some(v);
|
|
||||||
}
|
|
||||||
let env = build_envelope(&info, localization, content_country, None);
|
|
||||||
let mut body = envelope_to_body(env);
|
|
||||||
add_player_body_fields(&mut body, video_id, cpn);
|
|
||||||
let url = format!(
|
|
||||||
"{YOUTUBEI_V1_GAPIS_URL}player{DISABLE_PRETTY_PRINT_PARAM}&t={t}&id={video_id}",
|
|
||||||
t = generate_content_playback_nonce()
|
|
||||||
);
|
|
||||||
post_youtube(&url, &Value::Object(body), mobile_post_headers(&ua))
|
post_youtube(&url, &Value::Object(body), mobile_post_headers(&ua))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -471,36 +195,14 @@ fn post_youtube(
|
||||||
}
|
}
|
||||||
let resp = downloader.execute(builder.build())?;
|
let resp = downloader.execute(builder.build())?;
|
||||||
if resp.response_code() != 200 {
|
if resp.response_code() != 200 {
|
||||||
// Privacy: the full URL carries `id=<videoId>` in the query — never
|
|
||||||
// put it in an error string (they reach Kotlin exception messages
|
|
||||||
// and logs). Scheme+host+path identifies the endpoint just fine.
|
|
||||||
let endpoint = url.split(['?', '#']).next().unwrap_or(url);
|
|
||||||
// body_len is the bot-wall tell (an HTML challenge page has a
|
|
||||||
// distinctive size vs a JSON error). DEBUG — per-request breadcrumb;
|
|
||||||
// the semantic WARN is emitted by the caller.
|
|
||||||
log::debug!(
|
|
||||||
"post_youtube {endpoint} → HTTP {} ({}B body)",
|
|
||||||
resp.response_code(),
|
|
||||||
resp.response_body().len()
|
|
||||||
);
|
|
||||||
return Err(ExtractionError::Network(NetworkError::Transport(format!(
|
return Err(ExtractionError::Network(NetworkError::Transport(format!(
|
||||||
"HTTP {} from {endpoint}",
|
"HTTP {} from {url}",
|
||||||
resp.response_code()
|
resp.response_code()
|
||||||
))));
|
))));
|
||||||
}
|
}
|
||||||
match serde_json::from_str(resp.response_body()) {
|
let parsed: Value = serde_json::from_str(resp.response_body())
|
||||||
Ok(parsed) => Ok(parsed),
|
.map_err(|e| ExtractionError::Parsing(ParsingError::JsonShape(e.to_string())))?;
|
||||||
Err(e) => {
|
Ok(parsed)
|
||||||
let endpoint = url.split(['?', '#']).next().unwrap_or(url);
|
|
||||||
log::debug!(
|
|
||||||
"post_youtube {endpoint} → 200 but body was not JSON ({}B): {e}",
|
|
||||||
resp.response_body().len()
|
|
||||||
);
|
|
||||||
Err(ExtractionError::Parsing(ParsingError::JsonShape(
|
|
||||||
e.to_string(),
|
|
||||||
)))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|
@ -525,133 +227,4 @@ mod tests {
|
||||||
let b = generate_content_playback_nonce();
|
let b = generate_content_playback_nonce();
|
||||||
assert_ne!(a, b);
|
assert_ne!(a, b);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn visitor_data_parsed_from_response() {
|
|
||||||
let resp = json!({
|
|
||||||
"responseContext": { "visitorData": "Cgs1ZG1abc==" },
|
|
||||||
"other": 1
|
|
||||||
});
|
|
||||||
assert_eq!(
|
|
||||||
visitor_data_from_response(&resp).as_deref(),
|
|
||||||
Some("Cgs1ZG1abc==")
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn visitor_data_absent_or_empty_is_none_triggering_fallback() {
|
|
||||||
// Missing responseContext → None (→ caller proceeds without visitorData,
|
|
||||||
// i.e. exactly today's behavior).
|
|
||||||
assert!(visitor_data_from_response(&json!({})).is_none());
|
|
||||||
// Present but empty string → None (empty is not a valid visitorData).
|
|
||||||
let resp = json!({ "responseContext": { "visitorData": "" } });
|
|
||||||
assert!(visitor_data_from_response(&resp).is_none());
|
|
||||||
// responseContext without the field → None.
|
|
||||||
let resp = json!({ "responseContext": { "foo": "bar" } });
|
|
||||||
assert!(visitor_data_from_response(&resp).is_none());
|
|
||||||
}
|
|
||||||
|
|
||||||
// ---- visitorData cache: fresh-hit / TTL-expiry / reset ----------------
|
|
||||||
//
|
|
||||||
// These touch the process-global visitorData cache and (for the refetch
|
|
||||||
// test) the global Downloader, so they serialize on DOWNLOADER_TEST_LOCK,
|
|
||||||
// shared with the player_manager test module.
|
|
||||||
|
|
||||||
use crate::downloader::request::Request;
|
|
||||||
use crate::downloader::response::Response;
|
|
||||||
use crate::downloader::Downloader;
|
|
||||||
use crate::newpipe::DOWNLOADER_TEST_LOCK;
|
|
||||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
|
||||||
use std::sync::Arc;
|
|
||||||
|
|
||||||
/// Serves any `visitor_id` POST with a fixed visitorData; counts requests.
|
|
||||||
struct VisitorStub {
|
|
||||||
visitor: String,
|
|
||||||
requests: AtomicUsize,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Downloader for VisitorStub {
|
|
||||||
fn execute(&self, request: Request) -> Result<Response, NetworkError> {
|
|
||||||
self.requests.fetch_add(1, Ordering::SeqCst);
|
|
||||||
let url = request.url().to_string();
|
|
||||||
if url.contains("visitor_id") {
|
|
||||||
let body = format!(
|
|
||||||
r#"{{"responseContext":{{"visitorData":"{}"}}}}"#,
|
|
||||||
self.visitor
|
|
||||||
);
|
|
||||||
Ok(Response::new(200, "OK", Default::default(), body, url))
|
|
||||||
} else {
|
|
||||||
Err(NetworkError::Transport("visitor-stub: unexpected request".into()))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn fresh_cached_visitor_data_served_without_network() {
|
|
||||||
let _g = DOWNLOADER_TEST_LOCK.lock();
|
|
||||||
set_force_visitor_stale(false);
|
|
||||||
reset_visitor_data_cache();
|
|
||||||
seed_visitor_data_cache("FRESH_VD");
|
|
||||||
// No downloader is needed: a fresh cache hit returns before any fetch.
|
|
||||||
let info = InnertubeClientRequestInfo::of_android_client();
|
|
||||||
let got = get_visitor_data(
|
|
||||||
&info,
|
|
||||||
&Localization::default(),
|
|
||||||
&ContentCountry::default(),
|
|
||||||
mobile_post_headers("ua/1.0"),
|
|
||||||
YOUTUBEI_V1_GAPIS_URL,
|
|
||||||
);
|
|
||||||
assert_eq!(got.as_deref(), Some("FRESH_VD"));
|
|
||||||
reset_visitor_data_cache();
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn expired_visitor_data_triggers_refetch() {
|
|
||||||
let _g = DOWNLOADER_TEST_LOCK.lock();
|
|
||||||
let stub = Arc::new(VisitorStub {
|
|
||||||
visitor: "REFRESHED_VD".into(),
|
|
||||||
requests: AtomicUsize::new(0),
|
|
||||||
});
|
|
||||||
NewPipe::init(stub.clone() as Arc<dyn Downloader>);
|
|
||||||
|
|
||||||
reset_visitor_data_cache();
|
|
||||||
seed_visitor_data_cache("STALE_VD"); // fresh instant…
|
|
||||||
set_force_visitor_stale(true); // …but forced past its TTL
|
|
||||||
|
|
||||||
let info = InnertubeClientRequestInfo::of_android_client();
|
|
||||||
let got = get_visitor_data(
|
|
||||||
&info,
|
|
||||||
&Localization::default(),
|
|
||||||
&ContentCountry::default(),
|
|
||||||
mobile_post_headers("ua/1.0"),
|
|
||||||
YOUTUBEI_V1_GAPIS_URL,
|
|
||||||
);
|
|
||||||
|
|
||||||
assert_eq!(got.as_deref(), Some("REFRESHED_VD"), "expired entry must refetch");
|
|
||||||
assert_eq!(
|
|
||||||
stub.requests.load(Ordering::SeqCst),
|
|
||||||
1,
|
|
||||||
"exactly one visitor_id refetch"
|
|
||||||
);
|
|
||||||
// The refetched value replaced the stale one, stamped fresh.
|
|
||||||
set_force_visitor_stale(false);
|
|
||||||
assert_eq!(peek_visitor_data_cache().as_deref(), Some("REFRESHED_VD"));
|
|
||||||
|
|
||||||
set_force_visitor_stale(false);
|
|
||||||
reset_visitor_data_cache();
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn reset_clears_visitor_data_cache_for_self_heal() {
|
|
||||||
// Fix 2b mechanism: on a primary-android rejection the extractor calls
|
|
||||||
// reset_visitor_data_cache() so a poisoned value doesn't ride the
|
|
||||||
// primary reel call forever.
|
|
||||||
let _g = DOWNLOADER_TEST_LOCK.lock();
|
|
||||||
set_force_visitor_stale(false);
|
|
||||||
reset_visitor_data_cache();
|
|
||||||
seed_visitor_data_cache("POISONED_VD");
|
|
||||||
assert_eq!(peek_visitor_data_cache().as_deref(), Some("POISONED_VD"));
|
|
||||||
reset_visitor_data_cache();
|
|
||||||
assert!(peek_visitor_data_cache().is_none());
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,297 +0,0 @@
|
||||||
// Minimal reader for an itag format's `xtags` field → AudioTrackType.
|
|
||||||
//
|
|
||||||
// Mirrors NPE YoutubeParsingHelper.extractAudioTrackType (commits b0bca7e7 +
|
|
||||||
// e3479a7c, 2026-06-01), which replaced the old `audioTrack.audioIsDefault`
|
|
||||||
// heuristic with the authoritative `acont` value carried in the format's
|
|
||||||
// `xtags` blob.
|
|
||||||
//
|
|
||||||
// `xtags` is a base64url-encoded `youtube.video.XTags` protobuf (upstream
|
|
||||||
// proto/youtube/video/xtags.proto):
|
|
||||||
//
|
|
||||||
// message KeyValuePair { optional string key = 1; optional string value = 2; }
|
|
||||||
// message XTags { repeated KeyValuePair xtags = 1; }
|
|
||||||
//
|
|
||||||
// The structure is trivial (two nested length-delimited string fields), so we
|
|
||||||
// read it with a ~40-line varint/field walker instead of pulling in a protobuf
|
|
||||||
// crate — matching the codebase's minimal-dependency approach. Everything is
|
|
||||||
// best-effort: any malformed input (bad base64, truncated protobuf, unknown
|
|
||||||
// `acont` value) yields `None`, and the caller falls back to the legacy
|
|
||||||
// `audioIsDefault` heuristic — never a panic, never a break.
|
|
||||||
|
|
||||||
use crate::stream::AudioTrackType;
|
|
||||||
|
|
||||||
/// Decode a format's `xtags` string and return its audio track type, mirroring
|
|
||||||
/// NPE `extractAudioTrackType`. Returns `None` when the blob is absent-shaped,
|
|
||||||
/// undecodable, carries no `acont` key, or maps to an unknown value.
|
|
||||||
pub fn extract_audio_track_type(xtags: &str) -> Option<AudioTrackType> {
|
|
||||||
let bytes = base64url_decode(xtags)?;
|
|
||||||
let acont = xtags_find(&bytes, "acont")?;
|
|
||||||
match acont.as_str() {
|
|
||||||
"original" => Some(AudioTrackType::Original),
|
|
||||||
// NPE maps both "dubbed" and "dubbed-auto" → DUBBED.
|
|
||||||
"dubbed" | "dubbed-auto" => Some(AudioTrackType::Dubbed),
|
|
||||||
"descriptive" => Some(AudioTrackType::Descriptive),
|
|
||||||
"secondary" => Some(AudioTrackType::Secondary),
|
|
||||||
_ => None,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Walk the top-level `XTags` message, returning the `value` of the first
|
|
||||||
/// `KeyValuePair` (field 1) whose `key` equals `wanted`. Unknown fields and
|
|
||||||
/// non-string wire types are skipped; any framing error bails to `None`.
|
|
||||||
fn xtags_find(buf: &[u8], wanted: &str) -> Option<String> {
|
|
||||||
let mut i = 0;
|
|
||||||
while i < buf.len() {
|
|
||||||
let (tag, adv) = read_varint(buf, i)?;
|
|
||||||
i += adv;
|
|
||||||
let field = tag >> 3;
|
|
||||||
match wire_type(tag) {
|
|
||||||
// Length-delimited: field 1 is a KeyValuePair submessage.
|
|
||||||
2 => {
|
|
||||||
let payload = read_len_delimited(buf, &mut i)?;
|
|
||||||
if field == 1 {
|
|
||||||
if let Some((k, v)) = parse_key_value_pair(payload) {
|
|
||||||
if k.as_deref() == Some(wanted) {
|
|
||||||
return v;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
wt => skip_scalar(buf, &mut i, wt)?,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
None
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Parse a `KeyValuePair` submessage into (key, value); either may be absent.
|
|
||||||
fn parse_key_value_pair(buf: &[u8]) -> Option<(Option<String>, Option<String>)> {
|
|
||||||
let mut key = None;
|
|
||||||
let mut val = None;
|
|
||||||
let mut i = 0;
|
|
||||||
while i < buf.len() {
|
|
||||||
let (tag, adv) = read_varint(buf, i)?;
|
|
||||||
i += adv;
|
|
||||||
let field = tag >> 3;
|
|
||||||
match wire_type(tag) {
|
|
||||||
2 => {
|
|
||||||
let payload = read_len_delimited(buf, &mut i)?;
|
|
||||||
let s = std::str::from_utf8(payload).ok()?.to_string();
|
|
||||||
match field {
|
|
||||||
1 => key = Some(s),
|
|
||||||
2 => val = Some(s),
|
|
||||||
_ => {}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
wt => skip_scalar(buf, &mut i, wt)?,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Some((key, val))
|
|
||||||
}
|
|
||||||
|
|
||||||
#[inline]
|
|
||||||
fn wire_type(tag: u64) -> u8 {
|
|
||||||
(tag & 0x7) as u8
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Read a length-delimited (wire type 2) payload, advancing `*i` past it.
|
|
||||||
fn read_len_delimited<'a>(buf: &'a [u8], i: &mut usize) -> Option<&'a [u8]> {
|
|
||||||
let (len, adv) = read_varint(buf, *i)?;
|
|
||||||
*i += adv;
|
|
||||||
// `usize::try_from` (not `as usize`) so a >4 GiB length can't wrap on a
|
|
||||||
// 32-bit target (armeabi-v7a is a real Straw ABI) — it fails closed instead.
|
|
||||||
let len = usize::try_from(len).ok()?;
|
|
||||||
let end = i.checked_add(len)?;
|
|
||||||
if end > buf.len() {
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
let payload = &buf[*i..end];
|
|
||||||
*i = end;
|
|
||||||
Some(payload)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Advance `*i` past a non-length-delimited field. Groups (3/4) and any
|
|
||||||
/// unrecognized wire type bail out — they never appear in a well-formed XTags.
|
|
||||||
fn skip_scalar(buf: &[u8], i: &mut usize, wire: u8) -> Option<()> {
|
|
||||||
match wire {
|
|
||||||
0 => {
|
|
||||||
let (_, adv) = read_varint(buf, *i)?;
|
|
||||||
*i += adv;
|
|
||||||
}
|
|
||||||
1 => {
|
|
||||||
*i = i.checked_add(8)?;
|
|
||||||
if *i > buf.len() {
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
5 => {
|
|
||||||
*i = i.checked_add(4)?;
|
|
||||||
if *i > buf.len() {
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
_ => return None,
|
|
||||||
}
|
|
||||||
Some(())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Read a base-128 varint at `start`; returns (value, bytes_consumed). Caps at
|
|
||||||
/// 10 bytes (64 bits) so a malformed run can't spin.
|
|
||||||
fn read_varint(buf: &[u8], start: usize) -> Option<(u64, usize)> {
|
|
||||||
let mut result: u64 = 0;
|
|
||||||
let mut shift: u32 = 0;
|
|
||||||
let mut i = start;
|
|
||||||
loop {
|
|
||||||
if i >= buf.len() || shift >= 64 {
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
let byte = buf[i];
|
|
||||||
result |= u64::from(byte & 0x7f) << shift;
|
|
||||||
i += 1;
|
|
||||||
if byte & 0x80 == 0 {
|
|
||||||
return Some((result, i - start));
|
|
||||||
}
|
|
||||||
shift += 7;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Strict base64url (RFC 4648 §5, no padding required) decoder. Rejects any
|
|
||||||
/// non-url-safe byte (`+`, `/`, whitespace) → `None`, matching upstream's use
|
|
||||||
/// of `Base64.getUrlDecoder()`. Stops at the first `=` padding byte.
|
|
||||||
fn base64url_decode(s: &str) -> Option<Vec<u8>> {
|
|
||||||
fn sextet(c: u8) -> Option<u8> {
|
|
||||||
match c {
|
|
||||||
b'A'..=b'Z' => Some(c - b'A'),
|
|
||||||
b'a'..=b'z' => Some(c - b'a' + 26),
|
|
||||||
b'0'..=b'9' => Some(c - b'0' + 52),
|
|
||||||
b'-' => Some(62),
|
|
||||||
b'_' => Some(63),
|
|
||||||
_ => None,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if s.is_empty() {
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
let mut out = Vec::with_capacity(s.len() * 3 / 4 + 3);
|
|
||||||
let mut acc: u32 = 0;
|
|
||||||
let mut bits: u32 = 0;
|
|
||||||
for &c in s.as_bytes() {
|
|
||||||
if c == b'=' {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
let v = u32::from(sextet(c)?);
|
|
||||||
acc = (acc << 6) | v;
|
|
||||||
bits += 6;
|
|
||||||
if bits >= 8 {
|
|
||||||
bits -= 8;
|
|
||||||
out.push((acc >> bits) as u8);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Some(out)
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
use super::*;
|
|
||||||
|
|
||||||
// Independently-generated (Python urlsafe_b64encode of a hand-built XTags
|
|
||||||
// protobuf) so these vectors are an external oracle, not a round-trip of
|
|
||||||
// our own encoder. See the Loop-2 build notes for the generator.
|
|
||||||
const XT_ORIGINAL: &str = "ChEKBWFjb250EghvcmlnaW5hbA";
|
|
||||||
const XT_DUBBED: &str = "Cg8KBWFjb250EgZkdWJiZWQ";
|
|
||||||
const XT_DUBBED_AUTO: &str = "ChQKBWFjb250EgtkdWJiZWQtYXV0bw";
|
|
||||||
const XT_DESCRIPTIVE: &str = "ChQKBWFjb250EgtkZXNjcmlwdGl2ZQ";
|
|
||||||
const XT_SECONDARY: &str = "ChIKBWFjb250EglzZWNvbmRhcnk";
|
|
||||||
// Two pairs: {lang=en},{acont=descriptive} — proves we scan past a
|
|
||||||
// non-matching pair to find `acont`.
|
|
||||||
const XT_MULTI: &str = "CgoKBGxhbmcSAmVuChQKBWFjb250EgtkZXNjcmlwdGl2ZQ";
|
|
||||||
// Single pair {lang=en} — no `acont` key.
|
|
||||||
const XT_NO_ACONT: &str = "CgoKBGxhbmcSAmVu";
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn maps_each_known_acont_value() {
|
|
||||||
assert_eq!(
|
|
||||||
extract_audio_track_type(XT_ORIGINAL),
|
|
||||||
Some(AudioTrackType::Original)
|
|
||||||
);
|
|
||||||
assert_eq!(
|
|
||||||
extract_audio_track_type(XT_DUBBED),
|
|
||||||
Some(AudioTrackType::Dubbed)
|
|
||||||
);
|
|
||||||
assert_eq!(
|
|
||||||
extract_audio_track_type(XT_DUBBED_AUTO),
|
|
||||||
Some(AudioTrackType::Dubbed)
|
|
||||||
);
|
|
||||||
assert_eq!(
|
|
||||||
extract_audio_track_type(XT_DESCRIPTIVE),
|
|
||||||
Some(AudioTrackType::Descriptive)
|
|
||||||
);
|
|
||||||
assert_eq!(
|
|
||||||
extract_audio_track_type(XT_SECONDARY),
|
|
||||||
Some(AudioTrackType::Secondary)
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn finds_acont_among_multiple_pairs() {
|
|
||||||
assert_eq!(
|
|
||||||
extract_audio_track_type(XT_MULTI),
|
|
||||||
Some(AudioTrackType::Descriptive)
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn absent_acont_key_yields_none() {
|
|
||||||
assert_eq!(extract_audio_track_type(XT_NO_ACONT), None);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn malformed_input_yields_none_never_panics() {
|
|
||||||
// Empty, non-base64url chars, valid base64url but not a protobuf,
|
|
||||||
// and a truncated length-delimited frame.
|
|
||||||
assert_eq!(extract_audio_track_type(""), None);
|
|
||||||
assert_eq!(extract_audio_track_type("!!!not base64!!!"), None);
|
|
||||||
assert_eq!(extract_audio_track_type("++//"), None); // standard-b64 chars rejected
|
|
||||||
assert_eq!(extract_audio_track_type("Zm9vYmFy"), None); // "foobar", valid b64url, junk proto
|
|
||||||
// 0x0A (field1,LEN) claiming length 0x7F with no body → truncated.
|
|
||||||
assert_eq!(extract_audio_track_type("Cn8"), None);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn unknown_acont_value_yields_none() {
|
|
||||||
// {acont=weird} — well-formed protobuf, value not in the enum.
|
|
||||||
// ld(1, ld(1,"acont")+ld(2,"weird"))
|
|
||||||
let bytes = {
|
|
||||||
let kv = [
|
|
||||||
&[0x0a, 0x05][..],
|
|
||||||
b"acont",
|
|
||||||
&[0x12, 0x05],
|
|
||||||
b"weird",
|
|
||||||
]
|
|
||||||
.concat();
|
|
||||||
let mut top = vec![0x0a, kv.len() as u8];
|
|
||||||
top.extend_from_slice(&kv);
|
|
||||||
top
|
|
||||||
};
|
|
||||||
// sanity: our decoder round-trips what we assert on
|
|
||||||
let b64 = {
|
|
||||||
// encode without padding, url-safe
|
|
||||||
const A: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_";
|
|
||||||
let mut out = String::new();
|
|
||||||
for chunk in bytes.chunks(3) {
|
|
||||||
let b = [
|
|
||||||
chunk[0],
|
|
||||||
*chunk.get(1).unwrap_or(&0),
|
|
||||||
*chunk.get(2).unwrap_or(&0),
|
|
||||||
];
|
|
||||||
let n = (u32::from(b[0]) << 16) | (u32::from(b[1]) << 8) | u32::from(b[2]);
|
|
||||||
let take = chunk.len() + 1;
|
|
||||||
for k in 0..take {
|
|
||||||
out.push(A[((n >> (18 - 6 * k)) & 0x3f) as usize] as char);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
out
|
|
||||||
};
|
|
||||||
assert_eq!(base64url_decode(&b64).unwrap(), bytes);
|
|
||||||
assert_eq!(extract_audio_track_type(&b64), None);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -9,22 +9,16 @@
|
||||||
|
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use strawcore_core::downloader::request::Request;
|
use strawcore::downloader::request::Request;
|
||||||
use strawcore_core::downloader::ReqwestDownloader;
|
use strawcore::downloader::ReqwestDownloader;
|
||||||
use strawcore_core::exceptions::NetworkError;
|
use strawcore::exceptions::NetworkError;
|
||||||
use strawcore_core::localization::{ContentCountry, Localization};
|
use strawcore::localization::{ContentCountry, Localization};
|
||||||
use strawcore_core::{Downloader, NewPipe};
|
use strawcore::{Downloader, NewPipe};
|
||||||
|
|
||||||
/// The Downloader trait dropped its `get` convenience (commit f917e4a);
|
|
||||||
/// mirror it here so the suite reads as before.
|
|
||||||
fn get(dl: &impl Downloader, url: &str) -> Result<strawcore_core::Response, NetworkError> {
|
|
||||||
dl.execute(Request::get(url).build())
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn get_through_default_downloader() {
|
fn get_through_default_downloader() {
|
||||||
let dl = ReqwestDownloader::new().expect("build downloader");
|
let dl = ReqwestDownloader::new().expect("build downloader");
|
||||||
let resp = get(&dl, "https://httpbin.org/get").expect("transport");
|
let resp = dl.get("https://httpbin.org/get").expect("transport");
|
||||||
assert_eq!(resp.response_code(), 200);
|
assert_eq!(resp.response_code(), 200);
|
||||||
assert!(resp.response_body().contains("\"url\""));
|
assert!(resp.response_body().contains("\"url\""));
|
||||||
}
|
}
|
||||||
|
|
@ -32,7 +26,9 @@ fn get_through_default_downloader() {
|
||||||
#[test]
|
#[test]
|
||||||
fn latest_url_follows_redirects() {
|
fn latest_url_follows_redirects() {
|
||||||
let dl = ReqwestDownloader::new().expect("build downloader");
|
let dl = ReqwestDownloader::new().expect("build downloader");
|
||||||
let resp = get(&dl, "https://httpbin.org/redirect/3").expect("transport");
|
let resp = dl
|
||||||
|
.get("https://httpbin.org/redirect/3")
|
||||||
|
.expect("transport");
|
||||||
assert_eq!(resp.response_code(), 200);
|
assert_eq!(resp.response_code(), 200);
|
||||||
assert!(
|
assert!(
|
||||||
resp.latest_url().ends_with("/get"),
|
resp.latest_url().ends_with("/get"),
|
||||||
|
|
@ -44,14 +40,14 @@ fn latest_url_follows_redirects() {
|
||||||
#[test]
|
#[test]
|
||||||
fn non_2xx_returns_ok_not_err() {
|
fn non_2xx_returns_ok_not_err() {
|
||||||
let dl = ReqwestDownloader::new().expect("build downloader");
|
let dl = ReqwestDownloader::new().expect("build downloader");
|
||||||
let resp = get(&dl, "https://httpbin.org/status/404").expect("transport");
|
let resp = dl.get("https://httpbin.org/status/404").expect("transport");
|
||||||
assert_eq!(resp.response_code(), 404);
|
assert_eq!(resp.response_code(), 404);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn http_429_surfaces_as_recaptcha_err() {
|
fn http_429_surfaces_as_recaptcha_err() {
|
||||||
let dl = ReqwestDownloader::new().expect("build downloader");
|
let dl = ReqwestDownloader::new().expect("build downloader");
|
||||||
let err = get(&dl, "https://httpbin.org/status/429").expect_err("429 must be NetworkError");
|
let err = dl.get("https://httpbin.org/status/429").expect_err("429 must be NetworkError");
|
||||||
match err {
|
match err {
|
||||||
NetworkError::Recaptcha { url } => assert!(url.contains("/status/429")),
|
NetworkError::Recaptcha { url } => assert!(url.contains("/status/429")),
|
||||||
other => panic!("expected Recaptcha, got {other:?}"),
|
other => panic!("expected Recaptcha, got {other:?}"),
|
||||||
|
|
@ -76,8 +72,8 @@ fn localization_header_attached_when_enabled() {
|
||||||
#[test]
|
#[test]
|
||||||
fn header_keys_lowercased_in_response() {
|
fn header_keys_lowercased_in_response() {
|
||||||
let dl = ReqwestDownloader::new().expect("build downloader");
|
let dl = ReqwestDownloader::new().expect("build downloader");
|
||||||
let resp = get(&dl, "https://httpbin.org/get").expect("transport");
|
let resp = dl.get("https://httpbin.org/get").expect("transport");
|
||||||
for k in resp.response_headers().keys() {
|
for (k, _) in resp.response_headers() {
|
||||||
assert_eq!(k, &k.to_ascii_lowercase(), "header key {k} not lowercased");
|
assert_eq!(k, &k.to_ascii_lowercase(), "header key {k} not lowercased");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -92,9 +88,7 @@ fn newpipe_singleton_wires_downloader() {
|
||||||
);
|
);
|
||||||
|
|
||||||
let from_global = NewPipe::downloader().expect("downloader registered");
|
let from_global = NewPipe::downloader().expect("downloader registered");
|
||||||
let resp = from_global
|
let resp = from_global.get("https://httpbin.org/get").expect("transport");
|
||||||
.execute(Request::get("https://httpbin.org/get").build())
|
|
||||||
.expect("transport");
|
|
||||||
assert_eq!(resp.response_code(), 200);
|
assert_eq!(resp.response_code(), 200);
|
||||||
assert_eq!(NewPipe::preferred_localization().localization_code(), "en-GB");
|
assert_eq!(NewPipe::preferred_localization().localization_code(), "en-GB");
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -14,7 +14,7 @@
|
||||||
// * url_with_throttling_parameter_deobfuscated round-trip changes &n=
|
// * url_with_throttling_parameter_deobfuscated round-trip changes &n=
|
||||||
// and caches the result
|
// and caches the result
|
||||||
|
|
||||||
use strawcore_core::youtube::js::{signature, nsig, runtime, DeobfError};
|
use strawcore::youtube::js::{signature, nsig, runtime, DeobfError};
|
||||||
|
|
||||||
// Synthetic minified player.js — replicates the shape of real YT player.js.
|
// Synthetic minified player.js — replicates the shape of real YT player.js.
|
||||||
//
|
//
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue