diff --git a/src/downloader/default_impl.rs b/src/downloader/default_impl.rs index 9f3f5ba..9dcefea 100644 --- a/src/downloader/default_impl.rs +++ b/src/downloader/default_impl.rs @@ -81,7 +81,17 @@ impl Downloader for ReqwestDownloader { let url_after_redirects = resp.url().to_string(); if status.as_u16() == 429 { - return Err(NetworkError::Recaptcha { url: url_after_redirects }); + // Privacy: this URL ends up in error strings (and from there in + // Kotlin exception messages / logs, and the deobf failure memo). + // YouTube URLs carry the video id in the query (`?v=` / `&id=`) + // AND in some paths (`/embed/`, `/shorts/`), so keep only + // scheme+host. The Response's latest_url below stays full — it's + // data consumed by redirect-tracking logic, not a message. + let mut stripped = resp.url().clone(); + stripped.set_query(None); + stripped.set_fragment(None); + stripped.set_path("/"); + return Err(NetworkError::Recaptcha { url: stripped.to_string() }); } let code = status.as_u16(); diff --git a/src/exceptions.rs b/src/exceptions.rs index acdb096..cd19946 100644 --- a/src/exceptions.rs +++ b/src/exceptions.rs @@ -83,7 +83,12 @@ pub enum ExtractionError { impl From for NetworkError { fn from(e: reqwest::Error) -> Self { - NetworkError::Transport(e.to_string()) + // Privacy: reqwest's Display appends " for url (…)" whenever the + // error carries a URL — for player requests that URL contains + // `id=`, i.e. what the user was watching. Error strings + // become Kotlin exception messages and land in logcat / exported + // logs, so strip the URL at this single choke point. + NetworkError::Transport(e.without_url().to_string()) } } diff --git a/src/youtube/js/extractor.rs b/src/youtube/js/extractor.rs index 3ec0883..c694dd6 100644 --- a/src/youtube/js/extractor.rs +++ b/src/youtube/js/extractor.rs @@ -40,15 +40,28 @@ static SCRIPT_TAG: Lazy = /// Extracts the player.js URL + body. Tries iframe_api first, falls back /// to the embed page on any failure (matches NPE's try/catch flow). pub fn extract_javascript_player_code(video_id: &str) -> Result<(String, String), DeobfError> { - let downloader = NewPipe::downloader().ok_or(DeobfError::DownloaderMissing)?; + let url = extract_javascript_player_url(video_id)?; + let body = download_player_code(&url)?; + Ok((url, body)) +} +/// Discovers the CURRENT player.js URL only (a few-KB iframe_api fetch — +/// cheap next to the ~1.7 MB body). Split out so PlayerManager's failure +/// memo can probe "did YouTube rotate player.js?" without paying for a +/// body download that would deterministically re-fail extraction. +pub fn extract_javascript_player_url(video_id: &str) -> Result { + let downloader = NewPipe::downloader().ok_or(DeobfError::DownloaderMissing)?; let url = match extract_from_iframe(&*downloader) { Ok(u) => u, Err(_iframe_err) => extract_from_embed(&*downloader, video_id)?, }; - let cleaned = clean_javascript_url(&url)?; - let body = download_javascript_code(&*downloader, &cleaned)?; - Ok((cleaned, body)) + clean_javascript_url(&url) +} + +/// Downloads the player.js body for an already-discovered URL. +pub fn download_player_code(url: &str) -> Result { + let downloader = NewPipe::downloader().ok_or(DeobfError::DownloaderMissing)?; + download_javascript_code(&*downloader, url) } fn extract_from_iframe(downloader: &dyn Downloader) -> Result { diff --git a/src/youtube/js/mod.rs b/src/youtube/js/mod.rs index 691dc09..de1641c 100644 --- a/src/youtube/js/mod.rs +++ b/src/youtube/js/mod.rs @@ -23,7 +23,10 @@ pub mod signature; use thiserror::Error; -#[derive(Debug, Error)] +// Clone: PlayerManager's failure memo replays the original extraction +// error on calls suppressed by the cooldown (mirrors NPE's cached +// ParsingException fields). +#[derive(Debug, Clone, Error)] pub enum DeobfError { #[error("could not fetch iframe_api: {0}")] FetchIframe(String), diff --git a/src/youtube/js/nsig.rs b/src/youtube/js/nsig.rs index f70c493..3853669 100644 --- a/src/youtube/js/nsig.rs +++ b/src/youtube/js/nsig.rs @@ -112,6 +112,15 @@ fn deobfuscation_function_name(player_code: &str) -> Result // NPE's `groupCount()` excludes group 0, so: // len() == 2 → 1 capture → direct name // len() == 3 → 2 captures → array indirection + // + // BUT: `caps.len()` is pattern-static — it counts DECLARED groups, + // not the ones that PARTICIPATED in the match. Regexes 6/7 declare + // the array-access group inside an optional `(?:@ARRAY@)?`, so a + // direct-call player.js shape matches with group 2 absent. Route on + // participation: group 2 unmatched → the group-1 name IS the answer. + // (NPE's Java would NPE on `Integer.parseInt(matcher.group(2))` here + // — a latent upstream bug; the optional group's evident intent, and + // yt-dlp's guarded equivalent, is the direct name.) match caps.len() { 2 => { if let Some(m) = caps.get(1) { @@ -119,8 +128,14 @@ fn deobfuscation_function_name(player_code: &str) -> Result } } 3 => { + let Some(index_m) = caps.get(2) else { + if let Some(m) = caps.get(1) { + return Ok(m.as_str().to_string()); + } + continue; + }; let array_name = caps.get(1).map(|m| m.as_str()).unwrap_or_default(); - let index_str = caps.get(2).map(|m| m.as_str()).unwrap_or_default(); + let index_str = index_m.as_str(); let index: usize = index_str.parse().map_err(|_| { DeobfError::NsigArrayLookupFailed(format!("bad index: {index_str}")) })?; @@ -185,8 +200,18 @@ fn deobfuscation_function_body_regex( /// Strips `if(typeof X==="undefined")return ;` so the function /// actually runs standalone. NPE adds this 2024-12-29 (`56595bd9d`). +/// +/// Must handle BOTH body shapes this crate produces: +/// * lexer path: `name=function(a){…};` +/// * regex-fallback path: `function name(a){…}` +/// +/// NPE's FUNCTION_ARGUMENTS_REGEX requires the `=` (its regex-fallback +/// shape would throw from matchGroup1); ours accepts both so the fallback +/// path doesn't silently no-op — a surviving guard makes the function +/// return its input unchanged, which then gets CACHED as the deobfuscated +/// n-param → permanent silent throttling. pub fn fixup_function(function: &str) -> Result { - let args_re = Regex::new(r"=\s*function\s*\(\s*([^)]*)\s*\)") + let args_re = Regex::new(r"(?:=\s*)?function\s*[a-zA-Z0-9$_]*\s*\(\s*([^)]*)\s*\)") .map_err(|e| DeobfError::NsigBodyParseFailed(e.to_string()))?; let first_arg = args_re .captures(function) @@ -203,13 +228,18 @@ pub fn fixup_function(function: &str) -> Result { // Substitute with an alternation of fully-quoted `"undefined"` / // `'undefined'` forms. Loosens slightly (allows `"undefined'`) but // real player.js always uses balanced quotes; harmless. + // + // NPE's EARLY_RETURN_REGEX anchors on a preceding `;` only. When the + // guard is the FIRST statement of the body (regex-fallback shape: + // `function name(a){if(typeof X==="undefined")return a;…}`) the + // preceding token is `{` — accept both and re-emit whichever matched. let early_return_re_src = format!( - r#"(?s);\s*if\s*\(\s*typeof\s+[a-zA-Z0-9$_]+\s*===?\s*(?:"undefined"|'undefined')\s*\)\s*return\s+{};"#, + r#"(?s)([;{{])\s*if\s*\(\s*typeof\s+[a-zA-Z0-9$_]+\s*===?\s*(?:"undefined"|'undefined')\s*\)\s*return\s+{};"#, regex::escape(&first_arg) ); let er_re = Regex::new(&early_return_re_src) .map_err(|e| DeobfError::NsigBodyParseFailed(e.to_string()))?; - Ok(er_re.replace(function, ";").to_string()) + Ok(er_re.replace(function, "$1").to_string()) } #[cfg(test)] @@ -286,4 +316,63 @@ mod tests { Err(e) => panic!("expected name match, got {e:?}"), } } + + #[test] + fn regex_6_direct_call_without_array_access() { + // Regex 6 (String.fromCharCode(110)) with a DIRECT call — the + // optional `(?:\[(\d+)])?` group does not participate. Must route + // to the group-1 name, not the array-indirection branch (which + // would hard-fail with NsigArrayLookupFailed on the empty index). + let src = r#"WL=function(a){a.j=1};(b=String.fromCharCode(110),c=a.get(b))&&(c=mfn(c),a.set(b,c))"#; + match deobfuscation_function_name(src) { + Ok(n) => assert_eq!(n, "mfn"), + Err(e) => panic!("direct-call shape must yield the direct name, got {e:?}"), + } + } + + #[test] + fn regex_7_direct_call_without_array_access() { + // Regex 7 (.get("n")) with a direct call — same non-participating + // optional group. + let src = r#"(c=d.get("n"))&&(e=Nfn(e),d.set("n",e))"#; + match deobfuscation_function_name(src) { + Ok(n) => assert_eq!(n, "Nfn"), + Err(e) => panic!("direct-call shape must yield the direct name, got {e:?}"), + } + } + + #[test] + fn fixup_strips_guard_on_function_declaration_shape() { + // The regex-fallback body shape: `function name(a){…}` — no `=` + // before `function`, and the guard sits right after `{` with no + // preceding `;`. The old fixup no-op'd on both counts, letting the + // early return survive → nsig returns its input unchanged → the + // identity result gets cached → permanent silent throttling. + let body = r#"function m85(p){if(typeof RUQ==="undefined")return p;var a=p.split("");a.reverse();return a.join("");}"#; + let fixed = fixup_function(body).unwrap(); + assert!( + !fixed.contains("typeof RUQ"), + "guard must be stripped on the function-declaration shape, got: {fixed}" + ); + assert!(fixed.contains(r#"var a=p.split("");"#)); + } + + #[test] + fn fixup_function_declaration_shape_runs_non_identity() { + // End-to-end: the fixed fallback-shape body must actually transform + // its input when run (i.e. the guard is gone, not just renamed). + let body = r#"function m85(p){if(typeof RUQ==="undefined")return p;var a=p.split("");a.reverse();return a.join("");}"#; + let fixed = fixup_function(body).unwrap(); + let out = crate::youtube::js::runtime::run(&fixed, "m85", "abc123").unwrap(); + assert_eq!(out, "321cba"); + } + + #[test] + fn fixup_lexer_shape_guard_at_start_of_body() { + // Lexer shape (`name=function(...)`) whose guard is the FIRST + // statement — preceded by `{`, not `;`. Also must strip. + let body = r#"m85=function(p){if(typeof RUQ==="undefined")return p;var a=p.split("");return a.join("");}"#; + let fixed = fixup_function(body).unwrap(); + assert!(!fixed.contains("typeof RUQ")); + } } diff --git a/src/youtube/js/player_manager.rs b/src/youtube/js/player_manager.rs index 9c233dd..0a02fc0 100644 --- a/src/youtube/js/player_manager.rs +++ b/src/youtube/js/player_manager.rs @@ -9,21 +9,49 @@ // * cached_nsig_name + snippet // * cached_throttling_params — obfuscated → deobfuscated cache (per-session) // -// Failure handling: on ANY deobf/extraction error, invalidate() drops the -// fetched player.js and everything derived from it, so the NEXT call re-fetches -// a fresh player.js and retries. YouTube rotates player.js every few hours, so -// a failure is far more often "our cached player.js went stale" than "this -// player.js is unparseable" — invalidating lets the next attempt self-heal. -// (This replaces the previous sticky-error flags, which were set on failure and -// never cleared by any code path, permanently wedging ALL extraction for the -// process lifetime after a single routine rotation. straw audit, 2026-07-04.) +// Failure handling — three cases, each tuned so a single player.js rotation +// self-heals while a persistently-unparseable player.js can't trigger a +// per-format refetch storm (~20-25 × ~1.7 MB per video open — the H2 finding +// of the 2026-07-28 audit; the previous invalidate-on-any-error design had no +// memory of "I just refetched and it STILL failed"): // -// NPE uses static fields and is not thread-safe; callers serialize. We give the -// same shape via a `Mutex` — call sites can still hammer it from +// * Artifact extraction fails (regex bank miss / body parse): arm a +// per-artifact failure memo keyed by the player_url that failed — +// mirrors NPE's cached ParsingException fields (sigTimestampExtractionEx +// et al.), except NPE's are sticky until clearAllCaches() while ours +// expire after FAILURE_COOLDOWN. The installed player.js is KEPT so +// sibling artifacts (e.g. a working signature timestamp) stay usable. +// While the memo is live the original error is replayed with zero +// network. After the cooldown, discovery re-runs (a few-KB iframe_api +// probe): a rotated player.js is fetched fresh and retried; an +// unchanged one just re-arms the memo — extraction is deterministic, +// so re-downloading ~1.7 MB to re-fail is pure waste. +// +// * Eval (QuickJS run) fails: invalidate() everything so the next call +// re-fetches — the stale-rotation self-heal (a cached player.js whose +// function chokes on a new-generation n-param is fixed by fetching the +// current player.js). If the snippet came from a player.js downloaded +// by THIS call, a refetch would rebuild the identical snippet, so the +// memo is armed instead — bounding the storm to one refetch per open. +// +// * Fetch fails (network): plain error, no memo — transient by nature. +// +// Locking: `state` (Mutex) is NEVER held across the network. +// `fetch_gate` serializes player.js downloads so concurrent extractions +// don't each re-download (~1.7 MB); waiters re-check `state` after taking +// the gate and find the code already installed. Lock order is always +// fetch_gate → state; state is released before any fetch. (The previous +// design held the state mutex across ensure_player_code's downloads, +// serializing every concurrent extraction behind up-to-30 s requests — +// the S6 finding.) +// +// NPE uses static fields and is not thread-safe; callers serialize. We give +// the same shape via the state mutex — call sites can still hammer it from // multiple threads safely. use parking_lot::Mutex; use std::collections::HashMap; +use std::time::{Duration, Instant}; use crate::youtube::js::extractor; use crate::youtube::js::nsig; @@ -31,6 +59,96 @@ use crate::youtube::js::runtime; use crate::youtube::js::signature; use crate::youtube::js::DeobfError; +/// How long a failure memo suppresses re-fetch/re-parse attempts for the +/// player.js that produced it. YouTube rotates player.js every few hours, +/// so five minutes bounds the retry storm without meaningfully delaying +/// recovery once a genuinely new player.js ships (recovery is also +/// immediate whenever a different player_url is discovered). +const FAILURE_COOLDOWN: Duration = Duration::from_secs(5 * 60); + +/// One derived artifact's failure record. `player_url` pins the player.js +/// generation the failure belongs to: a different discovered URL retires +/// the memo instantly (rotation self-heal), an identical one replays +/// `error` without re-downloading (extraction is deterministic). +struct FailMemo { + player_url: String, + error: DeobfError, + at: Instant, +} + +/// The three derived artifacts, each with its own cache + failure memo — +/// mirrors NPE's three cached exception fields. Kept independent so e.g. +/// a signature-timestamp regex miss doesn't block nsig extraction that +/// would succeed on the same player.js. +#[derive(Clone, Copy)] +enum Derived { + SigTimestamp, + Sig, + Nsig, +} + +impl Derived { + fn memo<'a>(&self, state: &'a ManagerState) -> &'a Option { + match self { + Derived::SigTimestamp => &state.ts_fail, + Derived::Sig => &state.sig_fail, + Derived::Nsig => &state.nsig_fail, + } + } + + fn memo_mut<'a>(&self, state: &'a mut ManagerState) -> &'a mut Option { + match self { + Derived::SigTimestamp => &mut state.ts_fail, + Derived::Sig => &mut state.sig_fail, + Derived::Nsig => &mut state.nsig_fail, + } + } +} + +/// Verdict of a memo check, decided under the state lock. +enum MemoVerdict { + /// No live memo applies — proceed normally. + Proceed, + /// Memo cooldown expired — proceed, but treat this URL as known-bad: + /// discover first and skip the ~1.7 MB body download if it's unchanged. + Probe(String), + /// Memo is live — replay the memoized error, zero work. + Replay(DeobfError), +} + +fn memo_verdict(state: &ManagerState, memo: &Option) -> MemoVerdict { + let Some(m) = memo else { + return MemoVerdict::Proceed; + }; + let live = m.at.elapsed() < FAILURE_COOLDOWN; + let installed_same = state.player_code.is_some() + && state.player_url.as_deref() == Some(m.player_url.as_str()); + if installed_same { + if live { + MemoVerdict::Replay(m.error.clone()) + } else { + MemoVerdict::Probe(m.player_url.clone()) + } + } else if state.player_code.is_some() { + // A different player.js has been installed since the failure — + // retry this artifact against it. + MemoVerdict::Proceed + } else if live { + MemoVerdict::Replay(m.error.clone()) + } else { + MemoVerdict::Probe(m.player_url.clone()) + } +} + +/// Outcome of ensure_player_code. +enum Ensured { + /// player_code is installed; `fresh` = this call downloaded it. + Ready { fresh: bool }, + /// The probe found YouTube still serving the caller's known-bad URL — + /// no body was downloaded. + StillBad, +} + #[derive(Default)] struct ManagerState { player_url: Option, @@ -41,13 +159,21 @@ struct ManagerState { nsig_name: Option, nsig_snippet: Option, throttling_param_cache: HashMap, + + // Failure memos — deliberately NOT cleared by invalidate(): they must + // survive the invalidate that accompanies the failure they record. + // Retired by: a different player_url being installed, their artifact + // building successfully, clear_all_caches(), or cooldown expiry. + ts_fail: Option, + sig_fail: Option, + nsig_fail: Option, } impl ManagerState { /// Drop the fetched player.js and everything derived from it, so the next - /// call re-fetches a fresh player.js and re-extracts. Called on any - /// deobf/extraction failure so a stale-player.js error self-heals on retry - /// instead of wedging extraction for the process lifetime. + /// call re-fetches a fresh player.js and re-extracts. Called on eval + /// failures (stale-player.js self-heal) and before installing a newly + /// downloaded player.js. Failure memos are kept — see the field comment. fn invalidate(&mut self) { self.player_url = None; self.player_code = None; @@ -62,13 +188,17 @@ impl ManagerState { } pub struct PlayerManager { - inner: Mutex, + state: Mutex, + /// Serializes player.js downloads WITHOUT blocking `state` readers. + /// Held only while discovering/downloading; never while `state` is held. + fetch_gate: Mutex<()>, } impl PlayerManager { pub fn new() -> Self { Self { - inner: Mutex::new(ManagerState::default()), + state: Mutex::new(ManagerState::default()), + fetch_gate: Mutex::new(()), } } @@ -79,23 +209,14 @@ impl PlayerManager { } pub fn signature_timestamp(&self, video_id: &str) -> Result { - let r = self.signature_timestamp_inner(video_id); - if r.is_err() { - self.inner.lock().invalidate(); - } - r - } - - fn signature_timestamp_inner(&self, video_id: &str) -> Result { - let mut state = self.inner.lock(); - if let Some(ts) = state.signature_timestamp { - return Ok(ts); - } - Self::ensure_player_code(&mut state, video_id)?; - let code = state.player_code.as_deref().unwrap(); - let ts = signature::signature_timestamp(code)?; - state.signature_timestamp = Some(ts); - Ok(ts) + self.derived_artifact( + video_id, + Derived::SigTimestamp, + |state| state.signature_timestamp, + signature::signature_timestamp, + |state, ts| state.signature_timestamp = Some(*ts), + ) + .map(|(ts, _)| ts) } pub fn deobfuscate_signature( @@ -103,56 +224,37 @@ impl PlayerManager { video_id: &str, obfuscated_signature: &str, ) -> Result { - let r = self.deobfuscate_signature_inner(video_id, obfuscated_signature); - if r.is_err() { - self.inner.lock().invalidate(); - } - r - } + let (snippet, fresh) = self.derived_artifact( + video_id, + Derived::Sig, + |state| state.sig_snippet.clone(), + signature::build_deobfuscator, + |state, snippet| state.sig_snippet = Some(snippet.clone()), + )?; - fn deobfuscate_signature_inner( - &self, - video_id: &str, - obfuscated_signature: &str, - ) -> Result { - let snippet = { - let mut state = self.inner.lock(); - if state.sig_snippet.is_none() { - Self::ensure_player_code(&mut state, video_id)?; - let code = state.player_code.as_deref().unwrap(); - let s = signature::build_deobfuscator(code)?; - state.sig_snippet = Some(s); - } - state.sig_snippet.clone().unwrap() - }; - - let result = runtime::run( + match runtime::run( &snippet, signature::DEOBFUSCATION_FUNCTION_NAME, obfuscated_signature, - )?; - if result == "null" { - return Ok(String::new()); // NPE: Objects.requireNonNullElse(..., "") + ) { + Ok(result) => { + if result == "null" { + Ok(String::new()) // NPE: Objects.requireNonNullElse(..., "") + } else { + Ok(result) + } + } + Err(e) => { + self.on_eval_failure(Derived::Sig, fresh, &e); + Err(e) + } } - Ok(result) } pub fn url_with_throttling_parameter_deobfuscated( &self, video_id: &str, streaming_url: &str, - ) -> Result { - let r = self.url_with_throttling_parameter_deobfuscated_inner(video_id, streaming_url); - if r.is_err() { - self.inner.lock().invalidate(); - } - r - } - - fn url_with_throttling_parameter_deobfuscated_inner( - &self, - video_id: &str, - streaming_url: &str, ) -> Result { let obf = match nsig::throttling_parameter_from_url(streaming_url) { Some(s) => s, @@ -160,34 +262,41 @@ impl PlayerManager { }; { - let state = self.inner.lock(); + let state = self.state.lock(); if let Some(cached) = state.throttling_param_cache.get(&obf) { return Ok(streaming_url.replace(&obf, cached)); } } - let (name, snippet) = { - let mut state = self.inner.lock(); - if state.nsig_snippet.is_none() { - Self::ensure_player_code(&mut state, video_id)?; - let code = state.player_code.as_deref().unwrap(); - let (n, s) = nsig::build_deobfuscator(code)?; - state.nsig_name = Some(n); - state.nsig_snippet = Some(s); - } - ( - state.nsig_name.clone().unwrap(), - state.nsig_snippet.clone().unwrap(), - ) - }; + let ((name, snippet), fresh) = self.derived_artifact( + video_id, + Derived::Nsig, + |state| match (&state.nsig_name, &state.nsig_snippet) { + (Some(n), Some(s)) => Some((n.clone(), s.clone())), + _ => None, + }, + nsig::build_deobfuscator, + |state, (n, s)| { + state.nsig_name = Some(n.clone()); + state.nsig_snippet = Some(s.clone()); + }, + )?; - let deobf = runtime::run(&snippet, &name, &obf)?; + let deobf = match runtime::run(&snippet, &name, &obf) { + Ok(d) => d, + Err(e) => { + self.on_eval_failure(Derived::Nsig, fresh, &e); + return Err(e); + } + }; if deobf.is_empty() { - return Err(DeobfError::NsigEmpty); + let e = DeobfError::NsigEmpty; + self.on_eval_failure(Derived::Nsig, fresh, &e); + return Err(e); } { - let mut state = self.inner.lock(); + let mut state = self.state.lock(); state .throttling_param_cache .insert(obf.clone(), deobf.clone()); @@ -197,29 +306,230 @@ impl PlayerManager { } pub fn throttling_parameter_cache_size(&self) -> usize { - self.inner.lock().throttling_param_cache.len() + self.state.lock().throttling_param_cache.len() } pub fn clear_all_caches(&self) { - self.inner.lock().invalidate(); + let mut state = self.state.lock(); + state.invalidate(); + state.ts_fail = None; + state.sig_fail = None; + state.nsig_fail = None; } pub fn clear_throttling_parameters_cache(&self) { - self.inner.lock().throttling_param_cache.clear(); + self.state.lock().throttling_param_cache.clear(); } pub fn player_url(&self) -> Option { - self.inner.lock().player_url.clone() + self.state.lock().player_url.clone() } - fn ensure_player_code(state: &mut ManagerState, video_id: &str) -> Result<(), DeobfError> { - if state.player_code.is_some() { - return Ok(()); + /// Shared spine for the three derived artifacts: cache fast path → + /// failure-memo verdict → make player code available (fetching outside + /// the state lock) → build + store under the lock, arming the memo on + /// extraction failure. Returns the artifact plus whether the player.js + /// it was built from was downloaded by THIS call (`fresh` — feeds the + /// eval-failure policy). + fn derived_artifact( + &self, + video_id: &str, + which: Derived, + cached: impl Fn(&ManagerState) -> Option, + build: impl Fn(&str) -> Result, + store: impl Fn(&mut ManagerState, &T), + ) -> Result<(T, bool), DeobfError> { + // Two passes: the second absorbs the narrow races (concurrent + // invalidate between ensure and the build lock; memo retired by + // clear_all_caches between verdict and probe). + for _ in 0..2 { + let known_bad: Option = { + let state = self.state.lock(); + if let Some(v) = cached(&state) { + return Ok((v, false)); + } + match memo_verdict(&state, which.memo(&state)) { + MemoVerdict::Replay(e) => return Err(e), + MemoVerdict::Probe(url) => Some(url), + MemoVerdict::Proceed => None, + } + }; + + let ensured = match self.ensure_player_code(video_id, known_bad.as_deref()) { + Ok(e) => e, + Err(fetch_err) => { + if known_bad.is_some() { + // The post-cooldown probe itself failed (network). + // Re-arm the cooldown so we don't probe on every + // subsequent call while the network is down. + let mut state = self.state.lock(); + if let Some(m) = which.memo_mut(&mut state).as_mut() { + m.at = Instant::now(); + } + } + return Err(fetch_err); + } + }; + let fresh = match ensured { + Ensured::Ready { fresh } => fresh, + Ensured::StillBad => { + // YouTube still serves the player.js this artifact + // failed on — replay the original error and restart + // the cooldown. + let mut state = self.state.lock(); + match which.memo_mut(&mut state).as_mut() { + Some(m) => { + m.at = Instant::now(); + return Err(m.error.clone()); + } + // Memo retired concurrently (clear_all_caches) — + // loop and take the fresh-slate path. + None => continue, + } + } + }; + + let mut state = self.state.lock(); + let built = match state.player_code.as_deref() { + Some(code) => build(code), + // Invalidated between ensure and here — loop to re-fetch. + None => continue, + }; + return match built { + Ok(v) => { + store(&mut state, &v); + // A successful build proves any lingering memo for this + // artifact records a superseded player.js. + *which.memo_mut(&mut state) = None; + Ok((v, fresh)) + } + Err(e) => { + // Extraction failed. Keep the installed player.js — + // sibling artifacts may extract fine from it — and arm + // the memo so every further call within the cooldown + // replays instead of re-fetching ~1.7 MB per format. + let player_url = state.player_url.clone().unwrap_or_default(); + *which.memo_mut(&mut state) = Some(FailMemo { + player_url, + error: e.clone(), + at: Instant::now(), + }); + Err(e) + } + }; + } + Err(DeobfError::FetchPlayerCode( + "player.js unavailable after retry (concurrent cache invalidation)".into(), + )) + } + + /// Eval (QuickJS) failures: invalidate so the next call re-fetches — + /// the stale-rotation self-heal (a cached player.js whose function + /// chokes on a new-generation n-param is fixed by fetching the current + /// player.js). When the failing snippet came from a player.js this very + /// call downloaded (`fresh`), a refetch would rebuild the identical + /// snippet — arm the memo instead so the remaining ~20-25 formats of + /// the open replay the error instead of each re-downloading. + fn on_eval_failure(&self, which: Derived, fresh: bool, error: &DeobfError) { + let mut state = self.state.lock(); + let player_url = state.player_url.clone().unwrap_or_default(); + state.invalidate(); + if fresh { + *which.memo_mut(&mut state) = Some(FailMemo { + player_url, + error: error.clone(), + at: Instant::now(), + }); + } + } + + /// Makes `state.player_code` available. The network work happens with + /// only `fetch_gate` held — never the state lock — so cache-hit readers + /// stay unblocked and concurrent fetchers collapse into one download. + /// + /// `known_bad_url`: the caller's memoized failing player URL. Discovery + /// still runs, but if YouTube is serving that exact URL the ~1.7 MB body + /// download is skipped (`StillBad`) — extraction is deterministic, so it + /// could only re-fail. + fn ensure_player_code( + &self, + video_id: &str, + known_bad_url: Option<&str>, + ) -> Result { + // Fast path — usable code already installed. + { + let state = self.state.lock(); + if state.player_code.is_some() + && (known_bad_url.is_none() || state.player_url.as_deref() != known_bad_url) + { + return Ok(Ensured::Ready { fresh: false }); + } + } + + // One fetcher at a time. Waiters block HERE (not on `state`), then + // usually find the winner's code installed. + let _gate = self.fetch_gate.lock(); + { + let state = self.state.lock(); + if state.player_code.is_some() + && (known_bad_url.is_none() || state.player_url.as_deref() != known_bad_url) + { + return Ok(Ensured::Ready { fresh: false }); + } + } + + // Discovery (iframe_api / embed fallback) — a few KB. + let url = extractor::extract_javascript_player_url(video_id)?; + if Some(url.as_str()) == known_bad_url { + return Ok(Ensured::StillBad); + } + // Body download — ~1.7 MB. + let code = extractor::download_player_code(&url)?; + + let mut state = self.state.lock(); + let state: &mut ManagerState = &mut state; + // Everything derived from any previously installed player.js dies + // with it; memos recorded against OTHER player URLs are retired so + // a genuinely new player.js gets a clean slate for every artifact. + state.invalidate(); + for memo in [ + &mut state.ts_fail, + &mut state.sig_fail, + &mut state.nsig_fail, + ] { + if memo.as_ref().is_some_and(|m| m.player_url != url) { + *memo = None; + } } - let (url, code) = extractor::extract_javascript_player_code(video_id)?; state.player_url = Some(url); state.player_code = Some(code); - Ok(()) + Ok(Ensured::Ready { fresh: true }) + } + + /// Test hook: age every armed memo past FAILURE_COOLDOWN so the probe + /// path can be exercised without a five-minute sleep. + #[cfg(test)] + fn test_expire_memos(&self) { + let mut state = self.state.lock(); + let state: &mut ManagerState = &mut state; + for memo in [ + &mut state.ts_fail, + &mut state.sig_fail, + &mut state.nsig_fail, + ] { + if let Some(m) = memo.as_mut() { + if let Some(aged) = Instant::now().checked_sub(FAILURE_COOLDOWN) { + m.at = aged; + } + } + } + } + + /// Test hook: is the state mutex currently free? Used to prove the + /// player.js fetch runs without holding it. + #[cfg(test)] + fn state_lock_is_free(&self) -> bool { + self.state.try_lock().is_some() } } @@ -232,6 +542,13 @@ impl Default for PlayerManager { #[cfg(test)] mod tests { use super::*; + use crate::downloader::request::Request; + use crate::downloader::response::Response; + use crate::downloader::Downloader; + use crate::exceptions::NetworkError; + use crate::newpipe::NewPipe; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::{mpsc, Arc}; #[test] fn assembled_sig_snippet_runs_against_synthetic_player() { @@ -264,4 +581,281 @@ mod tests { mgr.clear_all_caches(); assert!(mgr.player_url().is_none()); } + + // ---- failure-memo / refetch-storm tests ------------------------------- + // + // These register a stub Downloader in the process-global NewPipe + // singleton, so they serialize on GLOBAL_DOWNLOADER_LOCK (no other lib + // test touches the downloader). Each uses its own PlayerManager + // instance, so manager state never crosses tests. + + static GLOBAL_DOWNLOADER_LOCK: Mutex<()> = Mutex::new(()); + + /// Both channel endpoints wrapped in Mutex for the Downloader Sync bound. + type FetchBlocker = (Mutex>, Mutex>); + + /// Serves a synthetic iframe_api (announcing `hash`) and a synthetic + /// player.js body; counts every request. + struct StubDownloader { + hash: Mutex, + player_js: Mutex, + requests: AtomicUsize, + /// When set, the player.js body request signals `entered` then + /// blocks until `release` fires — used by the lock-freedom test. + block_player_fetch: Option, + } + + impl StubDownloader { + fn new(hash: &str, player_js: &str) -> Self { + Self { + hash: Mutex::new(hash.to_string()), + player_js: Mutex::new(player_js.to_string()), + requests: AtomicUsize::new(0), + block_player_fetch: None, + } + } + + fn rotate(&self, hash: &str, player_js: &str) { + *self.hash.lock() = hash.to_string(); + *self.player_js.lock() = player_js.to_string(); + } + + fn count(&self) -> usize { + self.requests.load(Ordering::SeqCst) + } + } + + impl Downloader for StubDownloader { + fn execute(&self, request: Request) -> Result { + self.requests.fetch_add(1, Ordering::SeqCst); + let url = request.url().to_string(); + if url.contains("iframe_api") { + let h = self.hash.lock().clone(); + // Shape matched by IFRAME_RES_JS_BASE_PLAYER_HASH (escaped + // `\/` form, 8-char [a-z0-9] hash). + let body = format!( + r#"src:"https://www.youtube.com/s/player\/{h}\/player_ias.vflset/en_US/www-embed.js""# + ); + Ok(Response::new(200, "OK", Default::default(), body, url)) + } else if url.contains("/s/player/") { + if let Some((entered, release)) = &self.block_player_fetch { + let _ = entered.lock().send(()); + let _ = release + .lock() + .recv_timeout(std::time::Duration::from_secs(10)); + } + let body = self.player_js.lock().clone(); + Ok(Response::new(200, "OK", Default::default(), body, url)) + } else { + Err(NetworkError::Transport("stub: unexpected request".into())) + } + } + } + + // Parseable by ts + NOTHING nsig-shaped → nsig build fails while the + // signature timestamp extracts fine. + const PLAYER_NSIG_BROKEN: &str = r#"var foo={signatureTimestamp:19999};var unrelated=1;"#; + + // nsig name via regex 0 (m85 … the greedy `.*` reaches dummy's + // `return Q[1]` on the same line); body via lexer; self-contained + // (no outer-global references), so eval succeeds for any input. + const PLAYER_GOOD: &str = r#"m85=function(p){var a=p.split("");a.reverse();return a.join("");};dummy=function(q){return Q[1];};var foo={signatureTimestamp:20244};"#; + + // Parses fine but throws for EVERY input at eval time. + const PLAYER_EVAL_THROWS: &str = r#"m85=function(p){throw Error("boom");return Q[1];};"#; + + // Parses fine; throws only for the input "evil". + const PLAYER_EVAL_SELECTIVE: &str = r#"m85=function(p){if(p==="evil")throw Error("bad");var a=p.split("");a.reverse();return a.join("");};dummy=function(q){return Q[1];};"#; + + fn install(stub: &Arc) { + NewPipe::init(stub.clone() as Arc); + } + + #[test] + fn nsig_build_failure_memoizes_instead_of_refetching_per_format() { + let _g = GLOBAL_DOWNLOADER_LOCK.lock(); + let stub = Arc::new(StubDownloader::new("aaaaaaaa", PLAYER_NSIG_BROKEN)); + install(&stub); + let mgr = PlayerManager::new(); + + // First format: discovery + body download (2 requests), build fails. + let err = mgr + .url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=AAA") + .unwrap_err(); + assert!(matches!(err, DeobfError::NsigFuncNotFound), "got {err:?}"); + assert_eq!(stub.count(), 2); + + // Remaining formats of the same open: replayed error, ZERO network. + for _ in 0..5 { + let err = mgr + .url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=BBB") + .unwrap_err(); + assert!(matches!(err, DeobfError::NsigFuncNotFound)); + } + assert_eq!(stub.count(), 2, "memo must suppress refetching"); + } + + #[test] + fn artifact_memos_are_independent() { + let _g = GLOBAL_DOWNLOADER_LOCK.lock(); + let stub = Arc::new(StubDownloader::new("aaaaaaaa", PLAYER_NSIG_BROKEN)); + install(&stub); + let mgr = PlayerManager::new(); + + // nsig fails and memoizes… + assert!(mgr + .url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=AAA") + .is_err()); + assert_eq!(stub.count(), 2); + // …but the player.js stays installed, so the signature timestamp + // extracts from it without any further network. + assert_eq!(mgr.signature_timestamp("vid").unwrap(), 19999); + assert_eq!(stub.count(), 2); + } + + #[test] + fn rotation_after_cooldown_fetches_fresh_and_recovers() { + let _g = GLOBAL_DOWNLOADER_LOCK.lock(); + let stub = Arc::new(StubDownloader::new("aaaaaaaa", PLAYER_NSIG_BROKEN)); + install(&stub); + let mgr = PlayerManager::new(); + + assert!(mgr + .url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=AAA") + .is_err()); + assert_eq!(stub.count(), 2); + + // YouTube rotates to a player.js our bank parses. + stub.rotate("bbbbbbbb", PLAYER_GOOD); + + // Within the cooldown the memo still replays — no network. + assert!(mgr + .url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=AAA") + .is_err()); + assert_eq!(stub.count(), 2); + + // After the cooldown: probe discovers the NEW url → full fetch → + // extraction + eval succeed. + mgr.test_expire_memos(); + let out = mgr + .url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=abc") + .unwrap(); + assert_eq!(out, "https://x/?n=cba"); + assert_eq!(stub.count(), 4, "probe (iframe) + fresh body download"); + } + + #[test] + fn post_cooldown_probe_on_unchanged_player_skips_body_download() { + let _g = GLOBAL_DOWNLOADER_LOCK.lock(); + let stub = Arc::new(StubDownloader::new("aaaaaaaa", PLAYER_NSIG_BROKEN)); + install(&stub); + let mgr = PlayerManager::new(); + + assert!(mgr + .url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=AAA") + .is_err()); + assert_eq!(stub.count(), 2); + + mgr.test_expire_memos(); + // Probe: one iframe_api discovery, SAME hash → no body download, + // memo re-armed. + assert!(mgr + .url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=AAA") + .is_err()); + assert_eq!(stub.count(), 3); + + // Re-armed cooldown suppresses even the probe. + assert!(mgr + .url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=AAA") + .is_err()); + assert_eq!(stub.count(), 3); + } + + #[test] + fn transient_eval_failure_still_self_heals_via_refetch() { + let _g = GLOBAL_DOWNLOADER_LOCK.lock(); + let stub = Arc::new(StubDownloader::new("aaaaaaaa", PLAYER_EVAL_SELECTIVE)); + install(&stub); + let mgr = PlayerManager::new(); + + // Working nsig on the first video. + let out = mgr + .url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=good1") + .unwrap(); + assert_eq!(out, "https://x/?n=1doog"); + assert_eq!(stub.count(), 2); + + // A new-generation n-param chokes the CACHED (not fresh) snippet → + // invalidate, NO memo. No network spent on the failing call itself. + assert!(mgr + .url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=evil") + .is_err()); + assert_eq!(stub.count(), 2); + + // YouTube has rotated; the very next call re-fetches and succeeds — + // the transient-rotation self-heal must NOT be suppressed. + stub.rotate("bbbbbbbb", PLAYER_GOOD); + let out = mgr + .url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=evil") + .unwrap(); + assert_eq!(out, "https://x/?n=live"); + assert_eq!(stub.count(), 4); + } + + #[test] + fn persistent_eval_failure_on_fresh_player_memoizes() { + let _g = GLOBAL_DOWNLOADER_LOCK.lock(); + let stub = Arc::new(StubDownloader::new("aaaaaaaa", PLAYER_EVAL_THROWS)); + install(&stub); + let mgr = PlayerManager::new(); + + // Fresh download + successful build, then eval throws → memo. + let err = mgr + .url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=AAA") + .unwrap_err(); + assert!(matches!(err, DeobfError::JsRuntimeFailed(_)), "got {err:?}"); + assert_eq!(stub.count(), 2); + + // Every further format replays without network. + for _ in 0..5 { + assert!(mgr + .url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=BBB") + .is_err()); + } + assert_eq!(stub.count(), 2); + } + + #[test] + fn state_lock_stays_free_during_player_fetch() { + let _g = GLOBAL_DOWNLOADER_LOCK.lock(); + let (entered_tx, entered_rx) = mpsc::channel(); + let (release_tx, release_rx) = mpsc::channel(); + let mut stub = StubDownloader::new("aaaaaaaa", PLAYER_GOOD); + stub.block_player_fetch = Some((Mutex::new(entered_tx), Mutex::new(release_rx))); + let stub = Arc::new(stub); + install(&stub); + + let mgr: &'static PlayerManager = Box::leak(Box::new(PlayerManager::new())); + let worker = std::thread::spawn(move || { + mgr.url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=abc") + }); + + // Wait until the worker is INSIDE the player.js body download. + entered_rx + .recv_timeout(std::time::Duration::from_secs(10)) + .expect("worker never reached the player.js fetch"); + + // The whole point of the fetch_gate: readers aren't blocked while + // ~1.7 MB downloads. The old design held this mutex across the + // network and serialized everything behind it. + assert!( + mgr.state_lock_is_free(), + "state mutex must not be held across the player.js download" + ); + assert_eq!(mgr.throttling_parameter_cache_size(), 0); // must not block + + release_tx.send(()).unwrap(); + let out = worker.join().unwrap().unwrap(); + assert_eq!(out, "https://x/?n=cba"); + } } diff --git a/src/youtube/linkhandler/channel.rs b/src/youtube/linkhandler/channel.rs index d294655..33fde0e 100644 --- a/src/youtube/linkhandler/channel.rs +++ b/src/youtube/linkhandler/channel.rs @@ -10,7 +10,7 @@ use url::Url; -use crate::youtube::linkhandler::{host_is_youtube, LinkError}; +use crate::youtube::linkhandler::{host_is_youtube, redact_url, LinkError}; #[derive(Clone, Debug, Eq, PartialEq)] pub enum ChannelIdentifier { @@ -25,8 +25,10 @@ pub enum ChannelIdentifier { } pub fn parse(url_str: &str) -> Result { + // Privacy: never embed the raw input in the error (see stream.rs) — + // the parse reason alone suffices. let url = Url::parse(url_str) - .map_err(|e| LinkError::InvalidUrl(format!("{url_str}: {e}")))?; + .map_err(|e| LinkError::InvalidUrl(e.to_string()))?; let host = url .host_str() .ok_or_else(|| LinkError::InvalidUrl("no host".into()))?; @@ -37,32 +39,32 @@ pub fn parse(url_str: &str) -> Result { if let Some(rest) = path.strip_prefix("/channel/") { let id = rest.split('/').next().unwrap_or(""); if id.is_empty() { - return Err(LinkError::MissingId(url_str.into())); + return Err(LinkError::MissingId(redact_url(&url))); } return Ok(ChannelIdentifier::DirectId(id.into())); } if let Some(rest) = path.strip_prefix("/c/") { let s = rest.split('/').next().unwrap_or(""); if s.is_empty() { - return Err(LinkError::MissingId(url_str.into())); + return Err(LinkError::MissingId(redact_url(&url))); } return Ok(ChannelIdentifier::Custom(s.into())); } if let Some(rest) = path.strip_prefix("/user/") { let s = rest.split('/').next().unwrap_or(""); if s.is_empty() { - return Err(LinkError::MissingId(url_str.into())); + return Err(LinkError::MissingId(redact_url(&url))); } return Ok(ChannelIdentifier::LegacyUser(s.into())); } if let Some(rest) = path.strip_prefix("/@") { let s = rest.split('/').next().unwrap_or(""); if s.is_empty() { - return Err(LinkError::MissingId(url_str.into())); + return Err(LinkError::MissingId(redact_url(&url))); } return Ok(ChannelIdentifier::Handle(s.into())); } - Err(LinkError::MissingId(url_str.into())) + Err(LinkError::MissingId(redact_url(&url))) } pub fn channel_url(channel_id: &str) -> String { diff --git a/src/youtube/linkhandler/mod.rs b/src/youtube/linkhandler/mod.rs index 4f2f671..542f85d 100644 --- a/src/youtube/linkhandler/mod.rs +++ b/src/youtube/linkhandler/mod.rs @@ -34,6 +34,20 @@ pub const ACCEPTED_HOSTS: &[&str] = &[ "www.youtube-nocookie.com", ]; +/// Renders a URL as scheme+host only, for embedding in error strings. +/// Privacy: link errors reach Kotlin exception messages / logs. Query params +/// carry user browsing data (`v=`, `list=`) AND several +/// YouTube PATHS embed the id (`/embed/`, `/shorts/`, `/clip/`, +/// `/live/`, `youtu.be/`) — so drop the path too; the host alone says +/// which endpoint failed without leaking what the user was watching. +pub(crate) fn redact_url(url: &url::Url) -> String { + let mut u = url.clone(); + u.set_query(None); + u.set_fragment(None); + u.set_path("/"); + u.to_string() +} + pub fn host_is_youtube(host: &str) -> bool { let h = host.to_ascii_lowercase(); let h = h.strip_prefix("www.").unwrap_or(&h); @@ -66,4 +80,24 @@ mod tests { assert!(!host_is_youtube("piped.video")); assert!(!host_is_youtube("evil.com")); } + + #[test] + fn redact_url_drops_id_bearing_path_query_and_fragment() { + // Every one of these carries the video/playlist id somewhere the old + // query-only strip would have kept (the path, for embed/shorts/youtu.be). + let cases = [ + "https://www.youtube.com/embed/dQw4w9WgXcQ?v=secret#frag", + "https://www.youtube.com/shorts/dQw4w9WgXcQ", + "https://youtu.be/dQw4w9WgXcQ", + "https://www.youtube.com/watch?v=dQw4w9WgXcQ&list=PLsecretlist", + ]; + for c in cases { + let u = url::Url::parse(c).unwrap(); + let r = redact_url(&u); + assert!(!r.contains("dQw4w9WgXcQ"), "leaked video id: {r}"); + assert!(!r.contains("PLsecretlist"), "leaked list id: {r}"); + // scheme + host only (no port in any of these cases). + assert_eq!(r, format!("{}://{}/", u.scheme(), u.host_str().unwrap())); + } + } } diff --git a/src/youtube/linkhandler/stream.rs b/src/youtube/linkhandler/stream.rs index 9592138..dbeb732 100644 --- a/src/youtube/linkhandler/stream.rs +++ b/src/youtube/linkhandler/stream.rs @@ -15,7 +15,7 @@ use once_cell::sync::Lazy; use regex::Regex; use url::Url; -use crate::youtube::linkhandler::{host_is_youtube, LinkError}; +use crate::youtube::linkhandler::{host_is_youtube, redact_url, LinkError}; const VIDEO_ID_LEN: usize = 11; @@ -42,8 +42,11 @@ fn extract_video_id_inner(input_url: &str, depth: u8) -> Result Result` in the query — never + // put it in an error string (they reach Kotlin exception messages + // and logs). Scheme+host+path identifies the endpoint just fine. + let endpoint = url.split(['?', '#']).next().unwrap_or(url); return Err(ExtractionError::Network(NetworkError::Transport(format!( - "HTTP {} from {url}", + "HTTP {} from {endpoint}", resp.response_code() )))); } diff --git a/tests/foundation_smoke.rs b/tests/foundation_smoke.rs index 1fb88f2..2544e87 100644 --- a/tests/foundation_smoke.rs +++ b/tests/foundation_smoke.rs @@ -9,16 +9,22 @@ use std::sync::Arc; -use strawcore::downloader::request::Request; -use strawcore::downloader::ReqwestDownloader; -use strawcore::exceptions::NetworkError; -use strawcore::localization::{ContentCountry, Localization}; -use strawcore::{Downloader, NewPipe}; +use strawcore_core::downloader::request::Request; +use strawcore_core::downloader::ReqwestDownloader; +use strawcore_core::exceptions::NetworkError; +use strawcore_core::localization::{ContentCountry, Localization}; +use strawcore_core::{Downloader, NewPipe}; + +/// The Downloader trait dropped its `get` convenience (commit f917e4a); +/// mirror it here so the suite reads as before. +fn get(dl: &impl Downloader, url: &str) -> Result { + dl.execute(Request::get(url).build()) +} #[test] fn get_through_default_downloader() { let dl = ReqwestDownloader::new().expect("build downloader"); - let resp = dl.get("https://httpbin.org/get").expect("transport"); + let resp = get(&dl, "https://httpbin.org/get").expect("transport"); assert_eq!(resp.response_code(), 200); assert!(resp.response_body().contains("\"url\"")); } @@ -26,9 +32,7 @@ fn get_through_default_downloader() { #[test] fn latest_url_follows_redirects() { let dl = ReqwestDownloader::new().expect("build downloader"); - let resp = dl - .get("https://httpbin.org/redirect/3") - .expect("transport"); + let resp = get(&dl, "https://httpbin.org/redirect/3").expect("transport"); assert_eq!(resp.response_code(), 200); assert!( resp.latest_url().ends_with("/get"), @@ -40,14 +44,14 @@ fn latest_url_follows_redirects() { #[test] fn non_2xx_returns_ok_not_err() { let dl = ReqwestDownloader::new().expect("build downloader"); - let resp = dl.get("https://httpbin.org/status/404").expect("transport"); + let resp = get(&dl, "https://httpbin.org/status/404").expect("transport"); assert_eq!(resp.response_code(), 404); } #[test] fn http_429_surfaces_as_recaptcha_err() { let dl = ReqwestDownloader::new().expect("build downloader"); - let err = dl.get("https://httpbin.org/status/429").expect_err("429 must be NetworkError"); + let err = get(&dl, "https://httpbin.org/status/429").expect_err("429 must be NetworkError"); match err { NetworkError::Recaptcha { url } => assert!(url.contains("/status/429")), other => panic!("expected Recaptcha, got {other:?}"), @@ -72,8 +76,8 @@ fn localization_header_attached_when_enabled() { #[test] fn header_keys_lowercased_in_response() { let dl = ReqwestDownloader::new().expect("build downloader"); - let resp = dl.get("https://httpbin.org/get").expect("transport"); - for (k, _) in resp.response_headers() { + let resp = get(&dl, "https://httpbin.org/get").expect("transport"); + for k in resp.response_headers().keys() { assert_eq!(k, &k.to_ascii_lowercase(), "header key {k} not lowercased"); } } @@ -88,7 +92,9 @@ fn newpipe_singleton_wires_downloader() { ); let from_global = NewPipe::downloader().expect("downloader registered"); - let resp = from_global.get("https://httpbin.org/get").expect("transport"); + let resp = from_global + .execute(Request::get("https://httpbin.org/get").build()) + .expect("transport"); assert_eq!(resp.response_code(), 200); assert_eq!(NewPipe::preferred_localization().localization_code(), "en-GB"); } diff --git a/tests/js_phase2_offline.rs b/tests/js_phase2_offline.rs index ea34f12..3e3a6ad 100644 --- a/tests/js_phase2_offline.rs +++ b/tests/js_phase2_offline.rs @@ -14,7 +14,7 @@ // * url_with_throttling_parameter_deobfuscated round-trip changes &n= // and caches the result -use strawcore::youtube::js::{signature, nsig, runtime, DeobfError}; +use strawcore_core::youtube::js::{signature, nsig, runtime, DeobfError}; // Synthetic minified player.js — replicates the shape of real YT player.js. //