reliability+safety: nsig shape fixes, bounded self-heal, no url/id leaks, revived JS tests
Some checks failed
gitleaks / scan (push) Failing after 2s

- nsig name resolution: route direct-vs-array on capture-group PARTICIPATION,
  not declared-group count (regexes 6/7 carry an optional array group) — a
  direct-call player.js shape no longer hard-fails NsigArrayLookupFailed.
- nsig fixup_function: also strip the `if(typeof X==="undefined")return a;`
  guard on the `function name(a){...}` body shape (previously required a
  leading `;`), so the regex-fallback shape no longer silently yields an
  identity result that gets cached as a permanent throttle.
- player_manager: never hold the state lock across the network; serialize
  player.js downloads behind a fetch gate; add a per-artifact failure memo
  (5-min cooldown) so a persistently-broken player.js costs one refetch per
  open instead of a ~20-25 x 1.7MB storm. Transient-rotation self-heal
  preserved (eval failure -> invalidate + refetch, no memo).
- errors no longer leak watch URLs / video ids: From<reqwest::Error> uses
  without_url(); Recaptcha + link errors reduce to scheme+host (paths like
  /embed/<id> and /shorts/<id> carried the id in the path).
- revive tests/ after the strawcore-core rename; js_phase2_offline green.
  133 lib + 7 integration tests, clippy -D clean.
This commit is contained in:
Cobb 2026-07-28 22:31:20 -07:00
parent 50fd74c874
commit 988e67414d
13 changed files with 907 additions and 140 deletions

View file

@ -81,7 +81,17 @@ impl Downloader for ReqwestDownloader {
let url_after_redirects = resp.url().to_string();
if status.as_u16() == 429 {
return Err(NetworkError::Recaptcha { url: url_after_redirects });
// Privacy: this URL ends up in error strings (and from there in
// Kotlin exception messages / logs, and the deobf failure memo).
// YouTube URLs carry the video id in the query (`?v=` / `&id=`)
// AND in some paths (`/embed/<id>`, `/shorts/<id>`), so keep only
// scheme+host. The Response's latest_url below stays full — it's
// data consumed by redirect-tracking logic, not a message.
let mut stripped = resp.url().clone();
stripped.set_query(None);
stripped.set_fragment(None);
stripped.set_path("/");
return Err(NetworkError::Recaptcha { url: stripped.to_string() });
}
let code = status.as_u16();

View file

@ -83,7 +83,12 @@ pub enum ExtractionError {
impl From<reqwest::Error> for NetworkError {
fn from(e: reqwest::Error) -> Self {
NetworkError::Transport(e.to_string())
// Privacy: reqwest's Display appends " for url (…)" whenever the
// error carries a URL — for player requests that URL contains
// `id=<videoId>`, i.e. what the user was watching. Error strings
// become Kotlin exception messages and land in logcat / exported
// logs, so strip the URL at this single choke point.
NetworkError::Transport(e.without_url().to_string())
}
}

View file

@ -40,15 +40,28 @@ static SCRIPT_TAG: Lazy<Regex> =
/// Extracts the player.js URL + body. Tries iframe_api first, falls back
/// to the embed page on any failure (matches NPE's try/catch flow).
pub fn extract_javascript_player_code(video_id: &str) -> Result<(String, String), DeobfError> {
let downloader = NewPipe::downloader().ok_or(DeobfError::DownloaderMissing)?;
let url = extract_javascript_player_url(video_id)?;
let body = download_player_code(&url)?;
Ok((url, body))
}
/// Discovers the CURRENT player.js URL only (a few-KB iframe_api fetch —
/// cheap next to the ~1.7 MB body). Split out so PlayerManager's failure
/// memo can probe "did YouTube rotate player.js?" without paying for a
/// body download that would deterministically re-fail extraction.
pub fn extract_javascript_player_url(video_id: &str) -> Result<String, DeobfError> {
let downloader = NewPipe::downloader().ok_or(DeobfError::DownloaderMissing)?;
let url = match extract_from_iframe(&*downloader) {
Ok(u) => u,
Err(_iframe_err) => extract_from_embed(&*downloader, video_id)?,
};
let cleaned = clean_javascript_url(&url)?;
let body = download_javascript_code(&*downloader, &cleaned)?;
Ok((cleaned, body))
clean_javascript_url(&url)
}
/// Downloads the player.js body for an already-discovered URL.
pub fn download_player_code(url: &str) -> Result<String, DeobfError> {
let downloader = NewPipe::downloader().ok_or(DeobfError::DownloaderMissing)?;
download_javascript_code(&*downloader, url)
}
fn extract_from_iframe(downloader: &dyn Downloader) -> Result<String, DeobfError> {

View file

@ -23,7 +23,10 @@ pub mod signature;
use thiserror::Error;
#[derive(Debug, Error)]
// Clone: PlayerManager's failure memo replays the original extraction
// error on calls suppressed by the cooldown (mirrors NPE's cached
// ParsingException fields).
#[derive(Debug, Clone, Error)]
pub enum DeobfError {
#[error("could not fetch iframe_api: {0}")]
FetchIframe(String),

View file

@ -112,6 +112,15 @@ fn deobfuscation_function_name(player_code: &str) -> Result<String, DeobfError>
// NPE's `groupCount()` excludes group 0, so:
// len() == 2 → 1 capture → direct name
// len() == 3 → 2 captures → array indirection
//
// BUT: `caps.len()` is pattern-static — it counts DECLARED groups,
// not the ones that PARTICIPATED in the match. Regexes 6/7 declare
// the array-access group inside an optional `(?:@ARRAY@)?`, so a
// direct-call player.js shape matches with group 2 absent. Route on
// participation: group 2 unmatched → the group-1 name IS the answer.
// (NPE's Java would NPE on `Integer.parseInt(matcher.group(2))` here
// — a latent upstream bug; the optional group's evident intent, and
// yt-dlp's guarded equivalent, is the direct name.)
match caps.len() {
2 => {
if let Some(m) = caps.get(1) {
@ -119,8 +128,14 @@ fn deobfuscation_function_name(player_code: &str) -> Result<String, DeobfError>
}
}
3 => {
let Some(index_m) = caps.get(2) else {
if let Some(m) = caps.get(1) {
return Ok(m.as_str().to_string());
}
continue;
};
let array_name = caps.get(1).map(|m| m.as_str()).unwrap_or_default();
let index_str = caps.get(2).map(|m| m.as_str()).unwrap_or_default();
let index_str = index_m.as_str();
let index: usize = index_str.parse().map_err(|_| {
DeobfError::NsigArrayLookupFailed(format!("bad index: {index_str}"))
})?;
@ -185,8 +200,18 @@ fn deobfuscation_function_body_regex(
/// Strips `if(typeof X==="undefined")return <firstArg>;` so the function
/// actually runs standalone. NPE adds this 2024-12-29 (`56595bd9d`).
///
/// Must handle BOTH body shapes this crate produces:
/// * lexer path: `name=function(a){…};`
/// * regex-fallback path: `function name(a){…}`
///
/// NPE's FUNCTION_ARGUMENTS_REGEX requires the `=` (its regex-fallback
/// shape would throw from matchGroup1); ours accepts both so the fallback
/// path doesn't silently no-op — a surviving guard makes the function
/// return its input unchanged, which then gets CACHED as the deobfuscated
/// n-param → permanent silent throttling.
pub fn fixup_function(function: &str) -> Result<String, DeobfError> {
let args_re = Regex::new(r"=\s*function\s*\(\s*([^)]*)\s*\)")
let args_re = Regex::new(r"(?:=\s*)?function\s*[a-zA-Z0-9$_]*\s*\(\s*([^)]*)\s*\)")
.map_err(|e| DeobfError::NsigBodyParseFailed(e.to_string()))?;
let first_arg = args_re
.captures(function)
@ -203,13 +228,18 @@ pub fn fixup_function(function: &str) -> Result<String, DeobfError> {
// Substitute with an alternation of fully-quoted `"undefined"` /
// `'undefined'` forms. Loosens slightly (allows `"undefined'`) but
// real player.js always uses balanced quotes; harmless.
//
// NPE's EARLY_RETURN_REGEX anchors on a preceding `;` only. When the
// guard is the FIRST statement of the body (regex-fallback shape:
// `function name(a){if(typeof X==="undefined")return a;…}`) the
// preceding token is `{` — accept both and re-emit whichever matched.
let early_return_re_src = format!(
r#"(?s);\s*if\s*\(\s*typeof\s+[a-zA-Z0-9$_]+\s*===?\s*(?:"undefined"|'undefined')\s*\)\s*return\s+{};"#,
r#"(?s)([;{{])\s*if\s*\(\s*typeof\s+[a-zA-Z0-9$_]+\s*===?\s*(?:"undefined"|'undefined')\s*\)\s*return\s+{};"#,
regex::escape(&first_arg)
);
let er_re = Regex::new(&early_return_re_src)
.map_err(|e| DeobfError::NsigBodyParseFailed(e.to_string()))?;
Ok(er_re.replace(function, ";").to_string())
Ok(er_re.replace(function, "$1").to_string())
}
#[cfg(test)]
@ -286,4 +316,63 @@ mod tests {
Err(e) => panic!("expected name match, got {e:?}"),
}
}
#[test]
fn regex_6_direct_call_without_array_access() {
// Regex 6 (String.fromCharCode(110)) with a DIRECT call — the
// optional `(?:\[(\d+)])?` group does not participate. Must route
// to the group-1 name, not the array-indirection branch (which
// would hard-fail with NsigArrayLookupFailed on the empty index).
let src = r#"WL=function(a){a.j=1};(b=String.fromCharCode(110),c=a.get(b))&&(c=mfn(c),a.set(b,c))"#;
match deobfuscation_function_name(src) {
Ok(n) => assert_eq!(n, "mfn"),
Err(e) => panic!("direct-call shape must yield the direct name, got {e:?}"),
}
}
#[test]
fn regex_7_direct_call_without_array_access() {
// Regex 7 (.get("n")) with a direct call — same non-participating
// optional group.
let src = r#"(c=d.get("n"))&&(e=Nfn(e),d.set("n",e))"#;
match deobfuscation_function_name(src) {
Ok(n) => assert_eq!(n, "Nfn"),
Err(e) => panic!("direct-call shape must yield the direct name, got {e:?}"),
}
}
#[test]
fn fixup_strips_guard_on_function_declaration_shape() {
// The regex-fallback body shape: `function name(a){…}` — no `=`
// before `function`, and the guard sits right after `{` with no
// preceding `;`. The old fixup no-op'd on both counts, letting the
// early return survive → nsig returns its input unchanged → the
// identity result gets cached → permanent silent throttling.
let body = r#"function m85(p){if(typeof RUQ==="undefined")return p;var a=p.split("");a.reverse();return a.join("");}"#;
let fixed = fixup_function(body).unwrap();
assert!(
!fixed.contains("typeof RUQ"),
"guard must be stripped on the function-declaration shape, got: {fixed}"
);
assert!(fixed.contains(r#"var a=p.split("");"#));
}
#[test]
fn fixup_function_declaration_shape_runs_non_identity() {
// End-to-end: the fixed fallback-shape body must actually transform
// its input when run (i.e. the guard is gone, not just renamed).
let body = r#"function m85(p){if(typeof RUQ==="undefined")return p;var a=p.split("");a.reverse();return a.join("");}"#;
let fixed = fixup_function(body).unwrap();
let out = crate::youtube::js::runtime::run(&fixed, "m85", "abc123").unwrap();
assert_eq!(out, "321cba");
}
#[test]
fn fixup_lexer_shape_guard_at_start_of_body() {
// Lexer shape (`name=function(...)`) whose guard is the FIRST
// statement — preceded by `{`, not `;`. Also must strip.
let body = r#"m85=function(p){if(typeof RUQ==="undefined")return p;var a=p.split("");return a.join("");}"#;
let fixed = fixup_function(body).unwrap();
assert!(!fixed.contains("typeof RUQ"));
}
}

View file

@ -9,21 +9,49 @@
// * cached_nsig_name + snippet
// * cached_throttling_params — obfuscated → deobfuscated cache (per-session)
//
// Failure handling: on ANY deobf/extraction error, invalidate() drops the
// fetched player.js and everything derived from it, so the NEXT call re-fetches
// a fresh player.js and retries. YouTube rotates player.js every few hours, so
// a failure is far more often "our cached player.js went stale" than "this
// player.js is unparseable" — invalidating lets the next attempt self-heal.
// (This replaces the previous sticky-error flags, which were set on failure and
// never cleared by any code path, permanently wedging ALL extraction for the
// process lifetime after a single routine rotation. straw audit, 2026-07-04.)
// Failure handling — three cases, each tuned so a single player.js rotation
// self-heals while a persistently-unparseable player.js can't trigger a
// per-format refetch storm (~20-25 × ~1.7 MB per video open — the H2 finding
// of the 2026-07-28 audit; the previous invalidate-on-any-error design had no
// memory of "I just refetched and it STILL failed"):
//
// NPE uses static fields and is not thread-safe; callers serialize. We give the
// same shape via a `Mutex<ManagerState>` — call sites can still hammer it from
// * Artifact extraction fails (regex bank miss / body parse): arm a
// per-artifact failure memo keyed by the player_url that failed —
// mirrors NPE's cached ParsingException fields (sigTimestampExtractionEx
// et al.), except NPE's are sticky until clearAllCaches() while ours
// expire after FAILURE_COOLDOWN. The installed player.js is KEPT so
// sibling artifacts (e.g. a working signature timestamp) stay usable.
// While the memo is live the original error is replayed with zero
// network. After the cooldown, discovery re-runs (a few-KB iframe_api
// probe): a rotated player.js is fetched fresh and retried; an
// unchanged one just re-arms the memo — extraction is deterministic,
// so re-downloading ~1.7 MB to re-fail is pure waste.
//
// * Eval (QuickJS run) fails: invalidate() everything so the next call
// re-fetches — the stale-rotation self-heal (a cached player.js whose
// function chokes on a new-generation n-param is fixed by fetching the
// current player.js). If the snippet came from a player.js downloaded
// by THIS call, a refetch would rebuild the identical snippet, so the
// memo is armed instead — bounding the storm to one refetch per open.
//
// * Fetch fails (network): plain error, no memo — transient by nature.
//
// Locking: `state` (Mutex<ManagerState>) is NEVER held across the network.
// `fetch_gate` serializes player.js downloads so concurrent extractions
// don't each re-download (~1.7 MB); waiters re-check `state` after taking
// the gate and find the code already installed. Lock order is always
// fetch_gate → state; state is released before any fetch. (The previous
// design held the state mutex across ensure_player_code's downloads,
// serializing every concurrent extraction behind up-to-30 s requests —
// the S6 finding.)
//
// NPE uses static fields and is not thread-safe; callers serialize. We give
// the same shape via the state mutex — call sites can still hammer it from
// multiple threads safely.
use parking_lot::Mutex;
use std::collections::HashMap;
use std::time::{Duration, Instant};
use crate::youtube::js::extractor;
use crate::youtube::js::nsig;
@ -31,6 +59,96 @@ use crate::youtube::js::runtime;
use crate::youtube::js::signature;
use crate::youtube::js::DeobfError;
/// How long a failure memo suppresses re-fetch/re-parse attempts for the
/// player.js that produced it. YouTube rotates player.js every few hours,
/// so five minutes bounds the retry storm without meaningfully delaying
/// recovery once a genuinely new player.js ships (recovery is also
/// immediate whenever a different player_url is discovered).
const FAILURE_COOLDOWN: Duration = Duration::from_secs(5 * 60);
/// One derived artifact's failure record. `player_url` pins the player.js
/// generation the failure belongs to: a different discovered URL retires
/// the memo instantly (rotation self-heal), an identical one replays
/// `error` without re-downloading (extraction is deterministic).
struct FailMemo {
player_url: String,
error: DeobfError,
at: Instant,
}
/// The three derived artifacts, each with its own cache + failure memo —
/// mirrors NPE's three cached exception fields. Kept independent so e.g.
/// a signature-timestamp regex miss doesn't block nsig extraction that
/// would succeed on the same player.js.
#[derive(Clone, Copy)]
enum Derived {
SigTimestamp,
Sig,
Nsig,
}
impl Derived {
fn memo<'a>(&self, state: &'a ManagerState) -> &'a Option<FailMemo> {
match self {
Derived::SigTimestamp => &state.ts_fail,
Derived::Sig => &state.sig_fail,
Derived::Nsig => &state.nsig_fail,
}
}
fn memo_mut<'a>(&self, state: &'a mut ManagerState) -> &'a mut Option<FailMemo> {
match self {
Derived::SigTimestamp => &mut state.ts_fail,
Derived::Sig => &mut state.sig_fail,
Derived::Nsig => &mut state.nsig_fail,
}
}
}
/// Verdict of a memo check, decided under the state lock.
enum MemoVerdict {
/// No live memo applies — proceed normally.
Proceed,
/// Memo cooldown expired — proceed, but treat this URL as known-bad:
/// discover first and skip the ~1.7 MB body download if it's unchanged.
Probe(String),
/// Memo is live — replay the memoized error, zero work.
Replay(DeobfError),
}
fn memo_verdict(state: &ManagerState, memo: &Option<FailMemo>) -> MemoVerdict {
let Some(m) = memo else {
return MemoVerdict::Proceed;
};
let live = m.at.elapsed() < FAILURE_COOLDOWN;
let installed_same = state.player_code.is_some()
&& state.player_url.as_deref() == Some(m.player_url.as_str());
if installed_same {
if live {
MemoVerdict::Replay(m.error.clone())
} else {
MemoVerdict::Probe(m.player_url.clone())
}
} else if state.player_code.is_some() {
// A different player.js has been installed since the failure —
// retry this artifact against it.
MemoVerdict::Proceed
} else if live {
MemoVerdict::Replay(m.error.clone())
} else {
MemoVerdict::Probe(m.player_url.clone())
}
}
/// Outcome of ensure_player_code.
enum Ensured {
/// player_code is installed; `fresh` = this call downloaded it.
Ready { fresh: bool },
/// The probe found YouTube still serving the caller's known-bad URL —
/// no body was downloaded.
StillBad,
}
#[derive(Default)]
struct ManagerState {
player_url: Option<String>,
@ -41,13 +159,21 @@ struct ManagerState {
nsig_name: Option<String>,
nsig_snippet: Option<String>,
throttling_param_cache: HashMap<String, String>,
// Failure memos — deliberately NOT cleared by invalidate(): they must
// survive the invalidate that accompanies the failure they record.
// Retired by: a different player_url being installed, their artifact
// building successfully, clear_all_caches(), or cooldown expiry.
ts_fail: Option<FailMemo>,
sig_fail: Option<FailMemo>,
nsig_fail: Option<FailMemo>,
}
impl ManagerState {
/// Drop the fetched player.js and everything derived from it, so the next
/// call re-fetches a fresh player.js and re-extracts. Called on any
/// deobf/extraction failure so a stale-player.js error self-heals on retry
/// instead of wedging extraction for the process lifetime.
/// call re-fetches a fresh player.js and re-extracts. Called on eval
/// failures (stale-player.js self-heal) and before installing a newly
/// downloaded player.js. Failure memos are kept — see the field comment.
fn invalidate(&mut self) {
self.player_url = None;
self.player_code = None;
@ -62,13 +188,17 @@ impl ManagerState {
}
pub struct PlayerManager {
inner: Mutex<ManagerState>,
state: Mutex<ManagerState>,
/// Serializes player.js downloads WITHOUT blocking `state` readers.
/// Held only while discovering/downloading; never while `state` is held.
fetch_gate: Mutex<()>,
}
impl PlayerManager {
pub fn new() -> Self {
Self {
inner: Mutex::new(ManagerState::default()),
state: Mutex::new(ManagerState::default()),
fetch_gate: Mutex::new(()),
}
}
@ -79,23 +209,14 @@ impl PlayerManager {
}
pub fn signature_timestamp(&self, video_id: &str) -> Result<i32, DeobfError> {
let r = self.signature_timestamp_inner(video_id);
if r.is_err() {
self.inner.lock().invalidate();
}
r
}
fn signature_timestamp_inner(&self, video_id: &str) -> Result<i32, DeobfError> {
let mut state = self.inner.lock();
if let Some(ts) = state.signature_timestamp {
return Ok(ts);
}
Self::ensure_player_code(&mut state, video_id)?;
let code = state.player_code.as_deref().unwrap();
let ts = signature::signature_timestamp(code)?;
state.signature_timestamp = Some(ts);
Ok(ts)
self.derived_artifact(
video_id,
Derived::SigTimestamp,
|state| state.signature_timestamp,
signature::signature_timestamp,
|state, ts| state.signature_timestamp = Some(*ts),
)
.map(|(ts, _)| ts)
}
pub fn deobfuscate_signature(
@ -103,56 +224,37 @@ impl PlayerManager {
video_id: &str,
obfuscated_signature: &str,
) -> Result<String, DeobfError> {
let r = self.deobfuscate_signature_inner(video_id, obfuscated_signature);
if r.is_err() {
self.inner.lock().invalidate();
}
r
}
let (snippet, fresh) = self.derived_artifact(
video_id,
Derived::Sig,
|state| state.sig_snippet.clone(),
signature::build_deobfuscator,
|state, snippet| state.sig_snippet = Some(snippet.clone()),
)?;
fn deobfuscate_signature_inner(
&self,
video_id: &str,
obfuscated_signature: &str,
) -> Result<String, DeobfError> {
let snippet = {
let mut state = self.inner.lock();
if state.sig_snippet.is_none() {
Self::ensure_player_code(&mut state, video_id)?;
let code = state.player_code.as_deref().unwrap();
let s = signature::build_deobfuscator(code)?;
state.sig_snippet = Some(s);
}
state.sig_snippet.clone().unwrap()
};
let result = runtime::run(
match runtime::run(
&snippet,
signature::DEOBFUSCATION_FUNCTION_NAME,
obfuscated_signature,
)?;
if result == "null" {
return Ok(String::new()); // NPE: Objects.requireNonNullElse(..., "")
) {
Ok(result) => {
if result == "null" {
Ok(String::new()) // NPE: Objects.requireNonNullElse(..., "")
} else {
Ok(result)
}
}
Err(e) => {
self.on_eval_failure(Derived::Sig, fresh, &e);
Err(e)
}
}
Ok(result)
}
pub fn url_with_throttling_parameter_deobfuscated(
&self,
video_id: &str,
streaming_url: &str,
) -> Result<String, DeobfError> {
let r = self.url_with_throttling_parameter_deobfuscated_inner(video_id, streaming_url);
if r.is_err() {
self.inner.lock().invalidate();
}
r
}
fn url_with_throttling_parameter_deobfuscated_inner(
&self,
video_id: &str,
streaming_url: &str,
) -> Result<String, DeobfError> {
let obf = match nsig::throttling_parameter_from_url(streaming_url) {
Some(s) => s,
@ -160,34 +262,41 @@ impl PlayerManager {
};
{
let state = self.inner.lock();
let state = self.state.lock();
if let Some(cached) = state.throttling_param_cache.get(&obf) {
return Ok(streaming_url.replace(&obf, cached));
}
}
let (name, snippet) = {
let mut state = self.inner.lock();
if state.nsig_snippet.is_none() {
Self::ensure_player_code(&mut state, video_id)?;
let code = state.player_code.as_deref().unwrap();
let (n, s) = nsig::build_deobfuscator(code)?;
state.nsig_name = Some(n);
state.nsig_snippet = Some(s);
}
(
state.nsig_name.clone().unwrap(),
state.nsig_snippet.clone().unwrap(),
)
};
let ((name, snippet), fresh) = self.derived_artifact(
video_id,
Derived::Nsig,
|state| match (&state.nsig_name, &state.nsig_snippet) {
(Some(n), Some(s)) => Some((n.clone(), s.clone())),
_ => None,
},
nsig::build_deobfuscator,
|state, (n, s)| {
state.nsig_name = Some(n.clone());
state.nsig_snippet = Some(s.clone());
},
)?;
let deobf = runtime::run(&snippet, &name, &obf)?;
let deobf = match runtime::run(&snippet, &name, &obf) {
Ok(d) => d,
Err(e) => {
self.on_eval_failure(Derived::Nsig, fresh, &e);
return Err(e);
}
};
if deobf.is_empty() {
return Err(DeobfError::NsigEmpty);
let e = DeobfError::NsigEmpty;
self.on_eval_failure(Derived::Nsig, fresh, &e);
return Err(e);
}
{
let mut state = self.inner.lock();
let mut state = self.state.lock();
state
.throttling_param_cache
.insert(obf.clone(), deobf.clone());
@ -197,29 +306,230 @@ impl PlayerManager {
}
pub fn throttling_parameter_cache_size(&self) -> usize {
self.inner.lock().throttling_param_cache.len()
self.state.lock().throttling_param_cache.len()
}
pub fn clear_all_caches(&self) {
self.inner.lock().invalidate();
let mut state = self.state.lock();
state.invalidate();
state.ts_fail = None;
state.sig_fail = None;
state.nsig_fail = None;
}
pub fn clear_throttling_parameters_cache(&self) {
self.inner.lock().throttling_param_cache.clear();
self.state.lock().throttling_param_cache.clear();
}
pub fn player_url(&self) -> Option<String> {
self.inner.lock().player_url.clone()
self.state.lock().player_url.clone()
}
fn ensure_player_code(state: &mut ManagerState, video_id: &str) -> Result<(), DeobfError> {
if state.player_code.is_some() {
return Ok(());
/// Shared spine for the three derived artifacts: cache fast path →
/// failure-memo verdict → make player code available (fetching outside
/// the state lock) → build + store under the lock, arming the memo on
/// extraction failure. Returns the artifact plus whether the player.js
/// it was built from was downloaded by THIS call (`fresh` — feeds the
/// eval-failure policy).
fn derived_artifact<T>(
&self,
video_id: &str,
which: Derived,
cached: impl Fn(&ManagerState) -> Option<T>,
build: impl Fn(&str) -> Result<T, DeobfError>,
store: impl Fn(&mut ManagerState, &T),
) -> Result<(T, bool), DeobfError> {
// Two passes: the second absorbs the narrow races (concurrent
// invalidate between ensure and the build lock; memo retired by
// clear_all_caches between verdict and probe).
for _ in 0..2 {
let known_bad: Option<String> = {
let state = self.state.lock();
if let Some(v) = cached(&state) {
return Ok((v, false));
}
match memo_verdict(&state, which.memo(&state)) {
MemoVerdict::Replay(e) => return Err(e),
MemoVerdict::Probe(url) => Some(url),
MemoVerdict::Proceed => None,
}
};
let ensured = match self.ensure_player_code(video_id, known_bad.as_deref()) {
Ok(e) => e,
Err(fetch_err) => {
if known_bad.is_some() {
// The post-cooldown probe itself failed (network).
// Re-arm the cooldown so we don't probe on every
// subsequent call while the network is down.
let mut state = self.state.lock();
if let Some(m) = which.memo_mut(&mut state).as_mut() {
m.at = Instant::now();
}
}
return Err(fetch_err);
}
};
let fresh = match ensured {
Ensured::Ready { fresh } => fresh,
Ensured::StillBad => {
// YouTube still serves the player.js this artifact
// failed on — replay the original error and restart
// the cooldown.
let mut state = self.state.lock();
match which.memo_mut(&mut state).as_mut() {
Some(m) => {
m.at = Instant::now();
return Err(m.error.clone());
}
// Memo retired concurrently (clear_all_caches) —
// loop and take the fresh-slate path.
None => continue,
}
}
};
let mut state = self.state.lock();
let built = match state.player_code.as_deref() {
Some(code) => build(code),
// Invalidated between ensure and here — loop to re-fetch.
None => continue,
};
return match built {
Ok(v) => {
store(&mut state, &v);
// A successful build proves any lingering memo for this
// artifact records a superseded player.js.
*which.memo_mut(&mut state) = None;
Ok((v, fresh))
}
Err(e) => {
// Extraction failed. Keep the installed player.js —
// sibling artifacts may extract fine from it — and arm
// the memo so every further call within the cooldown
// replays instead of re-fetching ~1.7 MB per format.
let player_url = state.player_url.clone().unwrap_or_default();
*which.memo_mut(&mut state) = Some(FailMemo {
player_url,
error: e.clone(),
at: Instant::now(),
});
Err(e)
}
};
}
Err(DeobfError::FetchPlayerCode(
"player.js unavailable after retry (concurrent cache invalidation)".into(),
))
}
/// Eval (QuickJS) failures: invalidate so the next call re-fetches —
/// the stale-rotation self-heal (a cached player.js whose function
/// chokes on a new-generation n-param is fixed by fetching the current
/// player.js). When the failing snippet came from a player.js this very
/// call downloaded (`fresh`), a refetch would rebuild the identical
/// snippet — arm the memo instead so the remaining ~20-25 formats of
/// the open replay the error instead of each re-downloading.
fn on_eval_failure(&self, which: Derived, fresh: bool, error: &DeobfError) {
let mut state = self.state.lock();
let player_url = state.player_url.clone().unwrap_or_default();
state.invalidate();
if fresh {
*which.memo_mut(&mut state) = Some(FailMemo {
player_url,
error: error.clone(),
at: Instant::now(),
});
}
}
/// Makes `state.player_code` available. The network work happens with
/// only `fetch_gate` held — never the state lock — so cache-hit readers
/// stay unblocked and concurrent fetchers collapse into one download.
///
/// `known_bad_url`: the caller's memoized failing player URL. Discovery
/// still runs, but if YouTube is serving that exact URL the ~1.7 MB body
/// download is skipped (`StillBad`) — extraction is deterministic, so it
/// could only re-fail.
fn ensure_player_code(
&self,
video_id: &str,
known_bad_url: Option<&str>,
) -> Result<Ensured, DeobfError> {
// Fast path — usable code already installed.
{
let state = self.state.lock();
if state.player_code.is_some()
&& (known_bad_url.is_none() || state.player_url.as_deref() != known_bad_url)
{
return Ok(Ensured::Ready { fresh: false });
}
}
// One fetcher at a time. Waiters block HERE (not on `state`), then
// usually find the winner's code installed.
let _gate = self.fetch_gate.lock();
{
let state = self.state.lock();
if state.player_code.is_some()
&& (known_bad_url.is_none() || state.player_url.as_deref() != known_bad_url)
{
return Ok(Ensured::Ready { fresh: false });
}
}
// Discovery (iframe_api / embed fallback) — a few KB.
let url = extractor::extract_javascript_player_url(video_id)?;
if Some(url.as_str()) == known_bad_url {
return Ok(Ensured::StillBad);
}
// Body download — ~1.7 MB.
let code = extractor::download_player_code(&url)?;
let mut state = self.state.lock();
let state: &mut ManagerState = &mut state;
// Everything derived from any previously installed player.js dies
// with it; memos recorded against OTHER player URLs are retired so
// a genuinely new player.js gets a clean slate for every artifact.
state.invalidate();
for memo in [
&mut state.ts_fail,
&mut state.sig_fail,
&mut state.nsig_fail,
] {
if memo.as_ref().is_some_and(|m| m.player_url != url) {
*memo = None;
}
}
let (url, code) = extractor::extract_javascript_player_code(video_id)?;
state.player_url = Some(url);
state.player_code = Some(code);
Ok(())
Ok(Ensured::Ready { fresh: true })
}
/// Test hook: age every armed memo past FAILURE_COOLDOWN so the probe
/// path can be exercised without a five-minute sleep.
#[cfg(test)]
fn test_expire_memos(&self) {
let mut state = self.state.lock();
let state: &mut ManagerState = &mut state;
for memo in [
&mut state.ts_fail,
&mut state.sig_fail,
&mut state.nsig_fail,
] {
if let Some(m) = memo.as_mut() {
if let Some(aged) = Instant::now().checked_sub(FAILURE_COOLDOWN) {
m.at = aged;
}
}
}
}
/// Test hook: is the state mutex currently free? Used to prove the
/// player.js fetch runs without holding it.
#[cfg(test)]
fn state_lock_is_free(&self) -> bool {
self.state.try_lock().is_some()
}
}
@ -232,6 +542,13 @@ impl Default for PlayerManager {
#[cfg(test)]
mod tests {
use super::*;
use crate::downloader::request::Request;
use crate::downloader::response::Response;
use crate::downloader::Downloader;
use crate::exceptions::NetworkError;
use crate::newpipe::NewPipe;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{mpsc, Arc};
#[test]
fn assembled_sig_snippet_runs_against_synthetic_player() {
@ -264,4 +581,281 @@ mod tests {
mgr.clear_all_caches();
assert!(mgr.player_url().is_none());
}
// ---- failure-memo / refetch-storm tests -------------------------------
//
// These register a stub Downloader in the process-global NewPipe
// singleton, so they serialize on GLOBAL_DOWNLOADER_LOCK (no other lib
// test touches the downloader). Each uses its own PlayerManager
// instance, so manager state never crosses tests.
static GLOBAL_DOWNLOADER_LOCK: Mutex<()> = Mutex::new(());
/// Both channel endpoints wrapped in Mutex for the Downloader Sync bound.
type FetchBlocker = (Mutex<mpsc::Sender<()>>, Mutex<mpsc::Receiver<()>>);
/// Serves a synthetic iframe_api (announcing `hash`) and a synthetic
/// player.js body; counts every request.
struct StubDownloader {
hash: Mutex<String>,
player_js: Mutex<String>,
requests: AtomicUsize,
/// When set, the player.js body request signals `entered` then
/// blocks until `release` fires — used by the lock-freedom test.
block_player_fetch: Option<FetchBlocker>,
}
impl StubDownloader {
fn new(hash: &str, player_js: &str) -> Self {
Self {
hash: Mutex::new(hash.to_string()),
player_js: Mutex::new(player_js.to_string()),
requests: AtomicUsize::new(0),
block_player_fetch: None,
}
}
fn rotate(&self, hash: &str, player_js: &str) {
*self.hash.lock() = hash.to_string();
*self.player_js.lock() = player_js.to_string();
}
fn count(&self) -> usize {
self.requests.load(Ordering::SeqCst)
}
}
impl Downloader for StubDownloader {
fn execute(&self, request: Request) -> Result<Response, NetworkError> {
self.requests.fetch_add(1, Ordering::SeqCst);
let url = request.url().to_string();
if url.contains("iframe_api") {
let h = self.hash.lock().clone();
// Shape matched by IFRAME_RES_JS_BASE_PLAYER_HASH (escaped
// `\/` form, 8-char [a-z0-9] hash).
let body = format!(
r#"src:"https://www.youtube.com/s/player\/{h}\/player_ias.vflset/en_US/www-embed.js""#
);
Ok(Response::new(200, "OK", Default::default(), body, url))
} else if url.contains("/s/player/") {
if let Some((entered, release)) = &self.block_player_fetch {
let _ = entered.lock().send(());
let _ = release
.lock()
.recv_timeout(std::time::Duration::from_secs(10));
}
let body = self.player_js.lock().clone();
Ok(Response::new(200, "OK", Default::default(), body, url))
} else {
Err(NetworkError::Transport("stub: unexpected request".into()))
}
}
}
// Parseable by ts + NOTHING nsig-shaped → nsig build fails while the
// signature timestamp extracts fine.
const PLAYER_NSIG_BROKEN: &str = r#"var foo={signatureTimestamp:19999};var unrelated=1;"#;
// nsig name via regex 0 (m85 … the greedy `.*` reaches dummy's
// `return Q[1]` on the same line); body via lexer; self-contained
// (no outer-global references), so eval succeeds for any input.
const PLAYER_GOOD: &str = r#"m85=function(p){var a=p.split("");a.reverse();return a.join("");};dummy=function(q){return Q[1];};var foo={signatureTimestamp:20244};"#;
// Parses fine but throws for EVERY input at eval time.
const PLAYER_EVAL_THROWS: &str = r#"m85=function(p){throw Error("boom");return Q[1];};"#;
// Parses fine; throws only for the input "evil".
const PLAYER_EVAL_SELECTIVE: &str = r#"m85=function(p){if(p==="evil")throw Error("bad");var a=p.split("");a.reverse();return a.join("");};dummy=function(q){return Q[1];};"#;
fn install(stub: &Arc<StubDownloader>) {
NewPipe::init(stub.clone() as Arc<dyn Downloader>);
}
#[test]
fn nsig_build_failure_memoizes_instead_of_refetching_per_format() {
let _g = GLOBAL_DOWNLOADER_LOCK.lock();
let stub = Arc::new(StubDownloader::new("aaaaaaaa", PLAYER_NSIG_BROKEN));
install(&stub);
let mgr = PlayerManager::new();
// First format: discovery + body download (2 requests), build fails.
let err = mgr
.url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=AAA")
.unwrap_err();
assert!(matches!(err, DeobfError::NsigFuncNotFound), "got {err:?}");
assert_eq!(stub.count(), 2);
// Remaining formats of the same open: replayed error, ZERO network.
for _ in 0..5 {
let err = mgr
.url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=BBB")
.unwrap_err();
assert!(matches!(err, DeobfError::NsigFuncNotFound));
}
assert_eq!(stub.count(), 2, "memo must suppress refetching");
}
#[test]
fn artifact_memos_are_independent() {
let _g = GLOBAL_DOWNLOADER_LOCK.lock();
let stub = Arc::new(StubDownloader::new("aaaaaaaa", PLAYER_NSIG_BROKEN));
install(&stub);
let mgr = PlayerManager::new();
// nsig fails and memoizes…
assert!(mgr
.url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=AAA")
.is_err());
assert_eq!(stub.count(), 2);
// …but the player.js stays installed, so the signature timestamp
// extracts from it without any further network.
assert_eq!(mgr.signature_timestamp("vid").unwrap(), 19999);
assert_eq!(stub.count(), 2);
}
#[test]
fn rotation_after_cooldown_fetches_fresh_and_recovers() {
let _g = GLOBAL_DOWNLOADER_LOCK.lock();
let stub = Arc::new(StubDownloader::new("aaaaaaaa", PLAYER_NSIG_BROKEN));
install(&stub);
let mgr = PlayerManager::new();
assert!(mgr
.url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=AAA")
.is_err());
assert_eq!(stub.count(), 2);
// YouTube rotates to a player.js our bank parses.
stub.rotate("bbbbbbbb", PLAYER_GOOD);
// Within the cooldown the memo still replays — no network.
assert!(mgr
.url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=AAA")
.is_err());
assert_eq!(stub.count(), 2);
// After the cooldown: probe discovers the NEW url → full fetch →
// extraction + eval succeed.
mgr.test_expire_memos();
let out = mgr
.url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=abc")
.unwrap();
assert_eq!(out, "https://x/?n=cba");
assert_eq!(stub.count(), 4, "probe (iframe) + fresh body download");
}
#[test]
fn post_cooldown_probe_on_unchanged_player_skips_body_download() {
let _g = GLOBAL_DOWNLOADER_LOCK.lock();
let stub = Arc::new(StubDownloader::new("aaaaaaaa", PLAYER_NSIG_BROKEN));
install(&stub);
let mgr = PlayerManager::new();
assert!(mgr
.url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=AAA")
.is_err());
assert_eq!(stub.count(), 2);
mgr.test_expire_memos();
// Probe: one iframe_api discovery, SAME hash → no body download,
// memo re-armed.
assert!(mgr
.url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=AAA")
.is_err());
assert_eq!(stub.count(), 3);
// Re-armed cooldown suppresses even the probe.
assert!(mgr
.url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=AAA")
.is_err());
assert_eq!(stub.count(), 3);
}
#[test]
fn transient_eval_failure_still_self_heals_via_refetch() {
let _g = GLOBAL_DOWNLOADER_LOCK.lock();
let stub = Arc::new(StubDownloader::new("aaaaaaaa", PLAYER_EVAL_SELECTIVE));
install(&stub);
let mgr = PlayerManager::new();
// Working nsig on the first video.
let out = mgr
.url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=good1")
.unwrap();
assert_eq!(out, "https://x/?n=1doog");
assert_eq!(stub.count(), 2);
// A new-generation n-param chokes the CACHED (not fresh) snippet →
// invalidate, NO memo. No network spent on the failing call itself.
assert!(mgr
.url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=evil")
.is_err());
assert_eq!(stub.count(), 2);
// YouTube has rotated; the very next call re-fetches and succeeds —
// the transient-rotation self-heal must NOT be suppressed.
stub.rotate("bbbbbbbb", PLAYER_GOOD);
let out = mgr
.url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=evil")
.unwrap();
assert_eq!(out, "https://x/?n=live");
assert_eq!(stub.count(), 4);
}
#[test]
fn persistent_eval_failure_on_fresh_player_memoizes() {
let _g = GLOBAL_DOWNLOADER_LOCK.lock();
let stub = Arc::new(StubDownloader::new("aaaaaaaa", PLAYER_EVAL_THROWS));
install(&stub);
let mgr = PlayerManager::new();
// Fresh download + successful build, then eval throws → memo.
let err = mgr
.url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=AAA")
.unwrap_err();
assert!(matches!(err, DeobfError::JsRuntimeFailed(_)), "got {err:?}");
assert_eq!(stub.count(), 2);
// Every further format replays without network.
for _ in 0..5 {
assert!(mgr
.url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=BBB")
.is_err());
}
assert_eq!(stub.count(), 2);
}
#[test]
fn state_lock_stays_free_during_player_fetch() {
let _g = GLOBAL_DOWNLOADER_LOCK.lock();
let (entered_tx, entered_rx) = mpsc::channel();
let (release_tx, release_rx) = mpsc::channel();
let mut stub = StubDownloader::new("aaaaaaaa", PLAYER_GOOD);
stub.block_player_fetch = Some((Mutex::new(entered_tx), Mutex::new(release_rx)));
let stub = Arc::new(stub);
install(&stub);
let mgr: &'static PlayerManager = Box::leak(Box::new(PlayerManager::new()));
let worker = std::thread::spawn(move || {
mgr.url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=abc")
});
// Wait until the worker is INSIDE the player.js body download.
entered_rx
.recv_timeout(std::time::Duration::from_secs(10))
.expect("worker never reached the player.js fetch");
// The whole point of the fetch_gate: readers aren't blocked while
// ~1.7 MB downloads. The old design held this mutex across the
// network and serialized everything behind it.
assert!(
mgr.state_lock_is_free(),
"state mutex must not be held across the player.js download"
);
assert_eq!(mgr.throttling_parameter_cache_size(), 0); // must not block
release_tx.send(()).unwrap();
let out = worker.join().unwrap().unwrap();
assert_eq!(out, "https://x/?n=cba");
}
}

View file

@ -10,7 +10,7 @@
use url::Url;
use crate::youtube::linkhandler::{host_is_youtube, LinkError};
use crate::youtube::linkhandler::{host_is_youtube, redact_url, LinkError};
#[derive(Clone, Debug, Eq, PartialEq)]
pub enum ChannelIdentifier {
@ -25,8 +25,10 @@ pub enum ChannelIdentifier {
}
pub fn parse(url_str: &str) -> Result<ChannelIdentifier, LinkError> {
// Privacy: never embed the raw input in the error (see stream.rs) —
// the parse reason alone suffices.
let url = Url::parse(url_str)
.map_err(|e| LinkError::InvalidUrl(format!("{url_str}: {e}")))?;
.map_err(|e| LinkError::InvalidUrl(e.to_string()))?;
let host = url
.host_str()
.ok_or_else(|| LinkError::InvalidUrl("no host".into()))?;
@ -37,32 +39,32 @@ pub fn parse(url_str: &str) -> Result<ChannelIdentifier, LinkError> {
if let Some(rest) = path.strip_prefix("/channel/") {
let id = rest.split('/').next().unwrap_or("");
if id.is_empty() {
return Err(LinkError::MissingId(url_str.into()));
return Err(LinkError::MissingId(redact_url(&url)));
}
return Ok(ChannelIdentifier::DirectId(id.into()));
}
if let Some(rest) = path.strip_prefix("/c/") {
let s = rest.split('/').next().unwrap_or("");
if s.is_empty() {
return Err(LinkError::MissingId(url_str.into()));
return Err(LinkError::MissingId(redact_url(&url)));
}
return Ok(ChannelIdentifier::Custom(s.into()));
}
if let Some(rest) = path.strip_prefix("/user/") {
let s = rest.split('/').next().unwrap_or("");
if s.is_empty() {
return Err(LinkError::MissingId(url_str.into()));
return Err(LinkError::MissingId(redact_url(&url)));
}
return Ok(ChannelIdentifier::LegacyUser(s.into()));
}
if let Some(rest) = path.strip_prefix("/@") {
let s = rest.split('/').next().unwrap_or("");
if s.is_empty() {
return Err(LinkError::MissingId(url_str.into()));
return Err(LinkError::MissingId(redact_url(&url)));
}
return Ok(ChannelIdentifier::Handle(s.into()));
}
Err(LinkError::MissingId(url_str.into()))
Err(LinkError::MissingId(redact_url(&url)))
}
pub fn channel_url(channel_id: &str) -> String {

View file

@ -34,6 +34,20 @@ pub const ACCEPTED_HOSTS: &[&str] = &[
"www.youtube-nocookie.com",
];
/// Renders a URL as scheme+host only, for embedding in error strings.
/// Privacy: link errors reach Kotlin exception messages / logs. Query params
/// carry user browsing data (`v=<videoId>`, `list=<playlistId>`) AND several
/// YouTube PATHS embed the id (`/embed/<id>`, `/shorts/<id>`, `/clip/<id>`,
/// `/live/<id>`, `youtu.be/<id>`) — so drop the path too; the host alone says
/// which endpoint failed without leaking what the user was watching.
pub(crate) fn redact_url(url: &url::Url) -> String {
let mut u = url.clone();
u.set_query(None);
u.set_fragment(None);
u.set_path("/");
u.to_string()
}
pub fn host_is_youtube(host: &str) -> bool {
let h = host.to_ascii_lowercase();
let h = h.strip_prefix("www.").unwrap_or(&h);
@ -66,4 +80,24 @@ mod tests {
assert!(!host_is_youtube("piped.video"));
assert!(!host_is_youtube("evil.com"));
}
#[test]
fn redact_url_drops_id_bearing_path_query_and_fragment() {
// Every one of these carries the video/playlist id somewhere the old
// query-only strip would have kept (the path, for embed/shorts/youtu.be).
let cases = [
"https://www.youtube.com/embed/dQw4w9WgXcQ?v=secret#frag",
"https://www.youtube.com/shorts/dQw4w9WgXcQ",
"https://youtu.be/dQw4w9WgXcQ",
"https://www.youtube.com/watch?v=dQw4w9WgXcQ&list=PLsecretlist",
];
for c in cases {
let u = url::Url::parse(c).unwrap();
let r = redact_url(&u);
assert!(!r.contains("dQw4w9WgXcQ"), "leaked video id: {r}");
assert!(!r.contains("PLsecretlist"), "leaked list id: {r}");
// scheme + host only (no port in any of these cases).
assert_eq!(r, format!("{}://{}/", u.scheme(), u.host_str().unwrap()));
}
}
}

View file

@ -15,7 +15,7 @@ use once_cell::sync::Lazy;
use regex::Regex;
use url::Url;
use crate::youtube::linkhandler::{host_is_youtube, LinkError};
use crate::youtube::linkhandler::{host_is_youtube, redact_url, LinkError};
const VIDEO_ID_LEN: usize = 11;
@ -42,8 +42,11 @@ fn extract_video_id_inner(input_url: &str, depth: u8) -> Result<String, LinkErro
// the JVM via UniFFI. One level is enough for the legitimate
// share-from-attribution-app case.
const MAX_ATTRIBUTION_DEPTH: u8 = 1;
// Privacy: never embed the raw input in the error — even a malformed
// paste can carry a video id / user browsing data, and link errors
// reach exception messages / logs. The parse reason alone suffices.
let url = Url::parse(input_url)
.map_err(|e| LinkError::InvalidUrl(format!("{input_url}: {e}")))?;
.map_err(|e| LinkError::InvalidUrl(e.to_string()))?;
let host = url
.host_str()
.ok_or_else(|| LinkError::InvalidUrl("no host".into()))?;
@ -93,7 +96,7 @@ fn extract_video_id_inner(input_url: &str, depth: u8) -> Result<String, LinkErro
}
let id = candidate
.ok_or_else(|| LinkError::MissingId(input_url.into()))?;
.ok_or_else(|| LinkError::MissingId(redact_url(&url)))?;
if !is_valid_video_id(&id) {
return Err(LinkError::MalformedId(id));
}

View file

@ -640,11 +640,15 @@ fn process_url(
// nsig deobf — unconditional (quick-exits internally if no `n=` present).
// On failure, fall back to the throttled ORIGINAL url rather than dropping
// the format or aborting the video: YouTube still serves it (rate-limited),
// and PlayerManager has already invalidated its cache so the NEXT format
// re-fetches a fresh player.js and deobfuscates cleanly. One routine
// player.js rotation therefore costs at most one throttled format, not the
// whole video. (NPE parity; straw audit 2026-07-04.)
// the format or aborting the video: YouTube still serves it (rate-limited).
// A stale-cache eval failure invalidates PlayerManager's cache, so the
// NEXT format re-fetches a fresh player.js and deobfuscates cleanly — one
// routine rotation costs at most one throttled format. If the FRESH
// player.js also fails (regex bank miss — a persistent breakage), the
// manager's failure memo replays the error for the remaining formats with
// zero network instead of re-downloading ~1.7 MB per format, and retries
// after a cooldown / as soon as a new player.js ships. (NPE parity; straw
// audits 2026-07-04 + 2026-07-28 H2.)
url = PlayerManager::instance()
.url_with_throttling_parameter_deobfuscated(video_id, &url)
.unwrap_or(url);

View file

@ -195,8 +195,12 @@ fn post_youtube(
}
let resp = downloader.execute(builder.build())?;
if resp.response_code() != 200 {
// Privacy: the full URL carries `id=<videoId>` in the query — never
// put it in an error string (they reach Kotlin exception messages
// and logs). Scheme+host+path identifies the endpoint just fine.
let endpoint = url.split(['?', '#']).next().unwrap_or(url);
return Err(ExtractionError::Network(NetworkError::Transport(format!(
"HTTP {} from {url}",
"HTTP {} from {endpoint}",
resp.response_code()
))));
}

View file

@ -9,16 +9,22 @@
use std::sync::Arc;
use strawcore::downloader::request::Request;
use strawcore::downloader::ReqwestDownloader;
use strawcore::exceptions::NetworkError;
use strawcore::localization::{ContentCountry, Localization};
use strawcore::{Downloader, NewPipe};
use strawcore_core::downloader::request::Request;
use strawcore_core::downloader::ReqwestDownloader;
use strawcore_core::exceptions::NetworkError;
use strawcore_core::localization::{ContentCountry, Localization};
use strawcore_core::{Downloader, NewPipe};
/// The Downloader trait dropped its `get` convenience (commit f917e4a);
/// mirror it here so the suite reads as before.
fn get(dl: &impl Downloader, url: &str) -> Result<strawcore_core::Response, NetworkError> {
dl.execute(Request::get(url).build())
}
#[test]
fn get_through_default_downloader() {
let dl = ReqwestDownloader::new().expect("build downloader");
let resp = dl.get("https://httpbin.org/get").expect("transport");
let resp = get(&dl, "https://httpbin.org/get").expect("transport");
assert_eq!(resp.response_code(), 200);
assert!(resp.response_body().contains("\"url\""));
}
@ -26,9 +32,7 @@ fn get_through_default_downloader() {
#[test]
fn latest_url_follows_redirects() {
let dl = ReqwestDownloader::new().expect("build downloader");
let resp = dl
.get("https://httpbin.org/redirect/3")
.expect("transport");
let resp = get(&dl, "https://httpbin.org/redirect/3").expect("transport");
assert_eq!(resp.response_code(), 200);
assert!(
resp.latest_url().ends_with("/get"),
@ -40,14 +44,14 @@ fn latest_url_follows_redirects() {
#[test]
fn non_2xx_returns_ok_not_err() {
let dl = ReqwestDownloader::new().expect("build downloader");
let resp = dl.get("https://httpbin.org/status/404").expect("transport");
let resp = get(&dl, "https://httpbin.org/status/404").expect("transport");
assert_eq!(resp.response_code(), 404);
}
#[test]
fn http_429_surfaces_as_recaptcha_err() {
let dl = ReqwestDownloader::new().expect("build downloader");
let err = dl.get("https://httpbin.org/status/429").expect_err("429 must be NetworkError");
let err = get(&dl, "https://httpbin.org/status/429").expect_err("429 must be NetworkError");
match err {
NetworkError::Recaptcha { url } => assert!(url.contains("/status/429")),
other => panic!("expected Recaptcha, got {other:?}"),
@ -72,8 +76,8 @@ fn localization_header_attached_when_enabled() {
#[test]
fn header_keys_lowercased_in_response() {
let dl = ReqwestDownloader::new().expect("build downloader");
let resp = dl.get("https://httpbin.org/get").expect("transport");
for (k, _) in resp.response_headers() {
let resp = get(&dl, "https://httpbin.org/get").expect("transport");
for k in resp.response_headers().keys() {
assert_eq!(k, &k.to_ascii_lowercase(), "header key {k} not lowercased");
}
}
@ -88,7 +92,9 @@ fn newpipe_singleton_wires_downloader() {
);
let from_global = NewPipe::downloader().expect("downloader registered");
let resp = from_global.get("https://httpbin.org/get").expect("transport");
let resp = from_global
.execute(Request::get("https://httpbin.org/get").build())
.expect("transport");
assert_eq!(resp.response_code(), 200);
assert_eq!(NewPipe::preferred_localization().localization_code(), "en-GB");
}

View file

@ -14,7 +14,7 @@
// * url_with_throttling_parameter_deobfuscated round-trip changes &n=
// and caches the result
use strawcore::youtube::js::{signature, nsig, runtime, DeobfError};
use strawcore_core::youtube::js::{signature, nsig, runtime, DeobfError};
// Synthetic minified player.js — replicates the shape of real YT player.js.
//