reliability+safety: nsig shape fixes, bounded self-heal, no url/id leaks, revived JS tests
Some checks failed
gitleaks / scan (push) Failing after 2s
Some checks failed
gitleaks / scan (push) Failing after 2s
- nsig name resolution: route direct-vs-array on capture-group PARTICIPATION,
not declared-group count (regexes 6/7 carry an optional array group) — a
direct-call player.js shape no longer hard-fails NsigArrayLookupFailed.
- nsig fixup_function: also strip the `if(typeof X==="undefined")return a;`
guard on the `function name(a){...}` body shape (previously required a
leading `;`), so the regex-fallback shape no longer silently yields an
identity result that gets cached as a permanent throttle.
- player_manager: never hold the state lock across the network; serialize
player.js downloads behind a fetch gate; add a per-artifact failure memo
(5-min cooldown) so a persistently-broken player.js costs one refetch per
open instead of a ~20-25 x 1.7MB storm. Transient-rotation self-heal
preserved (eval failure -> invalidate + refetch, no memo).
- errors no longer leak watch URLs / video ids: From<reqwest::Error> uses
without_url(); Recaptcha + link errors reduce to scheme+host (paths like
/embed/<id> and /shorts/<id> carried the id in the path).
- revive tests/ after the strawcore-core rename; js_phase2_offline green.
133 lib + 7 integration tests, clippy -D clean.
This commit is contained in:
parent
50fd74c874
commit
988e67414d
13 changed files with 907 additions and 140 deletions
|
|
@ -81,7 +81,17 @@ impl Downloader for ReqwestDownloader {
|
||||||
let url_after_redirects = resp.url().to_string();
|
let url_after_redirects = resp.url().to_string();
|
||||||
|
|
||||||
if status.as_u16() == 429 {
|
if status.as_u16() == 429 {
|
||||||
return Err(NetworkError::Recaptcha { url: url_after_redirects });
|
// Privacy: this URL ends up in error strings (and from there in
|
||||||
|
// Kotlin exception messages / logs, and the deobf failure memo).
|
||||||
|
// YouTube URLs carry the video id in the query (`?v=` / `&id=`)
|
||||||
|
// AND in some paths (`/embed/<id>`, `/shorts/<id>`), so keep only
|
||||||
|
// scheme+host. The Response's latest_url below stays full — it's
|
||||||
|
// data consumed by redirect-tracking logic, not a message.
|
||||||
|
let mut stripped = resp.url().clone();
|
||||||
|
stripped.set_query(None);
|
||||||
|
stripped.set_fragment(None);
|
||||||
|
stripped.set_path("/");
|
||||||
|
return Err(NetworkError::Recaptcha { url: stripped.to_string() });
|
||||||
}
|
}
|
||||||
|
|
||||||
let code = status.as_u16();
|
let code = status.as_u16();
|
||||||
|
|
|
||||||
|
|
@ -83,7 +83,12 @@ pub enum ExtractionError {
|
||||||
|
|
||||||
impl From<reqwest::Error> for NetworkError {
|
impl From<reqwest::Error> for NetworkError {
|
||||||
fn from(e: reqwest::Error) -> Self {
|
fn from(e: reqwest::Error) -> Self {
|
||||||
NetworkError::Transport(e.to_string())
|
// Privacy: reqwest's Display appends " for url (…)" whenever the
|
||||||
|
// error carries a URL — for player requests that URL contains
|
||||||
|
// `id=<videoId>`, i.e. what the user was watching. Error strings
|
||||||
|
// become Kotlin exception messages and land in logcat / exported
|
||||||
|
// logs, so strip the URL at this single choke point.
|
||||||
|
NetworkError::Transport(e.without_url().to_string())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -40,15 +40,28 @@ static SCRIPT_TAG: Lazy<Regex> =
|
||||||
/// Extracts the player.js URL + body. Tries iframe_api first, falls back
|
/// Extracts the player.js URL + body. Tries iframe_api first, falls back
|
||||||
/// to the embed page on any failure (matches NPE's try/catch flow).
|
/// to the embed page on any failure (matches NPE's try/catch flow).
|
||||||
pub fn extract_javascript_player_code(video_id: &str) -> Result<(String, String), DeobfError> {
|
pub fn extract_javascript_player_code(video_id: &str) -> Result<(String, String), DeobfError> {
|
||||||
let downloader = NewPipe::downloader().ok_or(DeobfError::DownloaderMissing)?;
|
let url = extract_javascript_player_url(video_id)?;
|
||||||
|
let body = download_player_code(&url)?;
|
||||||
|
Ok((url, body))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Discovers the CURRENT player.js URL only (a few-KB iframe_api fetch —
|
||||||
|
/// cheap next to the ~1.7 MB body). Split out so PlayerManager's failure
|
||||||
|
/// memo can probe "did YouTube rotate player.js?" without paying for a
|
||||||
|
/// body download that would deterministically re-fail extraction.
|
||||||
|
pub fn extract_javascript_player_url(video_id: &str) -> Result<String, DeobfError> {
|
||||||
|
let downloader = NewPipe::downloader().ok_or(DeobfError::DownloaderMissing)?;
|
||||||
let url = match extract_from_iframe(&*downloader) {
|
let url = match extract_from_iframe(&*downloader) {
|
||||||
Ok(u) => u,
|
Ok(u) => u,
|
||||||
Err(_iframe_err) => extract_from_embed(&*downloader, video_id)?,
|
Err(_iframe_err) => extract_from_embed(&*downloader, video_id)?,
|
||||||
};
|
};
|
||||||
let cleaned = clean_javascript_url(&url)?;
|
clean_javascript_url(&url)
|
||||||
let body = download_javascript_code(&*downloader, &cleaned)?;
|
}
|
||||||
Ok((cleaned, body))
|
|
||||||
|
/// Downloads the player.js body for an already-discovered URL.
|
||||||
|
pub fn download_player_code(url: &str) -> Result<String, DeobfError> {
|
||||||
|
let downloader = NewPipe::downloader().ok_or(DeobfError::DownloaderMissing)?;
|
||||||
|
download_javascript_code(&*downloader, url)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn extract_from_iframe(downloader: &dyn Downloader) -> Result<String, DeobfError> {
|
fn extract_from_iframe(downloader: &dyn Downloader) -> Result<String, DeobfError> {
|
||||||
|
|
|
||||||
|
|
@ -23,7 +23,10 @@ pub mod signature;
|
||||||
|
|
||||||
use thiserror::Error;
|
use thiserror::Error;
|
||||||
|
|
||||||
#[derive(Debug, Error)]
|
// Clone: PlayerManager's failure memo replays the original extraction
|
||||||
|
// error on calls suppressed by the cooldown (mirrors NPE's cached
|
||||||
|
// ParsingException fields).
|
||||||
|
#[derive(Debug, Clone, Error)]
|
||||||
pub enum DeobfError {
|
pub enum DeobfError {
|
||||||
#[error("could not fetch iframe_api: {0}")]
|
#[error("could not fetch iframe_api: {0}")]
|
||||||
FetchIframe(String),
|
FetchIframe(String),
|
||||||
|
|
|
||||||
|
|
@ -112,6 +112,15 @@ fn deobfuscation_function_name(player_code: &str) -> Result<String, DeobfError>
|
||||||
// NPE's `groupCount()` excludes group 0, so:
|
// NPE's `groupCount()` excludes group 0, so:
|
||||||
// len() == 2 → 1 capture → direct name
|
// len() == 2 → 1 capture → direct name
|
||||||
// len() == 3 → 2 captures → array indirection
|
// len() == 3 → 2 captures → array indirection
|
||||||
|
//
|
||||||
|
// BUT: `caps.len()` is pattern-static — it counts DECLARED groups,
|
||||||
|
// not the ones that PARTICIPATED in the match. Regexes 6/7 declare
|
||||||
|
// the array-access group inside an optional `(?:@ARRAY@)?`, so a
|
||||||
|
// direct-call player.js shape matches with group 2 absent. Route on
|
||||||
|
// participation: group 2 unmatched → the group-1 name IS the answer.
|
||||||
|
// (NPE's Java would NPE on `Integer.parseInt(matcher.group(2))` here
|
||||||
|
// — a latent upstream bug; the optional group's evident intent, and
|
||||||
|
// yt-dlp's guarded equivalent, is the direct name.)
|
||||||
match caps.len() {
|
match caps.len() {
|
||||||
2 => {
|
2 => {
|
||||||
if let Some(m) = caps.get(1) {
|
if let Some(m) = caps.get(1) {
|
||||||
|
|
@ -119,8 +128,14 @@ fn deobfuscation_function_name(player_code: &str) -> Result<String, DeobfError>
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
3 => {
|
3 => {
|
||||||
|
let Some(index_m) = caps.get(2) else {
|
||||||
|
if let Some(m) = caps.get(1) {
|
||||||
|
return Ok(m.as_str().to_string());
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
};
|
||||||
let array_name = caps.get(1).map(|m| m.as_str()).unwrap_or_default();
|
let array_name = caps.get(1).map(|m| m.as_str()).unwrap_or_default();
|
||||||
let index_str = caps.get(2).map(|m| m.as_str()).unwrap_or_default();
|
let index_str = index_m.as_str();
|
||||||
let index: usize = index_str.parse().map_err(|_| {
|
let index: usize = index_str.parse().map_err(|_| {
|
||||||
DeobfError::NsigArrayLookupFailed(format!("bad index: {index_str}"))
|
DeobfError::NsigArrayLookupFailed(format!("bad index: {index_str}"))
|
||||||
})?;
|
})?;
|
||||||
|
|
@ -185,8 +200,18 @@ fn deobfuscation_function_body_regex(
|
||||||
|
|
||||||
/// Strips `if(typeof X==="undefined")return <firstArg>;` so the function
|
/// Strips `if(typeof X==="undefined")return <firstArg>;` so the function
|
||||||
/// actually runs standalone. NPE adds this 2024-12-29 (`56595bd9d`).
|
/// actually runs standalone. NPE adds this 2024-12-29 (`56595bd9d`).
|
||||||
|
///
|
||||||
|
/// Must handle BOTH body shapes this crate produces:
|
||||||
|
/// * lexer path: `name=function(a){…};`
|
||||||
|
/// * regex-fallback path: `function name(a){…}`
|
||||||
|
///
|
||||||
|
/// NPE's FUNCTION_ARGUMENTS_REGEX requires the `=` (its regex-fallback
|
||||||
|
/// shape would throw from matchGroup1); ours accepts both so the fallback
|
||||||
|
/// path doesn't silently no-op — a surviving guard makes the function
|
||||||
|
/// return its input unchanged, which then gets CACHED as the deobfuscated
|
||||||
|
/// n-param → permanent silent throttling.
|
||||||
pub fn fixup_function(function: &str) -> Result<String, DeobfError> {
|
pub fn fixup_function(function: &str) -> Result<String, DeobfError> {
|
||||||
let args_re = Regex::new(r"=\s*function\s*\(\s*([^)]*)\s*\)")
|
let args_re = Regex::new(r"(?:=\s*)?function\s*[a-zA-Z0-9$_]*\s*\(\s*([^)]*)\s*\)")
|
||||||
.map_err(|e| DeobfError::NsigBodyParseFailed(e.to_string()))?;
|
.map_err(|e| DeobfError::NsigBodyParseFailed(e.to_string()))?;
|
||||||
let first_arg = args_re
|
let first_arg = args_re
|
||||||
.captures(function)
|
.captures(function)
|
||||||
|
|
@ -203,13 +228,18 @@ pub fn fixup_function(function: &str) -> Result<String, DeobfError> {
|
||||||
// Substitute with an alternation of fully-quoted `"undefined"` /
|
// Substitute with an alternation of fully-quoted `"undefined"` /
|
||||||
// `'undefined'` forms. Loosens slightly (allows `"undefined'`) but
|
// `'undefined'` forms. Loosens slightly (allows `"undefined'`) but
|
||||||
// real player.js always uses balanced quotes; harmless.
|
// real player.js always uses balanced quotes; harmless.
|
||||||
|
//
|
||||||
|
// NPE's EARLY_RETURN_REGEX anchors on a preceding `;` only. When the
|
||||||
|
// guard is the FIRST statement of the body (regex-fallback shape:
|
||||||
|
// `function name(a){if(typeof X==="undefined")return a;…}`) the
|
||||||
|
// preceding token is `{` — accept both and re-emit whichever matched.
|
||||||
let early_return_re_src = format!(
|
let early_return_re_src = format!(
|
||||||
r#"(?s);\s*if\s*\(\s*typeof\s+[a-zA-Z0-9$_]+\s*===?\s*(?:"undefined"|'undefined')\s*\)\s*return\s+{};"#,
|
r#"(?s)([;{{])\s*if\s*\(\s*typeof\s+[a-zA-Z0-9$_]+\s*===?\s*(?:"undefined"|'undefined')\s*\)\s*return\s+{};"#,
|
||||||
regex::escape(&first_arg)
|
regex::escape(&first_arg)
|
||||||
);
|
);
|
||||||
let er_re = Regex::new(&early_return_re_src)
|
let er_re = Regex::new(&early_return_re_src)
|
||||||
.map_err(|e| DeobfError::NsigBodyParseFailed(e.to_string()))?;
|
.map_err(|e| DeobfError::NsigBodyParseFailed(e.to_string()))?;
|
||||||
Ok(er_re.replace(function, ";").to_string())
|
Ok(er_re.replace(function, "$1").to_string())
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|
@ -286,4 +316,63 @@ mod tests {
|
||||||
Err(e) => panic!("expected name match, got {e:?}"),
|
Err(e) => panic!("expected name match, got {e:?}"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn regex_6_direct_call_without_array_access() {
|
||||||
|
// Regex 6 (String.fromCharCode(110)) with a DIRECT call — the
|
||||||
|
// optional `(?:\[(\d+)])?` group does not participate. Must route
|
||||||
|
// to the group-1 name, not the array-indirection branch (which
|
||||||
|
// would hard-fail with NsigArrayLookupFailed on the empty index).
|
||||||
|
let src = r#"WL=function(a){a.j=1};(b=String.fromCharCode(110),c=a.get(b))&&(c=mfn(c),a.set(b,c))"#;
|
||||||
|
match deobfuscation_function_name(src) {
|
||||||
|
Ok(n) => assert_eq!(n, "mfn"),
|
||||||
|
Err(e) => panic!("direct-call shape must yield the direct name, got {e:?}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn regex_7_direct_call_without_array_access() {
|
||||||
|
// Regex 7 (.get("n")) with a direct call — same non-participating
|
||||||
|
// optional group.
|
||||||
|
let src = r#"(c=d.get("n"))&&(e=Nfn(e),d.set("n",e))"#;
|
||||||
|
match deobfuscation_function_name(src) {
|
||||||
|
Ok(n) => assert_eq!(n, "Nfn"),
|
||||||
|
Err(e) => panic!("direct-call shape must yield the direct name, got {e:?}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn fixup_strips_guard_on_function_declaration_shape() {
|
||||||
|
// The regex-fallback body shape: `function name(a){…}` — no `=`
|
||||||
|
// before `function`, and the guard sits right after `{` with no
|
||||||
|
// preceding `;`. The old fixup no-op'd on both counts, letting the
|
||||||
|
// early return survive → nsig returns its input unchanged → the
|
||||||
|
// identity result gets cached → permanent silent throttling.
|
||||||
|
let body = r#"function m85(p){if(typeof RUQ==="undefined")return p;var a=p.split("");a.reverse();return a.join("");}"#;
|
||||||
|
let fixed = fixup_function(body).unwrap();
|
||||||
|
assert!(
|
||||||
|
!fixed.contains("typeof RUQ"),
|
||||||
|
"guard must be stripped on the function-declaration shape, got: {fixed}"
|
||||||
|
);
|
||||||
|
assert!(fixed.contains(r#"var a=p.split("");"#));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn fixup_function_declaration_shape_runs_non_identity() {
|
||||||
|
// End-to-end: the fixed fallback-shape body must actually transform
|
||||||
|
// its input when run (i.e. the guard is gone, not just renamed).
|
||||||
|
let body = r#"function m85(p){if(typeof RUQ==="undefined")return p;var a=p.split("");a.reverse();return a.join("");}"#;
|
||||||
|
let fixed = fixup_function(body).unwrap();
|
||||||
|
let out = crate::youtube::js::runtime::run(&fixed, "m85", "abc123").unwrap();
|
||||||
|
assert_eq!(out, "321cba");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn fixup_lexer_shape_guard_at_start_of_body() {
|
||||||
|
// Lexer shape (`name=function(...)`) whose guard is the FIRST
|
||||||
|
// statement — preceded by `{`, not `;`. Also must strip.
|
||||||
|
let body = r#"m85=function(p){if(typeof RUQ==="undefined")return p;var a=p.split("");return a.join("");}"#;
|
||||||
|
let fixed = fixup_function(body).unwrap();
|
||||||
|
assert!(!fixed.contains("typeof RUQ"));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -9,21 +9,49 @@
|
||||||
// * cached_nsig_name + snippet
|
// * cached_nsig_name + snippet
|
||||||
// * cached_throttling_params — obfuscated → deobfuscated cache (per-session)
|
// * cached_throttling_params — obfuscated → deobfuscated cache (per-session)
|
||||||
//
|
//
|
||||||
// Failure handling: on ANY deobf/extraction error, invalidate() drops the
|
// Failure handling — three cases, each tuned so a single player.js rotation
|
||||||
// fetched player.js and everything derived from it, so the NEXT call re-fetches
|
// self-heals while a persistently-unparseable player.js can't trigger a
|
||||||
// a fresh player.js and retries. YouTube rotates player.js every few hours, so
|
// per-format refetch storm (~20-25 × ~1.7 MB per video open — the H2 finding
|
||||||
// a failure is far more often "our cached player.js went stale" than "this
|
// of the 2026-07-28 audit; the previous invalidate-on-any-error design had no
|
||||||
// player.js is unparseable" — invalidating lets the next attempt self-heal.
|
// memory of "I just refetched and it STILL failed"):
|
||||||
// (This replaces the previous sticky-error flags, which were set on failure and
|
|
||||||
// never cleared by any code path, permanently wedging ALL extraction for the
|
|
||||||
// process lifetime after a single routine rotation. straw audit, 2026-07-04.)
|
|
||||||
//
|
//
|
||||||
// NPE uses static fields and is not thread-safe; callers serialize. We give the
|
// * Artifact extraction fails (regex bank miss / body parse): arm a
|
||||||
// same shape via a `Mutex<ManagerState>` — call sites can still hammer it from
|
// per-artifact failure memo keyed by the player_url that failed —
|
||||||
|
// mirrors NPE's cached ParsingException fields (sigTimestampExtractionEx
|
||||||
|
// et al.), except NPE's are sticky until clearAllCaches() while ours
|
||||||
|
// expire after FAILURE_COOLDOWN. The installed player.js is KEPT so
|
||||||
|
// sibling artifacts (e.g. a working signature timestamp) stay usable.
|
||||||
|
// While the memo is live the original error is replayed with zero
|
||||||
|
// network. After the cooldown, discovery re-runs (a few-KB iframe_api
|
||||||
|
// probe): a rotated player.js is fetched fresh and retried; an
|
||||||
|
// unchanged one just re-arms the memo — extraction is deterministic,
|
||||||
|
// so re-downloading ~1.7 MB to re-fail is pure waste.
|
||||||
|
//
|
||||||
|
// * Eval (QuickJS run) fails: invalidate() everything so the next call
|
||||||
|
// re-fetches — the stale-rotation self-heal (a cached player.js whose
|
||||||
|
// function chokes on a new-generation n-param is fixed by fetching the
|
||||||
|
// current player.js). If the snippet came from a player.js downloaded
|
||||||
|
// by THIS call, a refetch would rebuild the identical snippet, so the
|
||||||
|
// memo is armed instead — bounding the storm to one refetch per open.
|
||||||
|
//
|
||||||
|
// * Fetch fails (network): plain error, no memo — transient by nature.
|
||||||
|
//
|
||||||
|
// Locking: `state` (Mutex<ManagerState>) is NEVER held across the network.
|
||||||
|
// `fetch_gate` serializes player.js downloads so concurrent extractions
|
||||||
|
// don't each re-download (~1.7 MB); waiters re-check `state` after taking
|
||||||
|
// the gate and find the code already installed. Lock order is always
|
||||||
|
// fetch_gate → state; state is released before any fetch. (The previous
|
||||||
|
// design held the state mutex across ensure_player_code's downloads,
|
||||||
|
// serializing every concurrent extraction behind up-to-30 s requests —
|
||||||
|
// the S6 finding.)
|
||||||
|
//
|
||||||
|
// NPE uses static fields and is not thread-safe; callers serialize. We give
|
||||||
|
// the same shape via the state mutex — call sites can still hammer it from
|
||||||
// multiple threads safely.
|
// multiple threads safely.
|
||||||
|
|
||||||
use parking_lot::Mutex;
|
use parking_lot::Mutex;
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
use crate::youtube::js::extractor;
|
use crate::youtube::js::extractor;
|
||||||
use crate::youtube::js::nsig;
|
use crate::youtube::js::nsig;
|
||||||
|
|
@ -31,6 +59,96 @@ use crate::youtube::js::runtime;
|
||||||
use crate::youtube::js::signature;
|
use crate::youtube::js::signature;
|
||||||
use crate::youtube::js::DeobfError;
|
use crate::youtube::js::DeobfError;
|
||||||
|
|
||||||
|
/// How long a failure memo suppresses re-fetch/re-parse attempts for the
|
||||||
|
/// player.js that produced it. YouTube rotates player.js every few hours,
|
||||||
|
/// so five minutes bounds the retry storm without meaningfully delaying
|
||||||
|
/// recovery once a genuinely new player.js ships (recovery is also
|
||||||
|
/// immediate whenever a different player_url is discovered).
|
||||||
|
const FAILURE_COOLDOWN: Duration = Duration::from_secs(5 * 60);
|
||||||
|
|
||||||
|
/// One derived artifact's failure record. `player_url` pins the player.js
|
||||||
|
/// generation the failure belongs to: a different discovered URL retires
|
||||||
|
/// the memo instantly (rotation self-heal), an identical one replays
|
||||||
|
/// `error` without re-downloading (extraction is deterministic).
|
||||||
|
struct FailMemo {
|
||||||
|
player_url: String,
|
||||||
|
error: DeobfError,
|
||||||
|
at: Instant,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The three derived artifacts, each with its own cache + failure memo —
|
||||||
|
/// mirrors NPE's three cached exception fields. Kept independent so e.g.
|
||||||
|
/// a signature-timestamp regex miss doesn't block nsig extraction that
|
||||||
|
/// would succeed on the same player.js.
|
||||||
|
#[derive(Clone, Copy)]
|
||||||
|
enum Derived {
|
||||||
|
SigTimestamp,
|
||||||
|
Sig,
|
||||||
|
Nsig,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Derived {
|
||||||
|
fn memo<'a>(&self, state: &'a ManagerState) -> &'a Option<FailMemo> {
|
||||||
|
match self {
|
||||||
|
Derived::SigTimestamp => &state.ts_fail,
|
||||||
|
Derived::Sig => &state.sig_fail,
|
||||||
|
Derived::Nsig => &state.nsig_fail,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn memo_mut<'a>(&self, state: &'a mut ManagerState) -> &'a mut Option<FailMemo> {
|
||||||
|
match self {
|
||||||
|
Derived::SigTimestamp => &mut state.ts_fail,
|
||||||
|
Derived::Sig => &mut state.sig_fail,
|
||||||
|
Derived::Nsig => &mut state.nsig_fail,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Verdict of a memo check, decided under the state lock.
|
||||||
|
enum MemoVerdict {
|
||||||
|
/// No live memo applies — proceed normally.
|
||||||
|
Proceed,
|
||||||
|
/// Memo cooldown expired — proceed, but treat this URL as known-bad:
|
||||||
|
/// discover first and skip the ~1.7 MB body download if it's unchanged.
|
||||||
|
Probe(String),
|
||||||
|
/// Memo is live — replay the memoized error, zero work.
|
||||||
|
Replay(DeobfError),
|
||||||
|
}
|
||||||
|
|
||||||
|
fn memo_verdict(state: &ManagerState, memo: &Option<FailMemo>) -> MemoVerdict {
|
||||||
|
let Some(m) = memo else {
|
||||||
|
return MemoVerdict::Proceed;
|
||||||
|
};
|
||||||
|
let live = m.at.elapsed() < FAILURE_COOLDOWN;
|
||||||
|
let installed_same = state.player_code.is_some()
|
||||||
|
&& state.player_url.as_deref() == Some(m.player_url.as_str());
|
||||||
|
if installed_same {
|
||||||
|
if live {
|
||||||
|
MemoVerdict::Replay(m.error.clone())
|
||||||
|
} else {
|
||||||
|
MemoVerdict::Probe(m.player_url.clone())
|
||||||
|
}
|
||||||
|
} else if state.player_code.is_some() {
|
||||||
|
// A different player.js has been installed since the failure —
|
||||||
|
// retry this artifact against it.
|
||||||
|
MemoVerdict::Proceed
|
||||||
|
} else if live {
|
||||||
|
MemoVerdict::Replay(m.error.clone())
|
||||||
|
} else {
|
||||||
|
MemoVerdict::Probe(m.player_url.clone())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Outcome of ensure_player_code.
|
||||||
|
enum Ensured {
|
||||||
|
/// player_code is installed; `fresh` = this call downloaded it.
|
||||||
|
Ready { fresh: bool },
|
||||||
|
/// The probe found YouTube still serving the caller's known-bad URL —
|
||||||
|
/// no body was downloaded.
|
||||||
|
StillBad,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Default)]
|
#[derive(Default)]
|
||||||
struct ManagerState {
|
struct ManagerState {
|
||||||
player_url: Option<String>,
|
player_url: Option<String>,
|
||||||
|
|
@ -41,13 +159,21 @@ struct ManagerState {
|
||||||
nsig_name: Option<String>,
|
nsig_name: Option<String>,
|
||||||
nsig_snippet: Option<String>,
|
nsig_snippet: Option<String>,
|
||||||
throttling_param_cache: HashMap<String, String>,
|
throttling_param_cache: HashMap<String, String>,
|
||||||
|
|
||||||
|
// Failure memos — deliberately NOT cleared by invalidate(): they must
|
||||||
|
// survive the invalidate that accompanies the failure they record.
|
||||||
|
// Retired by: a different player_url being installed, their artifact
|
||||||
|
// building successfully, clear_all_caches(), or cooldown expiry.
|
||||||
|
ts_fail: Option<FailMemo>,
|
||||||
|
sig_fail: Option<FailMemo>,
|
||||||
|
nsig_fail: Option<FailMemo>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ManagerState {
|
impl ManagerState {
|
||||||
/// Drop the fetched player.js and everything derived from it, so the next
|
/// Drop the fetched player.js and everything derived from it, so the next
|
||||||
/// call re-fetches a fresh player.js and re-extracts. Called on any
|
/// call re-fetches a fresh player.js and re-extracts. Called on eval
|
||||||
/// deobf/extraction failure so a stale-player.js error self-heals on retry
|
/// failures (stale-player.js self-heal) and before installing a newly
|
||||||
/// instead of wedging extraction for the process lifetime.
|
/// downloaded player.js. Failure memos are kept — see the field comment.
|
||||||
fn invalidate(&mut self) {
|
fn invalidate(&mut self) {
|
||||||
self.player_url = None;
|
self.player_url = None;
|
||||||
self.player_code = None;
|
self.player_code = None;
|
||||||
|
|
@ -62,13 +188,17 @@ impl ManagerState {
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct PlayerManager {
|
pub struct PlayerManager {
|
||||||
inner: Mutex<ManagerState>,
|
state: Mutex<ManagerState>,
|
||||||
|
/// Serializes player.js downloads WITHOUT blocking `state` readers.
|
||||||
|
/// Held only while discovering/downloading; never while `state` is held.
|
||||||
|
fetch_gate: Mutex<()>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl PlayerManager {
|
impl PlayerManager {
|
||||||
pub fn new() -> Self {
|
pub fn new() -> Self {
|
||||||
Self {
|
Self {
|
||||||
inner: Mutex::new(ManagerState::default()),
|
state: Mutex::new(ManagerState::default()),
|
||||||
|
fetch_gate: Mutex::new(()),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -79,23 +209,14 @@ impl PlayerManager {
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn signature_timestamp(&self, video_id: &str) -> Result<i32, DeobfError> {
|
pub fn signature_timestamp(&self, video_id: &str) -> Result<i32, DeobfError> {
|
||||||
let r = self.signature_timestamp_inner(video_id);
|
self.derived_artifact(
|
||||||
if r.is_err() {
|
video_id,
|
||||||
self.inner.lock().invalidate();
|
Derived::SigTimestamp,
|
||||||
}
|
|state| state.signature_timestamp,
|
||||||
r
|
signature::signature_timestamp,
|
||||||
}
|
|state, ts| state.signature_timestamp = Some(*ts),
|
||||||
|
)
|
||||||
fn signature_timestamp_inner(&self, video_id: &str) -> Result<i32, DeobfError> {
|
.map(|(ts, _)| ts)
|
||||||
let mut state = self.inner.lock();
|
|
||||||
if let Some(ts) = state.signature_timestamp {
|
|
||||||
return Ok(ts);
|
|
||||||
}
|
|
||||||
Self::ensure_player_code(&mut state, video_id)?;
|
|
||||||
let code = state.player_code.as_deref().unwrap();
|
|
||||||
let ts = signature::signature_timestamp(code)?;
|
|
||||||
state.signature_timestamp = Some(ts);
|
|
||||||
Ok(ts)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn deobfuscate_signature(
|
pub fn deobfuscate_signature(
|
||||||
|
|
@ -103,56 +224,37 @@ impl PlayerManager {
|
||||||
video_id: &str,
|
video_id: &str,
|
||||||
obfuscated_signature: &str,
|
obfuscated_signature: &str,
|
||||||
) -> Result<String, DeobfError> {
|
) -> Result<String, DeobfError> {
|
||||||
let r = self.deobfuscate_signature_inner(video_id, obfuscated_signature);
|
let (snippet, fresh) = self.derived_artifact(
|
||||||
if r.is_err() {
|
video_id,
|
||||||
self.inner.lock().invalidate();
|
Derived::Sig,
|
||||||
}
|
|state| state.sig_snippet.clone(),
|
||||||
r
|
signature::build_deobfuscator,
|
||||||
}
|
|state, snippet| state.sig_snippet = Some(snippet.clone()),
|
||||||
|
)?;
|
||||||
|
|
||||||
fn deobfuscate_signature_inner(
|
match runtime::run(
|
||||||
&self,
|
|
||||||
video_id: &str,
|
|
||||||
obfuscated_signature: &str,
|
|
||||||
) -> Result<String, DeobfError> {
|
|
||||||
let snippet = {
|
|
||||||
let mut state = self.inner.lock();
|
|
||||||
if state.sig_snippet.is_none() {
|
|
||||||
Self::ensure_player_code(&mut state, video_id)?;
|
|
||||||
let code = state.player_code.as_deref().unwrap();
|
|
||||||
let s = signature::build_deobfuscator(code)?;
|
|
||||||
state.sig_snippet = Some(s);
|
|
||||||
}
|
|
||||||
state.sig_snippet.clone().unwrap()
|
|
||||||
};
|
|
||||||
|
|
||||||
let result = runtime::run(
|
|
||||||
&snippet,
|
&snippet,
|
||||||
signature::DEOBFUSCATION_FUNCTION_NAME,
|
signature::DEOBFUSCATION_FUNCTION_NAME,
|
||||||
obfuscated_signature,
|
obfuscated_signature,
|
||||||
)?;
|
) {
|
||||||
if result == "null" {
|
Ok(result) => {
|
||||||
return Ok(String::new()); // NPE: Objects.requireNonNullElse(..., "")
|
if result == "null" {
|
||||||
|
Ok(String::new()) // NPE: Objects.requireNonNullElse(..., "")
|
||||||
|
} else {
|
||||||
|
Ok(result)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
self.on_eval_failure(Derived::Sig, fresh, &e);
|
||||||
|
Err(e)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
Ok(result)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn url_with_throttling_parameter_deobfuscated(
|
pub fn url_with_throttling_parameter_deobfuscated(
|
||||||
&self,
|
&self,
|
||||||
video_id: &str,
|
video_id: &str,
|
||||||
streaming_url: &str,
|
streaming_url: &str,
|
||||||
) -> Result<String, DeobfError> {
|
|
||||||
let r = self.url_with_throttling_parameter_deobfuscated_inner(video_id, streaming_url);
|
|
||||||
if r.is_err() {
|
|
||||||
self.inner.lock().invalidate();
|
|
||||||
}
|
|
||||||
r
|
|
||||||
}
|
|
||||||
|
|
||||||
fn url_with_throttling_parameter_deobfuscated_inner(
|
|
||||||
&self,
|
|
||||||
video_id: &str,
|
|
||||||
streaming_url: &str,
|
|
||||||
) -> Result<String, DeobfError> {
|
) -> Result<String, DeobfError> {
|
||||||
let obf = match nsig::throttling_parameter_from_url(streaming_url) {
|
let obf = match nsig::throttling_parameter_from_url(streaming_url) {
|
||||||
Some(s) => s,
|
Some(s) => s,
|
||||||
|
|
@ -160,34 +262,41 @@ impl PlayerManager {
|
||||||
};
|
};
|
||||||
|
|
||||||
{
|
{
|
||||||
let state = self.inner.lock();
|
let state = self.state.lock();
|
||||||
if let Some(cached) = state.throttling_param_cache.get(&obf) {
|
if let Some(cached) = state.throttling_param_cache.get(&obf) {
|
||||||
return Ok(streaming_url.replace(&obf, cached));
|
return Ok(streaming_url.replace(&obf, cached));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let (name, snippet) = {
|
let ((name, snippet), fresh) = self.derived_artifact(
|
||||||
let mut state = self.inner.lock();
|
video_id,
|
||||||
if state.nsig_snippet.is_none() {
|
Derived::Nsig,
|
||||||
Self::ensure_player_code(&mut state, video_id)?;
|
|state| match (&state.nsig_name, &state.nsig_snippet) {
|
||||||
let code = state.player_code.as_deref().unwrap();
|
(Some(n), Some(s)) => Some((n.clone(), s.clone())),
|
||||||
let (n, s) = nsig::build_deobfuscator(code)?;
|
_ => None,
|
||||||
state.nsig_name = Some(n);
|
},
|
||||||
state.nsig_snippet = Some(s);
|
nsig::build_deobfuscator,
|
||||||
}
|
|state, (n, s)| {
|
||||||
(
|
state.nsig_name = Some(n.clone());
|
||||||
state.nsig_name.clone().unwrap(),
|
state.nsig_snippet = Some(s.clone());
|
||||||
state.nsig_snippet.clone().unwrap(),
|
},
|
||||||
)
|
)?;
|
||||||
};
|
|
||||||
|
|
||||||
let deobf = runtime::run(&snippet, &name, &obf)?;
|
let deobf = match runtime::run(&snippet, &name, &obf) {
|
||||||
|
Ok(d) => d,
|
||||||
|
Err(e) => {
|
||||||
|
self.on_eval_failure(Derived::Nsig, fresh, &e);
|
||||||
|
return Err(e);
|
||||||
|
}
|
||||||
|
};
|
||||||
if deobf.is_empty() {
|
if deobf.is_empty() {
|
||||||
return Err(DeobfError::NsigEmpty);
|
let e = DeobfError::NsigEmpty;
|
||||||
|
self.on_eval_failure(Derived::Nsig, fresh, &e);
|
||||||
|
return Err(e);
|
||||||
}
|
}
|
||||||
|
|
||||||
{
|
{
|
||||||
let mut state = self.inner.lock();
|
let mut state = self.state.lock();
|
||||||
state
|
state
|
||||||
.throttling_param_cache
|
.throttling_param_cache
|
||||||
.insert(obf.clone(), deobf.clone());
|
.insert(obf.clone(), deobf.clone());
|
||||||
|
|
@ -197,29 +306,230 @@ impl PlayerManager {
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn throttling_parameter_cache_size(&self) -> usize {
|
pub fn throttling_parameter_cache_size(&self) -> usize {
|
||||||
self.inner.lock().throttling_param_cache.len()
|
self.state.lock().throttling_param_cache.len()
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn clear_all_caches(&self) {
|
pub fn clear_all_caches(&self) {
|
||||||
self.inner.lock().invalidate();
|
let mut state = self.state.lock();
|
||||||
|
state.invalidate();
|
||||||
|
state.ts_fail = None;
|
||||||
|
state.sig_fail = None;
|
||||||
|
state.nsig_fail = None;
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn clear_throttling_parameters_cache(&self) {
|
pub fn clear_throttling_parameters_cache(&self) {
|
||||||
self.inner.lock().throttling_param_cache.clear();
|
self.state.lock().throttling_param_cache.clear();
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn player_url(&self) -> Option<String> {
|
pub fn player_url(&self) -> Option<String> {
|
||||||
self.inner.lock().player_url.clone()
|
self.state.lock().player_url.clone()
|
||||||
}
|
}
|
||||||
|
|
||||||
fn ensure_player_code(state: &mut ManagerState, video_id: &str) -> Result<(), DeobfError> {
|
/// Shared spine for the three derived artifacts: cache fast path →
|
||||||
if state.player_code.is_some() {
|
/// failure-memo verdict → make player code available (fetching outside
|
||||||
return Ok(());
|
/// the state lock) → build + store under the lock, arming the memo on
|
||||||
|
/// extraction failure. Returns the artifact plus whether the player.js
|
||||||
|
/// it was built from was downloaded by THIS call (`fresh` — feeds the
|
||||||
|
/// eval-failure policy).
|
||||||
|
fn derived_artifact<T>(
|
||||||
|
&self,
|
||||||
|
video_id: &str,
|
||||||
|
which: Derived,
|
||||||
|
cached: impl Fn(&ManagerState) -> Option<T>,
|
||||||
|
build: impl Fn(&str) -> Result<T, DeobfError>,
|
||||||
|
store: impl Fn(&mut ManagerState, &T),
|
||||||
|
) -> Result<(T, bool), DeobfError> {
|
||||||
|
// Two passes: the second absorbs the narrow races (concurrent
|
||||||
|
// invalidate between ensure and the build lock; memo retired by
|
||||||
|
// clear_all_caches between verdict and probe).
|
||||||
|
for _ in 0..2 {
|
||||||
|
let known_bad: Option<String> = {
|
||||||
|
let state = self.state.lock();
|
||||||
|
if let Some(v) = cached(&state) {
|
||||||
|
return Ok((v, false));
|
||||||
|
}
|
||||||
|
match memo_verdict(&state, which.memo(&state)) {
|
||||||
|
MemoVerdict::Replay(e) => return Err(e),
|
||||||
|
MemoVerdict::Probe(url) => Some(url),
|
||||||
|
MemoVerdict::Proceed => None,
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let ensured = match self.ensure_player_code(video_id, known_bad.as_deref()) {
|
||||||
|
Ok(e) => e,
|
||||||
|
Err(fetch_err) => {
|
||||||
|
if known_bad.is_some() {
|
||||||
|
// The post-cooldown probe itself failed (network).
|
||||||
|
// Re-arm the cooldown so we don't probe on every
|
||||||
|
// subsequent call while the network is down.
|
||||||
|
let mut state = self.state.lock();
|
||||||
|
if let Some(m) = which.memo_mut(&mut state).as_mut() {
|
||||||
|
m.at = Instant::now();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return Err(fetch_err);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let fresh = match ensured {
|
||||||
|
Ensured::Ready { fresh } => fresh,
|
||||||
|
Ensured::StillBad => {
|
||||||
|
// YouTube still serves the player.js this artifact
|
||||||
|
// failed on — replay the original error and restart
|
||||||
|
// the cooldown.
|
||||||
|
let mut state = self.state.lock();
|
||||||
|
match which.memo_mut(&mut state).as_mut() {
|
||||||
|
Some(m) => {
|
||||||
|
m.at = Instant::now();
|
||||||
|
return Err(m.error.clone());
|
||||||
|
}
|
||||||
|
// Memo retired concurrently (clear_all_caches) —
|
||||||
|
// loop and take the fresh-slate path.
|
||||||
|
None => continue,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut state = self.state.lock();
|
||||||
|
let built = match state.player_code.as_deref() {
|
||||||
|
Some(code) => build(code),
|
||||||
|
// Invalidated between ensure and here — loop to re-fetch.
|
||||||
|
None => continue,
|
||||||
|
};
|
||||||
|
return match built {
|
||||||
|
Ok(v) => {
|
||||||
|
store(&mut state, &v);
|
||||||
|
// A successful build proves any lingering memo for this
|
||||||
|
// artifact records a superseded player.js.
|
||||||
|
*which.memo_mut(&mut state) = None;
|
||||||
|
Ok((v, fresh))
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
// Extraction failed. Keep the installed player.js —
|
||||||
|
// sibling artifacts may extract fine from it — and arm
|
||||||
|
// the memo so every further call within the cooldown
|
||||||
|
// replays instead of re-fetching ~1.7 MB per format.
|
||||||
|
let player_url = state.player_url.clone().unwrap_or_default();
|
||||||
|
*which.memo_mut(&mut state) = Some(FailMemo {
|
||||||
|
player_url,
|
||||||
|
error: e.clone(),
|
||||||
|
at: Instant::now(),
|
||||||
|
});
|
||||||
|
Err(e)
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
Err(DeobfError::FetchPlayerCode(
|
||||||
|
"player.js unavailable after retry (concurrent cache invalidation)".into(),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Eval (QuickJS) failures: invalidate so the next call re-fetches —
|
||||||
|
/// the stale-rotation self-heal (a cached player.js whose function
|
||||||
|
/// chokes on a new-generation n-param is fixed by fetching the current
|
||||||
|
/// player.js). When the failing snippet came from a player.js this very
|
||||||
|
/// call downloaded (`fresh`), a refetch would rebuild the identical
|
||||||
|
/// snippet — arm the memo instead so the remaining ~20-25 formats of
|
||||||
|
/// the open replay the error instead of each re-downloading.
|
||||||
|
fn on_eval_failure(&self, which: Derived, fresh: bool, error: &DeobfError) {
|
||||||
|
let mut state = self.state.lock();
|
||||||
|
let player_url = state.player_url.clone().unwrap_or_default();
|
||||||
|
state.invalidate();
|
||||||
|
if fresh {
|
||||||
|
*which.memo_mut(&mut state) = Some(FailMemo {
|
||||||
|
player_url,
|
||||||
|
error: error.clone(),
|
||||||
|
at: Instant::now(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Makes `state.player_code` available. The network work happens with
|
||||||
|
/// only `fetch_gate` held — never the state lock — so cache-hit readers
|
||||||
|
/// stay unblocked and concurrent fetchers collapse into one download.
|
||||||
|
///
|
||||||
|
/// `known_bad_url`: the caller's memoized failing player URL. Discovery
|
||||||
|
/// still runs, but if YouTube is serving that exact URL the ~1.7 MB body
|
||||||
|
/// download is skipped (`StillBad`) — extraction is deterministic, so it
|
||||||
|
/// could only re-fail.
|
||||||
|
fn ensure_player_code(
|
||||||
|
&self,
|
||||||
|
video_id: &str,
|
||||||
|
known_bad_url: Option<&str>,
|
||||||
|
) -> Result<Ensured, DeobfError> {
|
||||||
|
// Fast path — usable code already installed.
|
||||||
|
{
|
||||||
|
let state = self.state.lock();
|
||||||
|
if state.player_code.is_some()
|
||||||
|
&& (known_bad_url.is_none() || state.player_url.as_deref() != known_bad_url)
|
||||||
|
{
|
||||||
|
return Ok(Ensured::Ready { fresh: false });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// One fetcher at a time. Waiters block HERE (not on `state`), then
|
||||||
|
// usually find the winner's code installed.
|
||||||
|
let _gate = self.fetch_gate.lock();
|
||||||
|
{
|
||||||
|
let state = self.state.lock();
|
||||||
|
if state.player_code.is_some()
|
||||||
|
&& (known_bad_url.is_none() || state.player_url.as_deref() != known_bad_url)
|
||||||
|
{
|
||||||
|
return Ok(Ensured::Ready { fresh: false });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Discovery (iframe_api / embed fallback) — a few KB.
|
||||||
|
let url = extractor::extract_javascript_player_url(video_id)?;
|
||||||
|
if Some(url.as_str()) == known_bad_url {
|
||||||
|
return Ok(Ensured::StillBad);
|
||||||
|
}
|
||||||
|
// Body download — ~1.7 MB.
|
||||||
|
let code = extractor::download_player_code(&url)?;
|
||||||
|
|
||||||
|
let mut state = self.state.lock();
|
||||||
|
let state: &mut ManagerState = &mut state;
|
||||||
|
// Everything derived from any previously installed player.js dies
|
||||||
|
// with it; memos recorded against OTHER player URLs are retired so
|
||||||
|
// a genuinely new player.js gets a clean slate for every artifact.
|
||||||
|
state.invalidate();
|
||||||
|
for memo in [
|
||||||
|
&mut state.ts_fail,
|
||||||
|
&mut state.sig_fail,
|
||||||
|
&mut state.nsig_fail,
|
||||||
|
] {
|
||||||
|
if memo.as_ref().is_some_and(|m| m.player_url != url) {
|
||||||
|
*memo = None;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
let (url, code) = extractor::extract_javascript_player_code(video_id)?;
|
|
||||||
state.player_url = Some(url);
|
state.player_url = Some(url);
|
||||||
state.player_code = Some(code);
|
state.player_code = Some(code);
|
||||||
Ok(())
|
Ok(Ensured::Ready { fresh: true })
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Test hook: age every armed memo past FAILURE_COOLDOWN so the probe
|
||||||
|
/// path can be exercised without a five-minute sleep.
|
||||||
|
#[cfg(test)]
|
||||||
|
fn test_expire_memos(&self) {
|
||||||
|
let mut state = self.state.lock();
|
||||||
|
let state: &mut ManagerState = &mut state;
|
||||||
|
for memo in [
|
||||||
|
&mut state.ts_fail,
|
||||||
|
&mut state.sig_fail,
|
||||||
|
&mut state.nsig_fail,
|
||||||
|
] {
|
||||||
|
if let Some(m) = memo.as_mut() {
|
||||||
|
if let Some(aged) = Instant::now().checked_sub(FAILURE_COOLDOWN) {
|
||||||
|
m.at = aged;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Test hook: is the state mutex currently free? Used to prove the
|
||||||
|
/// player.js fetch runs without holding it.
|
||||||
|
#[cfg(test)]
|
||||||
|
fn state_lock_is_free(&self) -> bool {
|
||||||
|
self.state.try_lock().is_some()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -232,6 +542,13 @@ impl Default for PlayerManager {
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
use crate::downloader::request::Request;
|
||||||
|
use crate::downloader::response::Response;
|
||||||
|
use crate::downloader::Downloader;
|
||||||
|
use crate::exceptions::NetworkError;
|
||||||
|
use crate::newpipe::NewPipe;
|
||||||
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||||
|
use std::sync::{mpsc, Arc};
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn assembled_sig_snippet_runs_against_synthetic_player() {
|
fn assembled_sig_snippet_runs_against_synthetic_player() {
|
||||||
|
|
@ -264,4 +581,281 @@ mod tests {
|
||||||
mgr.clear_all_caches();
|
mgr.clear_all_caches();
|
||||||
assert!(mgr.player_url().is_none());
|
assert!(mgr.player_url().is_none());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ---- failure-memo / refetch-storm tests -------------------------------
|
||||||
|
//
|
||||||
|
// These register a stub Downloader in the process-global NewPipe
|
||||||
|
// singleton, so they serialize on GLOBAL_DOWNLOADER_LOCK (no other lib
|
||||||
|
// test touches the downloader). Each uses its own PlayerManager
|
||||||
|
// instance, so manager state never crosses tests.
|
||||||
|
|
||||||
|
static GLOBAL_DOWNLOADER_LOCK: Mutex<()> = Mutex::new(());
|
||||||
|
|
||||||
|
/// Both channel endpoints wrapped in Mutex for the Downloader Sync bound.
|
||||||
|
type FetchBlocker = (Mutex<mpsc::Sender<()>>, Mutex<mpsc::Receiver<()>>);
|
||||||
|
|
||||||
|
/// Serves a synthetic iframe_api (announcing `hash`) and a synthetic
|
||||||
|
/// player.js body; counts every request.
|
||||||
|
struct StubDownloader {
|
||||||
|
hash: Mutex<String>,
|
||||||
|
player_js: Mutex<String>,
|
||||||
|
requests: AtomicUsize,
|
||||||
|
/// When set, the player.js body request signals `entered` then
|
||||||
|
/// blocks until `release` fires — used by the lock-freedom test.
|
||||||
|
block_player_fetch: Option<FetchBlocker>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl StubDownloader {
|
||||||
|
fn new(hash: &str, player_js: &str) -> Self {
|
||||||
|
Self {
|
||||||
|
hash: Mutex::new(hash.to_string()),
|
||||||
|
player_js: Mutex::new(player_js.to_string()),
|
||||||
|
requests: AtomicUsize::new(0),
|
||||||
|
block_player_fetch: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn rotate(&self, hash: &str, player_js: &str) {
|
||||||
|
*self.hash.lock() = hash.to_string();
|
||||||
|
*self.player_js.lock() = player_js.to_string();
|
||||||
|
}
|
||||||
|
|
||||||
|
fn count(&self) -> usize {
|
||||||
|
self.requests.load(Ordering::SeqCst)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Downloader for StubDownloader {
|
||||||
|
fn execute(&self, request: Request) -> Result<Response, NetworkError> {
|
||||||
|
self.requests.fetch_add(1, Ordering::SeqCst);
|
||||||
|
let url = request.url().to_string();
|
||||||
|
if url.contains("iframe_api") {
|
||||||
|
let h = self.hash.lock().clone();
|
||||||
|
// Shape matched by IFRAME_RES_JS_BASE_PLAYER_HASH (escaped
|
||||||
|
// `\/` form, 8-char [a-z0-9] hash).
|
||||||
|
let body = format!(
|
||||||
|
r#"src:"https://www.youtube.com/s/player\/{h}\/player_ias.vflset/en_US/www-embed.js""#
|
||||||
|
);
|
||||||
|
Ok(Response::new(200, "OK", Default::default(), body, url))
|
||||||
|
} else if url.contains("/s/player/") {
|
||||||
|
if let Some((entered, release)) = &self.block_player_fetch {
|
||||||
|
let _ = entered.lock().send(());
|
||||||
|
let _ = release
|
||||||
|
.lock()
|
||||||
|
.recv_timeout(std::time::Duration::from_secs(10));
|
||||||
|
}
|
||||||
|
let body = self.player_js.lock().clone();
|
||||||
|
Ok(Response::new(200, "OK", Default::default(), body, url))
|
||||||
|
} else {
|
||||||
|
Err(NetworkError::Transport("stub: unexpected request".into()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parseable by ts + NOTHING nsig-shaped → nsig build fails while the
|
||||||
|
// signature timestamp extracts fine.
|
||||||
|
const PLAYER_NSIG_BROKEN: &str = r#"var foo={signatureTimestamp:19999};var unrelated=1;"#;
|
||||||
|
|
||||||
|
// nsig name via regex 0 (m85 … the greedy `.*` reaches dummy's
|
||||||
|
// `return Q[1]` on the same line); body via lexer; self-contained
|
||||||
|
// (no outer-global references), so eval succeeds for any input.
|
||||||
|
const PLAYER_GOOD: &str = r#"m85=function(p){var a=p.split("");a.reverse();return a.join("");};dummy=function(q){return Q[1];};var foo={signatureTimestamp:20244};"#;
|
||||||
|
|
||||||
|
// Parses fine but throws for EVERY input at eval time.
|
||||||
|
const PLAYER_EVAL_THROWS: &str = r#"m85=function(p){throw Error("boom");return Q[1];};"#;
|
||||||
|
|
||||||
|
// Parses fine; throws only for the input "evil".
|
||||||
|
const PLAYER_EVAL_SELECTIVE: &str = r#"m85=function(p){if(p==="evil")throw Error("bad");var a=p.split("");a.reverse();return a.join("");};dummy=function(q){return Q[1];};"#;
|
||||||
|
|
||||||
|
fn install(stub: &Arc<StubDownloader>) {
|
||||||
|
NewPipe::init(stub.clone() as Arc<dyn Downloader>);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn nsig_build_failure_memoizes_instead_of_refetching_per_format() {
|
||||||
|
let _g = GLOBAL_DOWNLOADER_LOCK.lock();
|
||||||
|
let stub = Arc::new(StubDownloader::new("aaaaaaaa", PLAYER_NSIG_BROKEN));
|
||||||
|
install(&stub);
|
||||||
|
let mgr = PlayerManager::new();
|
||||||
|
|
||||||
|
// First format: discovery + body download (2 requests), build fails.
|
||||||
|
let err = mgr
|
||||||
|
.url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=AAA")
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(matches!(err, DeobfError::NsigFuncNotFound), "got {err:?}");
|
||||||
|
assert_eq!(stub.count(), 2);
|
||||||
|
|
||||||
|
// Remaining formats of the same open: replayed error, ZERO network.
|
||||||
|
for _ in 0..5 {
|
||||||
|
let err = mgr
|
||||||
|
.url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=BBB")
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(matches!(err, DeobfError::NsigFuncNotFound));
|
||||||
|
}
|
||||||
|
assert_eq!(stub.count(), 2, "memo must suppress refetching");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn artifact_memos_are_independent() {
|
||||||
|
let _g = GLOBAL_DOWNLOADER_LOCK.lock();
|
||||||
|
let stub = Arc::new(StubDownloader::new("aaaaaaaa", PLAYER_NSIG_BROKEN));
|
||||||
|
install(&stub);
|
||||||
|
let mgr = PlayerManager::new();
|
||||||
|
|
||||||
|
// nsig fails and memoizes…
|
||||||
|
assert!(mgr
|
||||||
|
.url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=AAA")
|
||||||
|
.is_err());
|
||||||
|
assert_eq!(stub.count(), 2);
|
||||||
|
// …but the player.js stays installed, so the signature timestamp
|
||||||
|
// extracts from it without any further network.
|
||||||
|
assert_eq!(mgr.signature_timestamp("vid").unwrap(), 19999);
|
||||||
|
assert_eq!(stub.count(), 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn rotation_after_cooldown_fetches_fresh_and_recovers() {
|
||||||
|
let _g = GLOBAL_DOWNLOADER_LOCK.lock();
|
||||||
|
let stub = Arc::new(StubDownloader::new("aaaaaaaa", PLAYER_NSIG_BROKEN));
|
||||||
|
install(&stub);
|
||||||
|
let mgr = PlayerManager::new();
|
||||||
|
|
||||||
|
assert!(mgr
|
||||||
|
.url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=AAA")
|
||||||
|
.is_err());
|
||||||
|
assert_eq!(stub.count(), 2);
|
||||||
|
|
||||||
|
// YouTube rotates to a player.js our bank parses.
|
||||||
|
stub.rotate("bbbbbbbb", PLAYER_GOOD);
|
||||||
|
|
||||||
|
// Within the cooldown the memo still replays — no network.
|
||||||
|
assert!(mgr
|
||||||
|
.url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=AAA")
|
||||||
|
.is_err());
|
||||||
|
assert_eq!(stub.count(), 2);
|
||||||
|
|
||||||
|
// After the cooldown: probe discovers the NEW url → full fetch →
|
||||||
|
// extraction + eval succeed.
|
||||||
|
mgr.test_expire_memos();
|
||||||
|
let out = mgr
|
||||||
|
.url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=abc")
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(out, "https://x/?n=cba");
|
||||||
|
assert_eq!(stub.count(), 4, "probe (iframe) + fresh body download");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn post_cooldown_probe_on_unchanged_player_skips_body_download() {
|
||||||
|
let _g = GLOBAL_DOWNLOADER_LOCK.lock();
|
||||||
|
let stub = Arc::new(StubDownloader::new("aaaaaaaa", PLAYER_NSIG_BROKEN));
|
||||||
|
install(&stub);
|
||||||
|
let mgr = PlayerManager::new();
|
||||||
|
|
||||||
|
assert!(mgr
|
||||||
|
.url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=AAA")
|
||||||
|
.is_err());
|
||||||
|
assert_eq!(stub.count(), 2);
|
||||||
|
|
||||||
|
mgr.test_expire_memos();
|
||||||
|
// Probe: one iframe_api discovery, SAME hash → no body download,
|
||||||
|
// memo re-armed.
|
||||||
|
assert!(mgr
|
||||||
|
.url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=AAA")
|
||||||
|
.is_err());
|
||||||
|
assert_eq!(stub.count(), 3);
|
||||||
|
|
||||||
|
// Re-armed cooldown suppresses even the probe.
|
||||||
|
assert!(mgr
|
||||||
|
.url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=AAA")
|
||||||
|
.is_err());
|
||||||
|
assert_eq!(stub.count(), 3);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn transient_eval_failure_still_self_heals_via_refetch() {
|
||||||
|
let _g = GLOBAL_DOWNLOADER_LOCK.lock();
|
||||||
|
let stub = Arc::new(StubDownloader::new("aaaaaaaa", PLAYER_EVAL_SELECTIVE));
|
||||||
|
install(&stub);
|
||||||
|
let mgr = PlayerManager::new();
|
||||||
|
|
||||||
|
// Working nsig on the first video.
|
||||||
|
let out = mgr
|
||||||
|
.url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=good1")
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(out, "https://x/?n=1doog");
|
||||||
|
assert_eq!(stub.count(), 2);
|
||||||
|
|
||||||
|
// A new-generation n-param chokes the CACHED (not fresh) snippet →
|
||||||
|
// invalidate, NO memo. No network spent on the failing call itself.
|
||||||
|
assert!(mgr
|
||||||
|
.url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=evil")
|
||||||
|
.is_err());
|
||||||
|
assert_eq!(stub.count(), 2);
|
||||||
|
|
||||||
|
// YouTube has rotated; the very next call re-fetches and succeeds —
|
||||||
|
// the transient-rotation self-heal must NOT be suppressed.
|
||||||
|
stub.rotate("bbbbbbbb", PLAYER_GOOD);
|
||||||
|
let out = mgr
|
||||||
|
.url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=evil")
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(out, "https://x/?n=live");
|
||||||
|
assert_eq!(stub.count(), 4);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn persistent_eval_failure_on_fresh_player_memoizes() {
|
||||||
|
let _g = GLOBAL_DOWNLOADER_LOCK.lock();
|
||||||
|
let stub = Arc::new(StubDownloader::new("aaaaaaaa", PLAYER_EVAL_THROWS));
|
||||||
|
install(&stub);
|
||||||
|
let mgr = PlayerManager::new();
|
||||||
|
|
||||||
|
// Fresh download + successful build, then eval throws → memo.
|
||||||
|
let err = mgr
|
||||||
|
.url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=AAA")
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(matches!(err, DeobfError::JsRuntimeFailed(_)), "got {err:?}");
|
||||||
|
assert_eq!(stub.count(), 2);
|
||||||
|
|
||||||
|
// Every further format replays without network.
|
||||||
|
for _ in 0..5 {
|
||||||
|
assert!(mgr
|
||||||
|
.url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=BBB")
|
||||||
|
.is_err());
|
||||||
|
}
|
||||||
|
assert_eq!(stub.count(), 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn state_lock_stays_free_during_player_fetch() {
|
||||||
|
let _g = GLOBAL_DOWNLOADER_LOCK.lock();
|
||||||
|
let (entered_tx, entered_rx) = mpsc::channel();
|
||||||
|
let (release_tx, release_rx) = mpsc::channel();
|
||||||
|
let mut stub = StubDownloader::new("aaaaaaaa", PLAYER_GOOD);
|
||||||
|
stub.block_player_fetch = Some((Mutex::new(entered_tx), Mutex::new(release_rx)));
|
||||||
|
let stub = Arc::new(stub);
|
||||||
|
install(&stub);
|
||||||
|
|
||||||
|
let mgr: &'static PlayerManager = Box::leak(Box::new(PlayerManager::new()));
|
||||||
|
let worker = std::thread::spawn(move || {
|
||||||
|
mgr.url_with_throttling_parameter_deobfuscated("vid", "https://x/?n=abc")
|
||||||
|
});
|
||||||
|
|
||||||
|
// Wait until the worker is INSIDE the player.js body download.
|
||||||
|
entered_rx
|
||||||
|
.recv_timeout(std::time::Duration::from_secs(10))
|
||||||
|
.expect("worker never reached the player.js fetch");
|
||||||
|
|
||||||
|
// The whole point of the fetch_gate: readers aren't blocked while
|
||||||
|
// ~1.7 MB downloads. The old design held this mutex across the
|
||||||
|
// network and serialized everything behind it.
|
||||||
|
assert!(
|
||||||
|
mgr.state_lock_is_free(),
|
||||||
|
"state mutex must not be held across the player.js download"
|
||||||
|
);
|
||||||
|
assert_eq!(mgr.throttling_parameter_cache_size(), 0); // must not block
|
||||||
|
|
||||||
|
release_tx.send(()).unwrap();
|
||||||
|
let out = worker.join().unwrap().unwrap();
|
||||||
|
assert_eq!(out, "https://x/?n=cba");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -10,7 +10,7 @@
|
||||||
|
|
||||||
use url::Url;
|
use url::Url;
|
||||||
|
|
||||||
use crate::youtube::linkhandler::{host_is_youtube, LinkError};
|
use crate::youtube::linkhandler::{host_is_youtube, redact_url, LinkError};
|
||||||
|
|
||||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||||
pub enum ChannelIdentifier {
|
pub enum ChannelIdentifier {
|
||||||
|
|
@ -25,8 +25,10 @@ pub enum ChannelIdentifier {
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn parse(url_str: &str) -> Result<ChannelIdentifier, LinkError> {
|
pub fn parse(url_str: &str) -> Result<ChannelIdentifier, LinkError> {
|
||||||
|
// Privacy: never embed the raw input in the error (see stream.rs) —
|
||||||
|
// the parse reason alone suffices.
|
||||||
let url = Url::parse(url_str)
|
let url = Url::parse(url_str)
|
||||||
.map_err(|e| LinkError::InvalidUrl(format!("{url_str}: {e}")))?;
|
.map_err(|e| LinkError::InvalidUrl(e.to_string()))?;
|
||||||
let host = url
|
let host = url
|
||||||
.host_str()
|
.host_str()
|
||||||
.ok_or_else(|| LinkError::InvalidUrl("no host".into()))?;
|
.ok_or_else(|| LinkError::InvalidUrl("no host".into()))?;
|
||||||
|
|
@ -37,32 +39,32 @@ pub fn parse(url_str: &str) -> Result<ChannelIdentifier, LinkError> {
|
||||||
if let Some(rest) = path.strip_prefix("/channel/") {
|
if let Some(rest) = path.strip_prefix("/channel/") {
|
||||||
let id = rest.split('/').next().unwrap_or("");
|
let id = rest.split('/').next().unwrap_or("");
|
||||||
if id.is_empty() {
|
if id.is_empty() {
|
||||||
return Err(LinkError::MissingId(url_str.into()));
|
return Err(LinkError::MissingId(redact_url(&url)));
|
||||||
}
|
}
|
||||||
return Ok(ChannelIdentifier::DirectId(id.into()));
|
return Ok(ChannelIdentifier::DirectId(id.into()));
|
||||||
}
|
}
|
||||||
if let Some(rest) = path.strip_prefix("/c/") {
|
if let Some(rest) = path.strip_prefix("/c/") {
|
||||||
let s = rest.split('/').next().unwrap_or("");
|
let s = rest.split('/').next().unwrap_or("");
|
||||||
if s.is_empty() {
|
if s.is_empty() {
|
||||||
return Err(LinkError::MissingId(url_str.into()));
|
return Err(LinkError::MissingId(redact_url(&url)));
|
||||||
}
|
}
|
||||||
return Ok(ChannelIdentifier::Custom(s.into()));
|
return Ok(ChannelIdentifier::Custom(s.into()));
|
||||||
}
|
}
|
||||||
if let Some(rest) = path.strip_prefix("/user/") {
|
if let Some(rest) = path.strip_prefix("/user/") {
|
||||||
let s = rest.split('/').next().unwrap_or("");
|
let s = rest.split('/').next().unwrap_or("");
|
||||||
if s.is_empty() {
|
if s.is_empty() {
|
||||||
return Err(LinkError::MissingId(url_str.into()));
|
return Err(LinkError::MissingId(redact_url(&url)));
|
||||||
}
|
}
|
||||||
return Ok(ChannelIdentifier::LegacyUser(s.into()));
|
return Ok(ChannelIdentifier::LegacyUser(s.into()));
|
||||||
}
|
}
|
||||||
if let Some(rest) = path.strip_prefix("/@") {
|
if let Some(rest) = path.strip_prefix("/@") {
|
||||||
let s = rest.split('/').next().unwrap_or("");
|
let s = rest.split('/').next().unwrap_or("");
|
||||||
if s.is_empty() {
|
if s.is_empty() {
|
||||||
return Err(LinkError::MissingId(url_str.into()));
|
return Err(LinkError::MissingId(redact_url(&url)));
|
||||||
}
|
}
|
||||||
return Ok(ChannelIdentifier::Handle(s.into()));
|
return Ok(ChannelIdentifier::Handle(s.into()));
|
||||||
}
|
}
|
||||||
Err(LinkError::MissingId(url_str.into()))
|
Err(LinkError::MissingId(redact_url(&url)))
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn channel_url(channel_id: &str) -> String {
|
pub fn channel_url(channel_id: &str) -> String {
|
||||||
|
|
|
||||||
|
|
@ -34,6 +34,20 @@ pub const ACCEPTED_HOSTS: &[&str] = &[
|
||||||
"www.youtube-nocookie.com",
|
"www.youtube-nocookie.com",
|
||||||
];
|
];
|
||||||
|
|
||||||
|
/// Renders a URL as scheme+host only, for embedding in error strings.
|
||||||
|
/// Privacy: link errors reach Kotlin exception messages / logs. Query params
|
||||||
|
/// carry user browsing data (`v=<videoId>`, `list=<playlistId>`) AND several
|
||||||
|
/// YouTube PATHS embed the id (`/embed/<id>`, `/shorts/<id>`, `/clip/<id>`,
|
||||||
|
/// `/live/<id>`, `youtu.be/<id>`) — so drop the path too; the host alone says
|
||||||
|
/// which endpoint failed without leaking what the user was watching.
|
||||||
|
pub(crate) fn redact_url(url: &url::Url) -> String {
|
||||||
|
let mut u = url.clone();
|
||||||
|
u.set_query(None);
|
||||||
|
u.set_fragment(None);
|
||||||
|
u.set_path("/");
|
||||||
|
u.to_string()
|
||||||
|
}
|
||||||
|
|
||||||
pub fn host_is_youtube(host: &str) -> bool {
|
pub fn host_is_youtube(host: &str) -> bool {
|
||||||
let h = host.to_ascii_lowercase();
|
let h = host.to_ascii_lowercase();
|
||||||
let h = h.strip_prefix("www.").unwrap_or(&h);
|
let h = h.strip_prefix("www.").unwrap_or(&h);
|
||||||
|
|
@ -66,4 +80,24 @@ mod tests {
|
||||||
assert!(!host_is_youtube("piped.video"));
|
assert!(!host_is_youtube("piped.video"));
|
||||||
assert!(!host_is_youtube("evil.com"));
|
assert!(!host_is_youtube("evil.com"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn redact_url_drops_id_bearing_path_query_and_fragment() {
|
||||||
|
// Every one of these carries the video/playlist id somewhere the old
|
||||||
|
// query-only strip would have kept (the path, for embed/shorts/youtu.be).
|
||||||
|
let cases = [
|
||||||
|
"https://www.youtube.com/embed/dQw4w9WgXcQ?v=secret#frag",
|
||||||
|
"https://www.youtube.com/shorts/dQw4w9WgXcQ",
|
||||||
|
"https://youtu.be/dQw4w9WgXcQ",
|
||||||
|
"https://www.youtube.com/watch?v=dQw4w9WgXcQ&list=PLsecretlist",
|
||||||
|
];
|
||||||
|
for c in cases {
|
||||||
|
let u = url::Url::parse(c).unwrap();
|
||||||
|
let r = redact_url(&u);
|
||||||
|
assert!(!r.contains("dQw4w9WgXcQ"), "leaked video id: {r}");
|
||||||
|
assert!(!r.contains("PLsecretlist"), "leaked list id: {r}");
|
||||||
|
// scheme + host only (no port in any of these cases).
|
||||||
|
assert_eq!(r, format!("{}://{}/", u.scheme(), u.host_str().unwrap()));
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -15,7 +15,7 @@ use once_cell::sync::Lazy;
|
||||||
use regex::Regex;
|
use regex::Regex;
|
||||||
use url::Url;
|
use url::Url;
|
||||||
|
|
||||||
use crate::youtube::linkhandler::{host_is_youtube, LinkError};
|
use crate::youtube::linkhandler::{host_is_youtube, redact_url, LinkError};
|
||||||
|
|
||||||
const VIDEO_ID_LEN: usize = 11;
|
const VIDEO_ID_LEN: usize = 11;
|
||||||
|
|
||||||
|
|
@ -42,8 +42,11 @@ fn extract_video_id_inner(input_url: &str, depth: u8) -> Result<String, LinkErro
|
||||||
// the JVM via UniFFI. One level is enough for the legitimate
|
// the JVM via UniFFI. One level is enough for the legitimate
|
||||||
// share-from-attribution-app case.
|
// share-from-attribution-app case.
|
||||||
const MAX_ATTRIBUTION_DEPTH: u8 = 1;
|
const MAX_ATTRIBUTION_DEPTH: u8 = 1;
|
||||||
|
// Privacy: never embed the raw input in the error — even a malformed
|
||||||
|
// paste can carry a video id / user browsing data, and link errors
|
||||||
|
// reach exception messages / logs. The parse reason alone suffices.
|
||||||
let url = Url::parse(input_url)
|
let url = Url::parse(input_url)
|
||||||
.map_err(|e| LinkError::InvalidUrl(format!("{input_url}: {e}")))?;
|
.map_err(|e| LinkError::InvalidUrl(e.to_string()))?;
|
||||||
let host = url
|
let host = url
|
||||||
.host_str()
|
.host_str()
|
||||||
.ok_or_else(|| LinkError::InvalidUrl("no host".into()))?;
|
.ok_or_else(|| LinkError::InvalidUrl("no host".into()))?;
|
||||||
|
|
@ -93,7 +96,7 @@ fn extract_video_id_inner(input_url: &str, depth: u8) -> Result<String, LinkErro
|
||||||
}
|
}
|
||||||
|
|
||||||
let id = candidate
|
let id = candidate
|
||||||
.ok_or_else(|| LinkError::MissingId(input_url.into()))?;
|
.ok_or_else(|| LinkError::MissingId(redact_url(&url)))?;
|
||||||
if !is_valid_video_id(&id) {
|
if !is_valid_video_id(&id) {
|
||||||
return Err(LinkError::MalformedId(id));
|
return Err(LinkError::MalformedId(id));
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -640,11 +640,15 @@ fn process_url(
|
||||||
|
|
||||||
// nsig deobf — unconditional (quick-exits internally if no `n=` present).
|
// nsig deobf — unconditional (quick-exits internally if no `n=` present).
|
||||||
// On failure, fall back to the throttled ORIGINAL url rather than dropping
|
// On failure, fall back to the throttled ORIGINAL url rather than dropping
|
||||||
// the format or aborting the video: YouTube still serves it (rate-limited),
|
// the format or aborting the video: YouTube still serves it (rate-limited).
|
||||||
// and PlayerManager has already invalidated its cache so the NEXT format
|
// A stale-cache eval failure invalidates PlayerManager's cache, so the
|
||||||
// re-fetches a fresh player.js and deobfuscates cleanly. One routine
|
// NEXT format re-fetches a fresh player.js and deobfuscates cleanly — one
|
||||||
// player.js rotation therefore costs at most one throttled format, not the
|
// routine rotation costs at most one throttled format. If the FRESH
|
||||||
// whole video. (NPE parity; straw audit 2026-07-04.)
|
// player.js also fails (regex bank miss — a persistent breakage), the
|
||||||
|
// manager's failure memo replays the error for the remaining formats with
|
||||||
|
// zero network instead of re-downloading ~1.7 MB per format, and retries
|
||||||
|
// after a cooldown / as soon as a new player.js ships. (NPE parity; straw
|
||||||
|
// audits 2026-07-04 + 2026-07-28 H2.)
|
||||||
url = PlayerManager::instance()
|
url = PlayerManager::instance()
|
||||||
.url_with_throttling_parameter_deobfuscated(video_id, &url)
|
.url_with_throttling_parameter_deobfuscated(video_id, &url)
|
||||||
.unwrap_or(url);
|
.unwrap_or(url);
|
||||||
|
|
|
||||||
|
|
@ -195,8 +195,12 @@ fn post_youtube(
|
||||||
}
|
}
|
||||||
let resp = downloader.execute(builder.build())?;
|
let resp = downloader.execute(builder.build())?;
|
||||||
if resp.response_code() != 200 {
|
if resp.response_code() != 200 {
|
||||||
|
// Privacy: the full URL carries `id=<videoId>` in the query — never
|
||||||
|
// put it in an error string (they reach Kotlin exception messages
|
||||||
|
// and logs). Scheme+host+path identifies the endpoint just fine.
|
||||||
|
let endpoint = url.split(['?', '#']).next().unwrap_or(url);
|
||||||
return Err(ExtractionError::Network(NetworkError::Transport(format!(
|
return Err(ExtractionError::Network(NetworkError::Transport(format!(
|
||||||
"HTTP {} from {url}",
|
"HTTP {} from {endpoint}",
|
||||||
resp.response_code()
|
resp.response_code()
|
||||||
))));
|
))));
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -9,16 +9,22 @@
|
||||||
|
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use strawcore::downloader::request::Request;
|
use strawcore_core::downloader::request::Request;
|
||||||
use strawcore::downloader::ReqwestDownloader;
|
use strawcore_core::downloader::ReqwestDownloader;
|
||||||
use strawcore::exceptions::NetworkError;
|
use strawcore_core::exceptions::NetworkError;
|
||||||
use strawcore::localization::{ContentCountry, Localization};
|
use strawcore_core::localization::{ContentCountry, Localization};
|
||||||
use strawcore::{Downloader, NewPipe};
|
use strawcore_core::{Downloader, NewPipe};
|
||||||
|
|
||||||
|
/// The Downloader trait dropped its `get` convenience (commit f917e4a);
|
||||||
|
/// mirror it here so the suite reads as before.
|
||||||
|
fn get(dl: &impl Downloader, url: &str) -> Result<strawcore_core::Response, NetworkError> {
|
||||||
|
dl.execute(Request::get(url).build())
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn get_through_default_downloader() {
|
fn get_through_default_downloader() {
|
||||||
let dl = ReqwestDownloader::new().expect("build downloader");
|
let dl = ReqwestDownloader::new().expect("build downloader");
|
||||||
let resp = dl.get("https://httpbin.org/get").expect("transport");
|
let resp = get(&dl, "https://httpbin.org/get").expect("transport");
|
||||||
assert_eq!(resp.response_code(), 200);
|
assert_eq!(resp.response_code(), 200);
|
||||||
assert!(resp.response_body().contains("\"url\""));
|
assert!(resp.response_body().contains("\"url\""));
|
||||||
}
|
}
|
||||||
|
|
@ -26,9 +32,7 @@ fn get_through_default_downloader() {
|
||||||
#[test]
|
#[test]
|
||||||
fn latest_url_follows_redirects() {
|
fn latest_url_follows_redirects() {
|
||||||
let dl = ReqwestDownloader::new().expect("build downloader");
|
let dl = ReqwestDownloader::new().expect("build downloader");
|
||||||
let resp = dl
|
let resp = get(&dl, "https://httpbin.org/redirect/3").expect("transport");
|
||||||
.get("https://httpbin.org/redirect/3")
|
|
||||||
.expect("transport");
|
|
||||||
assert_eq!(resp.response_code(), 200);
|
assert_eq!(resp.response_code(), 200);
|
||||||
assert!(
|
assert!(
|
||||||
resp.latest_url().ends_with("/get"),
|
resp.latest_url().ends_with("/get"),
|
||||||
|
|
@ -40,14 +44,14 @@ fn latest_url_follows_redirects() {
|
||||||
#[test]
|
#[test]
|
||||||
fn non_2xx_returns_ok_not_err() {
|
fn non_2xx_returns_ok_not_err() {
|
||||||
let dl = ReqwestDownloader::new().expect("build downloader");
|
let dl = ReqwestDownloader::new().expect("build downloader");
|
||||||
let resp = dl.get("https://httpbin.org/status/404").expect("transport");
|
let resp = get(&dl, "https://httpbin.org/status/404").expect("transport");
|
||||||
assert_eq!(resp.response_code(), 404);
|
assert_eq!(resp.response_code(), 404);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn http_429_surfaces_as_recaptcha_err() {
|
fn http_429_surfaces_as_recaptcha_err() {
|
||||||
let dl = ReqwestDownloader::new().expect("build downloader");
|
let dl = ReqwestDownloader::new().expect("build downloader");
|
||||||
let err = dl.get("https://httpbin.org/status/429").expect_err("429 must be NetworkError");
|
let err = get(&dl, "https://httpbin.org/status/429").expect_err("429 must be NetworkError");
|
||||||
match err {
|
match err {
|
||||||
NetworkError::Recaptcha { url } => assert!(url.contains("/status/429")),
|
NetworkError::Recaptcha { url } => assert!(url.contains("/status/429")),
|
||||||
other => panic!("expected Recaptcha, got {other:?}"),
|
other => panic!("expected Recaptcha, got {other:?}"),
|
||||||
|
|
@ -72,8 +76,8 @@ fn localization_header_attached_when_enabled() {
|
||||||
#[test]
|
#[test]
|
||||||
fn header_keys_lowercased_in_response() {
|
fn header_keys_lowercased_in_response() {
|
||||||
let dl = ReqwestDownloader::new().expect("build downloader");
|
let dl = ReqwestDownloader::new().expect("build downloader");
|
||||||
let resp = dl.get("https://httpbin.org/get").expect("transport");
|
let resp = get(&dl, "https://httpbin.org/get").expect("transport");
|
||||||
for (k, _) in resp.response_headers() {
|
for k in resp.response_headers().keys() {
|
||||||
assert_eq!(k, &k.to_ascii_lowercase(), "header key {k} not lowercased");
|
assert_eq!(k, &k.to_ascii_lowercase(), "header key {k} not lowercased");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -88,7 +92,9 @@ fn newpipe_singleton_wires_downloader() {
|
||||||
);
|
);
|
||||||
|
|
||||||
let from_global = NewPipe::downloader().expect("downloader registered");
|
let from_global = NewPipe::downloader().expect("downloader registered");
|
||||||
let resp = from_global.get("https://httpbin.org/get").expect("transport");
|
let resp = from_global
|
||||||
|
.execute(Request::get("https://httpbin.org/get").build())
|
||||||
|
.expect("transport");
|
||||||
assert_eq!(resp.response_code(), 200);
|
assert_eq!(resp.response_code(), 200);
|
||||||
assert_eq!(NewPipe::preferred_localization().localization_code(), "en-GB");
|
assert_eq!(NewPipe::preferred_localization().localization_code(), "en-GB");
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -14,7 +14,7 @@
|
||||||
// * url_with_throttling_parameter_deobfuscated round-trip changes &n=
|
// * url_with_throttling_parameter_deobfuscated round-trip changes &n=
|
||||||
// and caches the result
|
// and caches the result
|
||||||
|
|
||||||
use strawcore::youtube::js::{signature, nsig, runtime, DeobfError};
|
use strawcore_core::youtube::js::{signature, nsig, runtime, DeobfError};
|
||||||
|
|
||||||
// Synthetic minified player.js — replicates the shape of real YT player.js.
|
// Synthetic minified player.js — replicates the shape of real YT player.js.
|
||||||
//
|
//
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue