reliability: bound the JS engine + self-heal on player.js rotation
Some checks failed
gitleaks / scan (push) Failing after 5s
Some checks failed
gitleaks / scan (push) Failing after 5s
Straw audit 2026-07-04. Three availability defects in the extraction core: - runtime.rs: cap the embedded QuickJS runtime (64 MiB mem / 2 MiB stack / 5s wall-clock interrupt) so a hostile or looping player.js can't hang the extraction thread or OOM the process. Previously Runtime::new() set no limits. - player_manager.rs: replace the never-cleared sticky-error flags with invalidate-on-failure. Any deobf/extract error now drops the cached player.js so the next call re-fetches fresh and retries, instead of one routine player.js rotation permanently wedging ALL extraction until app restart (clear_all_caches existed but was never called). - stream_extractor.rs: per-format graceful degradation. nsig-deobf failure falls back to the throttled original URL (YouTube still serves it, rate-limited); sig-cipher failure skips just that format. One bad format no longer aborts the whole video, and manifests/captions still populate. NPE parity. - Cargo.toml: release panic=unwind (inert as a straw dep; kept consistent for standalone builds/tests). 120 tests pass, clippy clean.
This commit is contained in:
parent
3740ec8f75
commit
4f56893da6
4 changed files with 127 additions and 77 deletions
|
|
@ -39,7 +39,11 @@ online-tests = []
|
|||
strip = true
|
||||
lto = "thin"
|
||||
codegen-units = 1
|
||||
panic = "abort"
|
||||
# unwind: inert when built as a dependency of the Straw cdylib (that workspace's
|
||||
# profile governs the shipped .so — the load-bearing setting lives in
|
||||
# straw/rust/Cargo.toml), but kept consistent so a standalone release build/test
|
||||
# of strawcore-core also unwinds instead of aborting on a core panic.
|
||||
panic = "unwind"
|
||||
opt-level = "z"
|
||||
|
||||
[profile.dev]
|
||||
|
|
|
|||
|
|
@ -3,18 +3,24 @@
|
|||
// sole public class in the JS subsystem).
|
||||
//
|
||||
// Cache layout:
|
||||
// * cached_player_code — process-lifetime, until clear_all_caches
|
||||
// * cached_player_code — process-lifetime, until invalidate()
|
||||
// * cached_signature_timestamp
|
||||
// * cached_sig_snippet — assembled JS, ready for runtime::run
|
||||
// * cached_nsig_name + snippet
|
||||
// * cached_throttling_params — obfuscated → deobfuscated cache (per-session)
|
||||
// * sticky error flags — once an extraction-stage throws, every
|
||||
// subsequent call re-throws the same error
|
||||
// until clear_all_caches resets it
|
||||
//
|
||||
// NPE uses static fields and is not thread-safe; callers serialize. We
|
||||
// give the same shape via a `Mutex<ManagerState>` — call sites can still
|
||||
// hammer it from multiple threads safely.
|
||||
// Failure handling: on ANY deobf/extraction error, invalidate() drops the
|
||||
// fetched player.js and everything derived from it, so the NEXT call re-fetches
|
||||
// a fresh player.js and retries. YouTube rotates player.js every few hours, so
|
||||
// a failure is far more often "our cached player.js went stale" than "this
|
||||
// player.js is unparseable" — invalidating lets the next attempt self-heal.
|
||||
// (This replaces the previous sticky-error flags, which were set on failure and
|
||||
// never cleared by any code path, permanently wedging ALL extraction for the
|
||||
// process lifetime after a single routine rotation. straw audit, 2026-07-04.)
|
||||
//
|
||||
// NPE uses static fields and is not thread-safe; callers serialize. We give the
|
||||
// same shape via a `Mutex<ManagerState>` — call sites can still hammer it from
|
||||
// multiple threads safely.
|
||||
|
||||
use parking_lot::Mutex;
|
||||
use std::collections::HashMap;
|
||||
|
|
@ -35,12 +41,24 @@ struct ManagerState {
|
|||
nsig_name: Option<String>,
|
||||
nsig_snippet: Option<String>,
|
||||
throttling_param_cache: HashMap<String, String>,
|
||||
}
|
||||
|
||||
// sticky errors — once set, re-throw immediately on every call until
|
||||
// clear_all_caches resets them.
|
||||
sig_timestamp_err: Option<String>,
|
||||
sig_extract_err: Option<String>,
|
||||
nsig_extract_err: Option<String>,
|
||||
impl ManagerState {
|
||||
/// Drop the fetched player.js and everything derived from it, so the next
|
||||
/// call re-fetches a fresh player.js and re-extracts. Called on any
|
||||
/// deobf/extraction failure so a stale-player.js error self-heals on retry
|
||||
/// instead of wedging extraction for the process lifetime.
|
||||
fn invalidate(&mut self) {
|
||||
self.player_url = None;
|
||||
self.player_code = None;
|
||||
self.signature_timestamp = None;
|
||||
self.sig_snippet = None;
|
||||
self.nsig_name = None;
|
||||
self.nsig_snippet = None;
|
||||
// Keyed by the obfuscated n-param, not by player.js, but it was produced
|
||||
// by the now-suspect deobfuscator, so clear it too.
|
||||
self.throttling_param_cache.clear();
|
||||
}
|
||||
}
|
||||
|
||||
pub struct PlayerManager {
|
||||
|
|
@ -49,7 +67,9 @@ pub struct PlayerManager {
|
|||
|
||||
impl PlayerManager {
|
||||
pub fn new() -> Self {
|
||||
Self { inner: Mutex::new(ManagerState::default()) }
|
||||
Self {
|
||||
inner: Mutex::new(ManagerState::default()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn instance() -> &'static PlayerManager {
|
||||
|
|
@ -59,54 +79,58 @@ impl PlayerManager {
|
|||
}
|
||||
|
||||
pub fn signature_timestamp(&self, video_id: &str) -> Result<i32, DeobfError> {
|
||||
let mut state = self.inner.lock();
|
||||
if let Some(e) = &state.sig_timestamp_err {
|
||||
return Err(DeobfError::SigTimestampMissing).map_err(|_| {
|
||||
DeobfError::JsRuntimeFailed(format!("sticky-cached: {e}"))
|
||||
});
|
||||
let r = self.signature_timestamp_inner(video_id);
|
||||
if r.is_err() {
|
||||
self.inner.lock().invalidate();
|
||||
}
|
||||
r
|
||||
}
|
||||
|
||||
fn signature_timestamp_inner(&self, video_id: &str) -> Result<i32, DeobfError> {
|
||||
let mut state = self.inner.lock();
|
||||
if let Some(ts) = state.signature_timestamp {
|
||||
return Ok(ts);
|
||||
}
|
||||
Self::ensure_player_code(&mut state, video_id)?;
|
||||
let code = state.player_code.as_deref().unwrap();
|
||||
match signature::signature_timestamp(code) {
|
||||
Ok(ts) => {
|
||||
state.signature_timestamp = Some(ts);
|
||||
Ok(ts)
|
||||
}
|
||||
Err(e) => {
|
||||
state.sig_timestamp_err = Some(e.to_string());
|
||||
Err(e)
|
||||
}
|
||||
}
|
||||
let ts = signature::signature_timestamp(code)?;
|
||||
state.signature_timestamp = Some(ts);
|
||||
Ok(ts)
|
||||
}
|
||||
|
||||
pub fn deobfuscate_signature(
|
||||
&self,
|
||||
video_id: &str,
|
||||
obfuscated_signature: &str,
|
||||
) -> Result<String, DeobfError> {
|
||||
let r = self.deobfuscate_signature_inner(video_id, obfuscated_signature);
|
||||
if r.is_err() {
|
||||
self.inner.lock().invalidate();
|
||||
}
|
||||
r
|
||||
}
|
||||
|
||||
fn deobfuscate_signature_inner(
|
||||
&self,
|
||||
video_id: &str,
|
||||
obfuscated_signature: &str,
|
||||
) -> Result<String, DeobfError> {
|
||||
let snippet = {
|
||||
let mut state = self.inner.lock();
|
||||
if let Some(e) = &state.sig_extract_err {
|
||||
return Err(DeobfError::JsRuntimeFailed(format!("sticky-cached: {e}")));
|
||||
}
|
||||
if state.sig_snippet.is_none() {
|
||||
Self::ensure_player_code(&mut state, video_id)?;
|
||||
let code = state.player_code.as_deref().unwrap();
|
||||
match signature::build_deobfuscator(code) {
|
||||
Ok(s) => state.sig_snippet = Some(s),
|
||||
Err(e) => {
|
||||
state.sig_extract_err = Some(e.to_string());
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
let s = signature::build_deobfuscator(code)?;
|
||||
state.sig_snippet = Some(s);
|
||||
}
|
||||
state.sig_snippet.clone().unwrap()
|
||||
};
|
||||
|
||||
let result = runtime::run(&snippet, signature::DEOBFUSCATION_FUNCTION_NAME, obfuscated_signature)?;
|
||||
let result = runtime::run(
|
||||
&snippet,
|
||||
signature::DEOBFUSCATION_FUNCTION_NAME,
|
||||
obfuscated_signature,
|
||||
)?;
|
||||
if result == "null" {
|
||||
return Ok(String::new()); // NPE: Objects.requireNonNullElse(..., "")
|
||||
}
|
||||
|
|
@ -117,6 +141,18 @@ impl PlayerManager {
|
|||
&self,
|
||||
video_id: &str,
|
||||
streaming_url: &str,
|
||||
) -> Result<String, DeobfError> {
|
||||
let r = self.url_with_throttling_parameter_deobfuscated_inner(video_id, streaming_url);
|
||||
if r.is_err() {
|
||||
self.inner.lock().invalidate();
|
||||
}
|
||||
r
|
||||
}
|
||||
|
||||
fn url_with_throttling_parameter_deobfuscated_inner(
|
||||
&self,
|
||||
video_id: &str,
|
||||
streaming_url: &str,
|
||||
) -> Result<String, DeobfError> {
|
||||
let obf = match nsig::throttling_parameter_from_url(streaming_url) {
|
||||
Some(s) => s,
|
||||
|
|
@ -132,22 +168,12 @@ impl PlayerManager {
|
|||
|
||||
let (name, snippet) = {
|
||||
let mut state = self.inner.lock();
|
||||
if let Some(e) = &state.nsig_extract_err {
|
||||
return Err(DeobfError::JsRuntimeFailed(format!("sticky-cached: {e}")));
|
||||
}
|
||||
if state.nsig_snippet.is_none() {
|
||||
Self::ensure_player_code(&mut state, video_id)?;
|
||||
let code = state.player_code.as_deref().unwrap();
|
||||
match nsig::build_deobfuscator(code) {
|
||||
Ok((n, s)) => {
|
||||
state.nsig_name = Some(n);
|
||||
state.nsig_snippet = Some(s);
|
||||
}
|
||||
Err(e) => {
|
||||
state.nsig_extract_err = Some(e.to_string());
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
let (n, s) = nsig::build_deobfuscator(code)?;
|
||||
state.nsig_name = Some(n);
|
||||
state.nsig_snippet = Some(s);
|
||||
}
|
||||
(
|
||||
state.nsig_name.clone().unwrap(),
|
||||
|
|
@ -162,7 +188,9 @@ impl PlayerManager {
|
|||
|
||||
{
|
||||
let mut state = self.inner.lock();
|
||||
state.throttling_param_cache.insert(obf.clone(), deobf.clone());
|
||||
state
|
||||
.throttling_param_cache
|
||||
.insert(obf.clone(), deobf.clone());
|
||||
}
|
||||
|
||||
Ok(streaming_url.replace(&obf, &deobf))
|
||||
|
|
@ -173,17 +201,7 @@ impl PlayerManager {
|
|||
}
|
||||
|
||||
pub fn clear_all_caches(&self) {
|
||||
let mut state = self.inner.lock();
|
||||
state.player_url = None;
|
||||
state.player_code = None;
|
||||
state.signature_timestamp = None;
|
||||
state.sig_snippet = None;
|
||||
state.nsig_name = None;
|
||||
state.nsig_snippet = None;
|
||||
state.throttling_param_cache.clear();
|
||||
state.sig_timestamp_err = None;
|
||||
state.sig_extract_err = None;
|
||||
state.nsig_extract_err = None;
|
||||
self.inner.lock().invalidate();
|
||||
}
|
||||
|
||||
pub fn clear_throttling_parameters_cache(&self) {
|
||||
|
|
|
|||
|
|
@ -15,10 +15,32 @@
|
|||
// sig and treats empty as failure for nsig — caller-side decision,
|
||||
// handled in player_manager.
|
||||
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use rquickjs::{Context, Function, Runtime};
|
||||
|
||||
use crate::youtube::js::DeobfError;
|
||||
|
||||
// Resource caps on the embedded QuickJS runtime. The deobfuscation snippets are
|
||||
// assembled from attacker-influenceable player.js, so a hostile or looping
|
||||
// script must not be able to hang the extraction thread or OOM the process.
|
||||
const JS_MEMORY_LIMIT: usize = 64 * 1024 * 1024; // 64 MiB — deobf needs a fraction.
|
||||
const JS_MAX_STACK: usize = 2 * 1024 * 1024; // 2 MiB — bounds deep recursion.
|
||||
const JS_DEADLINE: Duration = Duration::from_secs(5); // wall-clock cap on execution.
|
||||
|
||||
/// A `Runtime` with memory / stack / wall-clock limits installed. The interrupt
|
||||
/// handler fires periodically during execution and aborts it once the deadline
|
||||
/// passes; the abort surfaces as an ordinary QuickJS error at the eval site
|
||||
/// (caught by the existing `map_err`) instead of wedging the blocking thread.
|
||||
fn guarded_runtime() -> Result<Runtime, rquickjs::Error> {
|
||||
let runtime = Runtime::new()?;
|
||||
runtime.set_memory_limit(JS_MEMORY_LIMIT);
|
||||
runtime.set_max_stack_size(JS_MAX_STACK);
|
||||
let deadline = Instant::now() + JS_DEADLINE;
|
||||
runtime.set_interrupt_handler(Some(Box::new(move || Instant::now() >= deadline)));
|
||||
Ok(runtime)
|
||||
}
|
||||
|
||||
/// Returns Ok(()) if the snippet parses; otherwise returns the QuickJS
|
||||
/// error message. Mirrors NPE `JavaScript.compileOrThrow`.
|
||||
///
|
||||
|
|
@ -27,7 +49,7 @@ use crate::youtube::js::DeobfError;
|
|||
/// This forces the parser to walk the whole body without executing any
|
||||
/// side-effects.
|
||||
pub fn compile_or_throw(snippet: &str) -> Result<(), DeobfError> {
|
||||
let runtime = Runtime::new().map_err(|e| DeobfError::JsCompileFailed(e.to_string()))?;
|
||||
let runtime = guarded_runtime().map_err(|e| DeobfError::JsCompileFailed(e.to_string()))?;
|
||||
let context =
|
||||
Context::full(&runtime).map_err(|e| DeobfError::JsCompileFailed(e.to_string()))?;
|
||||
let wrapped = format!("(function(){{{snippet}}})");
|
||||
|
|
@ -42,7 +64,7 @@ pub fn compile_or_throw(snippet: &str) -> Result<(), DeobfError> {
|
|||
/// with one string argument, returns the toString of the result.
|
||||
/// Mirrors NPE `JavaScript.run(snippet, functionName, parameters)`.
|
||||
pub fn run(snippet: &str, function_name: &str, parameter: &str) -> Result<String, DeobfError> {
|
||||
let runtime = Runtime::new().map_err(|e| DeobfError::JsRuntimeFailed(e.to_string()))?;
|
||||
let runtime = guarded_runtime().map_err(|e| DeobfError::JsRuntimeFailed(e.to_string()))?;
|
||||
let context =
|
||||
Context::full(&runtime).map_err(|e| DeobfError::JsRuntimeFailed(e.to_string()))?;
|
||||
context.with(|ctx| -> Result<String, DeobfError> {
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@
|
|||
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::exceptions::{ContentUnavailable, ExtractionError, ParsingError};
|
||||
use crate::exceptions::{ContentUnavailable, ExtractionError};
|
||||
use crate::image::{Image, ResolutionLevel};
|
||||
use crate::localization::{ContentCountry, Localization};
|
||||
use crate::newpipe::NewPipe;
|
||||
|
|
@ -628,20 +628,26 @@ fn process_url(
|
|||
if base.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
let deobf = PlayerManager::instance()
|
||||
.deobfuscate_signature(video_id, s)
|
||||
.map_err(|e| {
|
||||
ExtractionError::Parsing(ParsingError::Invalid(format!("sig deobf: {e}")))
|
||||
})?;
|
||||
// Without the signature there's no playable URL — skip THIS format
|
||||
// rather than aborting the whole video. (WEB-family path, rarely hit in
|
||||
// the Android-primary flow.)
|
||||
let deobf = match PlayerManager::instance().deobfuscate_signature(video_id, s) {
|
||||
Ok(d) => d,
|
||||
Err(_) => return Ok(None),
|
||||
};
|
||||
format!("{base}&{sp}={deobf}")
|
||||
};
|
||||
|
||||
// nsig deobf — unconditional. Quick-exit if no `n=` present.
|
||||
// nsig deobf — unconditional (quick-exits internally if no `n=` present).
|
||||
// On failure, fall back to the throttled ORIGINAL url rather than dropping
|
||||
// the format or aborting the video: YouTube still serves it (rate-limited),
|
||||
// and PlayerManager has already invalidated its cache so the NEXT format
|
||||
// re-fetches a fresh player.js and deobfuscates cleanly. One routine
|
||||
// player.js rotation therefore costs at most one throttled format, not the
|
||||
// whole video. (NPE parity; straw audit 2026-07-04.)
|
||||
url = PlayerManager::instance()
|
||||
.url_with_throttling_parameter_deobfuscated(video_id, &url)
|
||||
.map_err(|e| {
|
||||
ExtractionError::Parsing(ParsingError::Invalid(format!("nsig deobf: {e}")))
|
||||
})?;
|
||||
.unwrap_or(url);
|
||||
|
||||
let sep_cpn = if url.contains('?') { '&' } else { '?' };
|
||||
url = format!("{url}{sep_cpn}cpn={cpn}");
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue