reliability: bound the JS engine + self-heal on player.js rotation
Some checks failed
gitleaks / scan (push) Failing after 5s
Some checks failed
gitleaks / scan (push) Failing after 5s
Straw audit 2026-07-04. Three availability defects in the extraction core: - runtime.rs: cap the embedded QuickJS runtime (64 MiB mem / 2 MiB stack / 5s wall-clock interrupt) so a hostile or looping player.js can't hang the extraction thread or OOM the process. Previously Runtime::new() set no limits. - player_manager.rs: replace the never-cleared sticky-error flags with invalidate-on-failure. Any deobf/extract error now drops the cached player.js so the next call re-fetches fresh and retries, instead of one routine player.js rotation permanently wedging ALL extraction until app restart (clear_all_caches existed but was never called). - stream_extractor.rs: per-format graceful degradation. nsig-deobf failure falls back to the throttled original URL (YouTube still serves it, rate-limited); sig-cipher failure skips just that format. One bad format no longer aborts the whole video, and manifests/captions still populate. NPE parity. - Cargo.toml: release panic=unwind (inert as a straw dep; kept consistent for standalone builds/tests). 120 tests pass, clippy clean.
This commit is contained in:
parent
3740ec8f75
commit
4f56893da6
4 changed files with 127 additions and 77 deletions
|
|
@ -39,7 +39,11 @@ online-tests = []
|
||||||
strip = true
|
strip = true
|
||||||
lto = "thin"
|
lto = "thin"
|
||||||
codegen-units = 1
|
codegen-units = 1
|
||||||
panic = "abort"
|
# unwind: inert when built as a dependency of the Straw cdylib (that workspace's
|
||||||
|
# profile governs the shipped .so — the load-bearing setting lives in
|
||||||
|
# straw/rust/Cargo.toml), but kept consistent so a standalone release build/test
|
||||||
|
# of strawcore-core also unwinds instead of aborting on a core panic.
|
||||||
|
panic = "unwind"
|
||||||
opt-level = "z"
|
opt-level = "z"
|
||||||
|
|
||||||
[profile.dev]
|
[profile.dev]
|
||||||
|
|
|
||||||
|
|
@ -3,18 +3,24 @@
|
||||||
// sole public class in the JS subsystem).
|
// sole public class in the JS subsystem).
|
||||||
//
|
//
|
||||||
// Cache layout:
|
// Cache layout:
|
||||||
// * cached_player_code — process-lifetime, until clear_all_caches
|
// * cached_player_code — process-lifetime, until invalidate()
|
||||||
// * cached_signature_timestamp
|
// * cached_signature_timestamp
|
||||||
// * cached_sig_snippet — assembled JS, ready for runtime::run
|
// * cached_sig_snippet — assembled JS, ready for runtime::run
|
||||||
// * cached_nsig_name + snippet
|
// * cached_nsig_name + snippet
|
||||||
// * cached_throttling_params — obfuscated → deobfuscated cache (per-session)
|
// * cached_throttling_params — obfuscated → deobfuscated cache (per-session)
|
||||||
// * sticky error flags — once an extraction-stage throws, every
|
|
||||||
// subsequent call re-throws the same error
|
|
||||||
// until clear_all_caches resets it
|
|
||||||
//
|
//
|
||||||
// NPE uses static fields and is not thread-safe; callers serialize. We
|
// Failure handling: on ANY deobf/extraction error, invalidate() drops the
|
||||||
// give the same shape via a `Mutex<ManagerState>` — call sites can still
|
// fetched player.js and everything derived from it, so the NEXT call re-fetches
|
||||||
// hammer it from multiple threads safely.
|
// a fresh player.js and retries. YouTube rotates player.js every few hours, so
|
||||||
|
// a failure is far more often "our cached player.js went stale" than "this
|
||||||
|
// player.js is unparseable" — invalidating lets the next attempt self-heal.
|
||||||
|
// (This replaces the previous sticky-error flags, which were set on failure and
|
||||||
|
// never cleared by any code path, permanently wedging ALL extraction for the
|
||||||
|
// process lifetime after a single routine rotation. straw audit, 2026-07-04.)
|
||||||
|
//
|
||||||
|
// NPE uses static fields and is not thread-safe; callers serialize. We give the
|
||||||
|
// same shape via a `Mutex<ManagerState>` — call sites can still hammer it from
|
||||||
|
// multiple threads safely.
|
||||||
|
|
||||||
use parking_lot::Mutex;
|
use parking_lot::Mutex;
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
|
|
@ -35,12 +41,24 @@ struct ManagerState {
|
||||||
nsig_name: Option<String>,
|
nsig_name: Option<String>,
|
||||||
nsig_snippet: Option<String>,
|
nsig_snippet: Option<String>,
|
||||||
throttling_param_cache: HashMap<String, String>,
|
throttling_param_cache: HashMap<String, String>,
|
||||||
|
}
|
||||||
|
|
||||||
// sticky errors — once set, re-throw immediately on every call until
|
impl ManagerState {
|
||||||
// clear_all_caches resets them.
|
/// Drop the fetched player.js and everything derived from it, so the next
|
||||||
sig_timestamp_err: Option<String>,
|
/// call re-fetches a fresh player.js and re-extracts. Called on any
|
||||||
sig_extract_err: Option<String>,
|
/// deobf/extraction failure so a stale-player.js error self-heals on retry
|
||||||
nsig_extract_err: Option<String>,
|
/// instead of wedging extraction for the process lifetime.
|
||||||
|
fn invalidate(&mut self) {
|
||||||
|
self.player_url = None;
|
||||||
|
self.player_code = None;
|
||||||
|
self.signature_timestamp = None;
|
||||||
|
self.sig_snippet = None;
|
||||||
|
self.nsig_name = None;
|
||||||
|
self.nsig_snippet = None;
|
||||||
|
// Keyed by the obfuscated n-param, not by player.js, but it was produced
|
||||||
|
// by the now-suspect deobfuscator, so clear it too.
|
||||||
|
self.throttling_param_cache.clear();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct PlayerManager {
|
pub struct PlayerManager {
|
||||||
|
|
@ -49,7 +67,9 @@ pub struct PlayerManager {
|
||||||
|
|
||||||
impl PlayerManager {
|
impl PlayerManager {
|
||||||
pub fn new() -> Self {
|
pub fn new() -> Self {
|
||||||
Self { inner: Mutex::new(ManagerState::default()) }
|
Self {
|
||||||
|
inner: Mutex::new(ManagerState::default()),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn instance() -> &'static PlayerManager {
|
pub fn instance() -> &'static PlayerManager {
|
||||||
|
|
@ -59,54 +79,58 @@ impl PlayerManager {
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn signature_timestamp(&self, video_id: &str) -> Result<i32, DeobfError> {
|
pub fn signature_timestamp(&self, video_id: &str) -> Result<i32, DeobfError> {
|
||||||
let mut state = self.inner.lock();
|
let r = self.signature_timestamp_inner(video_id);
|
||||||
if let Some(e) = &state.sig_timestamp_err {
|
if r.is_err() {
|
||||||
return Err(DeobfError::SigTimestampMissing).map_err(|_| {
|
self.inner.lock().invalidate();
|
||||||
DeobfError::JsRuntimeFailed(format!("sticky-cached: {e}"))
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
r
|
||||||
|
}
|
||||||
|
|
||||||
|
fn signature_timestamp_inner(&self, video_id: &str) -> Result<i32, DeobfError> {
|
||||||
|
let mut state = self.inner.lock();
|
||||||
if let Some(ts) = state.signature_timestamp {
|
if let Some(ts) = state.signature_timestamp {
|
||||||
return Ok(ts);
|
return Ok(ts);
|
||||||
}
|
}
|
||||||
Self::ensure_player_code(&mut state, video_id)?;
|
Self::ensure_player_code(&mut state, video_id)?;
|
||||||
let code = state.player_code.as_deref().unwrap();
|
let code = state.player_code.as_deref().unwrap();
|
||||||
match signature::signature_timestamp(code) {
|
let ts = signature::signature_timestamp(code)?;
|
||||||
Ok(ts) => {
|
state.signature_timestamp = Some(ts);
|
||||||
state.signature_timestamp = Some(ts);
|
Ok(ts)
|
||||||
Ok(ts)
|
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
state.sig_timestamp_err = Some(e.to_string());
|
|
||||||
Err(e)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn deobfuscate_signature(
|
pub fn deobfuscate_signature(
|
||||||
&self,
|
&self,
|
||||||
video_id: &str,
|
video_id: &str,
|
||||||
obfuscated_signature: &str,
|
obfuscated_signature: &str,
|
||||||
|
) -> Result<String, DeobfError> {
|
||||||
|
let r = self.deobfuscate_signature_inner(video_id, obfuscated_signature);
|
||||||
|
if r.is_err() {
|
||||||
|
self.inner.lock().invalidate();
|
||||||
|
}
|
||||||
|
r
|
||||||
|
}
|
||||||
|
|
||||||
|
fn deobfuscate_signature_inner(
|
||||||
|
&self,
|
||||||
|
video_id: &str,
|
||||||
|
obfuscated_signature: &str,
|
||||||
) -> Result<String, DeobfError> {
|
) -> Result<String, DeobfError> {
|
||||||
let snippet = {
|
let snippet = {
|
||||||
let mut state = self.inner.lock();
|
let mut state = self.inner.lock();
|
||||||
if let Some(e) = &state.sig_extract_err {
|
|
||||||
return Err(DeobfError::JsRuntimeFailed(format!("sticky-cached: {e}")));
|
|
||||||
}
|
|
||||||
if state.sig_snippet.is_none() {
|
if state.sig_snippet.is_none() {
|
||||||
Self::ensure_player_code(&mut state, video_id)?;
|
Self::ensure_player_code(&mut state, video_id)?;
|
||||||
let code = state.player_code.as_deref().unwrap();
|
let code = state.player_code.as_deref().unwrap();
|
||||||
match signature::build_deobfuscator(code) {
|
let s = signature::build_deobfuscator(code)?;
|
||||||
Ok(s) => state.sig_snippet = Some(s),
|
state.sig_snippet = Some(s);
|
||||||
Err(e) => {
|
|
||||||
state.sig_extract_err = Some(e.to_string());
|
|
||||||
return Err(e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
state.sig_snippet.clone().unwrap()
|
state.sig_snippet.clone().unwrap()
|
||||||
};
|
};
|
||||||
|
|
||||||
let result = runtime::run(&snippet, signature::DEOBFUSCATION_FUNCTION_NAME, obfuscated_signature)?;
|
let result = runtime::run(
|
||||||
|
&snippet,
|
||||||
|
signature::DEOBFUSCATION_FUNCTION_NAME,
|
||||||
|
obfuscated_signature,
|
||||||
|
)?;
|
||||||
if result == "null" {
|
if result == "null" {
|
||||||
return Ok(String::new()); // NPE: Objects.requireNonNullElse(..., "")
|
return Ok(String::new()); // NPE: Objects.requireNonNullElse(..., "")
|
||||||
}
|
}
|
||||||
|
|
@ -117,6 +141,18 @@ impl PlayerManager {
|
||||||
&self,
|
&self,
|
||||||
video_id: &str,
|
video_id: &str,
|
||||||
streaming_url: &str,
|
streaming_url: &str,
|
||||||
|
) -> Result<String, DeobfError> {
|
||||||
|
let r = self.url_with_throttling_parameter_deobfuscated_inner(video_id, streaming_url);
|
||||||
|
if r.is_err() {
|
||||||
|
self.inner.lock().invalidate();
|
||||||
|
}
|
||||||
|
r
|
||||||
|
}
|
||||||
|
|
||||||
|
fn url_with_throttling_parameter_deobfuscated_inner(
|
||||||
|
&self,
|
||||||
|
video_id: &str,
|
||||||
|
streaming_url: &str,
|
||||||
) -> Result<String, DeobfError> {
|
) -> Result<String, DeobfError> {
|
||||||
let obf = match nsig::throttling_parameter_from_url(streaming_url) {
|
let obf = match nsig::throttling_parameter_from_url(streaming_url) {
|
||||||
Some(s) => s,
|
Some(s) => s,
|
||||||
|
|
@ -132,22 +168,12 @@ impl PlayerManager {
|
||||||
|
|
||||||
let (name, snippet) = {
|
let (name, snippet) = {
|
||||||
let mut state = self.inner.lock();
|
let mut state = self.inner.lock();
|
||||||
if let Some(e) = &state.nsig_extract_err {
|
|
||||||
return Err(DeobfError::JsRuntimeFailed(format!("sticky-cached: {e}")));
|
|
||||||
}
|
|
||||||
if state.nsig_snippet.is_none() {
|
if state.nsig_snippet.is_none() {
|
||||||
Self::ensure_player_code(&mut state, video_id)?;
|
Self::ensure_player_code(&mut state, video_id)?;
|
||||||
let code = state.player_code.as_deref().unwrap();
|
let code = state.player_code.as_deref().unwrap();
|
||||||
match nsig::build_deobfuscator(code) {
|
let (n, s) = nsig::build_deobfuscator(code)?;
|
||||||
Ok((n, s)) => {
|
state.nsig_name = Some(n);
|
||||||
state.nsig_name = Some(n);
|
state.nsig_snippet = Some(s);
|
||||||
state.nsig_snippet = Some(s);
|
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
state.nsig_extract_err = Some(e.to_string());
|
|
||||||
return Err(e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
(
|
(
|
||||||
state.nsig_name.clone().unwrap(),
|
state.nsig_name.clone().unwrap(),
|
||||||
|
|
@ -162,7 +188,9 @@ impl PlayerManager {
|
||||||
|
|
||||||
{
|
{
|
||||||
let mut state = self.inner.lock();
|
let mut state = self.inner.lock();
|
||||||
state.throttling_param_cache.insert(obf.clone(), deobf.clone());
|
state
|
||||||
|
.throttling_param_cache
|
||||||
|
.insert(obf.clone(), deobf.clone());
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(streaming_url.replace(&obf, &deobf))
|
Ok(streaming_url.replace(&obf, &deobf))
|
||||||
|
|
@ -173,17 +201,7 @@ impl PlayerManager {
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn clear_all_caches(&self) {
|
pub fn clear_all_caches(&self) {
|
||||||
let mut state = self.inner.lock();
|
self.inner.lock().invalidate();
|
||||||
state.player_url = None;
|
|
||||||
state.player_code = None;
|
|
||||||
state.signature_timestamp = None;
|
|
||||||
state.sig_snippet = None;
|
|
||||||
state.nsig_name = None;
|
|
||||||
state.nsig_snippet = None;
|
|
||||||
state.throttling_param_cache.clear();
|
|
||||||
state.sig_timestamp_err = None;
|
|
||||||
state.sig_extract_err = None;
|
|
||||||
state.nsig_extract_err = None;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn clear_throttling_parameters_cache(&self) {
|
pub fn clear_throttling_parameters_cache(&self) {
|
||||||
|
|
|
||||||
|
|
@ -15,10 +15,32 @@
|
||||||
// sig and treats empty as failure for nsig — caller-side decision,
|
// sig and treats empty as failure for nsig — caller-side decision,
|
||||||
// handled in player_manager.
|
// handled in player_manager.
|
||||||
|
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
use rquickjs::{Context, Function, Runtime};
|
use rquickjs::{Context, Function, Runtime};
|
||||||
|
|
||||||
use crate::youtube::js::DeobfError;
|
use crate::youtube::js::DeobfError;
|
||||||
|
|
||||||
|
// Resource caps on the embedded QuickJS runtime. The deobfuscation snippets are
|
||||||
|
// assembled from attacker-influenceable player.js, so a hostile or looping
|
||||||
|
// script must not be able to hang the extraction thread or OOM the process.
|
||||||
|
const JS_MEMORY_LIMIT: usize = 64 * 1024 * 1024; // 64 MiB — deobf needs a fraction.
|
||||||
|
const JS_MAX_STACK: usize = 2 * 1024 * 1024; // 2 MiB — bounds deep recursion.
|
||||||
|
const JS_DEADLINE: Duration = Duration::from_secs(5); // wall-clock cap on execution.
|
||||||
|
|
||||||
|
/// A `Runtime` with memory / stack / wall-clock limits installed. The interrupt
|
||||||
|
/// handler fires periodically during execution and aborts it once the deadline
|
||||||
|
/// passes; the abort surfaces as an ordinary QuickJS error at the eval site
|
||||||
|
/// (caught by the existing `map_err`) instead of wedging the blocking thread.
|
||||||
|
fn guarded_runtime() -> Result<Runtime, rquickjs::Error> {
|
||||||
|
let runtime = Runtime::new()?;
|
||||||
|
runtime.set_memory_limit(JS_MEMORY_LIMIT);
|
||||||
|
runtime.set_max_stack_size(JS_MAX_STACK);
|
||||||
|
let deadline = Instant::now() + JS_DEADLINE;
|
||||||
|
runtime.set_interrupt_handler(Some(Box::new(move || Instant::now() >= deadline)));
|
||||||
|
Ok(runtime)
|
||||||
|
}
|
||||||
|
|
||||||
/// Returns Ok(()) if the snippet parses; otherwise returns the QuickJS
|
/// Returns Ok(()) if the snippet parses; otherwise returns the QuickJS
|
||||||
/// error message. Mirrors NPE `JavaScript.compileOrThrow`.
|
/// error message. Mirrors NPE `JavaScript.compileOrThrow`.
|
||||||
///
|
///
|
||||||
|
|
@ -27,7 +49,7 @@ use crate::youtube::js::DeobfError;
|
||||||
/// This forces the parser to walk the whole body without executing any
|
/// This forces the parser to walk the whole body without executing any
|
||||||
/// side-effects.
|
/// side-effects.
|
||||||
pub fn compile_or_throw(snippet: &str) -> Result<(), DeobfError> {
|
pub fn compile_or_throw(snippet: &str) -> Result<(), DeobfError> {
|
||||||
let runtime = Runtime::new().map_err(|e| DeobfError::JsCompileFailed(e.to_string()))?;
|
let runtime = guarded_runtime().map_err(|e| DeobfError::JsCompileFailed(e.to_string()))?;
|
||||||
let context =
|
let context =
|
||||||
Context::full(&runtime).map_err(|e| DeobfError::JsCompileFailed(e.to_string()))?;
|
Context::full(&runtime).map_err(|e| DeobfError::JsCompileFailed(e.to_string()))?;
|
||||||
let wrapped = format!("(function(){{{snippet}}})");
|
let wrapped = format!("(function(){{{snippet}}})");
|
||||||
|
|
@ -42,7 +64,7 @@ pub fn compile_or_throw(snippet: &str) -> Result<(), DeobfError> {
|
||||||
/// with one string argument, returns the toString of the result.
|
/// with one string argument, returns the toString of the result.
|
||||||
/// Mirrors NPE `JavaScript.run(snippet, functionName, parameters)`.
|
/// Mirrors NPE `JavaScript.run(snippet, functionName, parameters)`.
|
||||||
pub fn run(snippet: &str, function_name: &str, parameter: &str) -> Result<String, DeobfError> {
|
pub fn run(snippet: &str, function_name: &str, parameter: &str) -> Result<String, DeobfError> {
|
||||||
let runtime = Runtime::new().map_err(|e| DeobfError::JsRuntimeFailed(e.to_string()))?;
|
let runtime = guarded_runtime().map_err(|e| DeobfError::JsRuntimeFailed(e.to_string()))?;
|
||||||
let context =
|
let context =
|
||||||
Context::full(&runtime).map_err(|e| DeobfError::JsRuntimeFailed(e.to_string()))?;
|
Context::full(&runtime).map_err(|e| DeobfError::JsRuntimeFailed(e.to_string()))?;
|
||||||
context.with(|ctx| -> Result<String, DeobfError> {
|
context.with(|ctx| -> Result<String, DeobfError> {
|
||||||
|
|
|
||||||
|
|
@ -23,7 +23,7 @@
|
||||||
|
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
|
|
||||||
use crate::exceptions::{ContentUnavailable, ExtractionError, ParsingError};
|
use crate::exceptions::{ContentUnavailable, ExtractionError};
|
||||||
use crate::image::{Image, ResolutionLevel};
|
use crate::image::{Image, ResolutionLevel};
|
||||||
use crate::localization::{ContentCountry, Localization};
|
use crate::localization::{ContentCountry, Localization};
|
||||||
use crate::newpipe::NewPipe;
|
use crate::newpipe::NewPipe;
|
||||||
|
|
@ -628,20 +628,26 @@ fn process_url(
|
||||||
if base.is_empty() {
|
if base.is_empty() {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
}
|
}
|
||||||
let deobf = PlayerManager::instance()
|
// Without the signature there's no playable URL — skip THIS format
|
||||||
.deobfuscate_signature(video_id, s)
|
// rather than aborting the whole video. (WEB-family path, rarely hit in
|
||||||
.map_err(|e| {
|
// the Android-primary flow.)
|
||||||
ExtractionError::Parsing(ParsingError::Invalid(format!("sig deobf: {e}")))
|
let deobf = match PlayerManager::instance().deobfuscate_signature(video_id, s) {
|
||||||
})?;
|
Ok(d) => d,
|
||||||
|
Err(_) => return Ok(None),
|
||||||
|
};
|
||||||
format!("{base}&{sp}={deobf}")
|
format!("{base}&{sp}={deobf}")
|
||||||
};
|
};
|
||||||
|
|
||||||
// nsig deobf — unconditional. Quick-exit if no `n=` present.
|
// nsig deobf — unconditional (quick-exits internally if no `n=` present).
|
||||||
|
// On failure, fall back to the throttled ORIGINAL url rather than dropping
|
||||||
|
// the format or aborting the video: YouTube still serves it (rate-limited),
|
||||||
|
// and PlayerManager has already invalidated its cache so the NEXT format
|
||||||
|
// re-fetches a fresh player.js and deobfuscates cleanly. One routine
|
||||||
|
// player.js rotation therefore costs at most one throttled format, not the
|
||||||
|
// whole video. (NPE parity; straw audit 2026-07-04.)
|
||||||
url = PlayerManager::instance()
|
url = PlayerManager::instance()
|
||||||
.url_with_throttling_parameter_deobfuscated(video_id, &url)
|
.url_with_throttling_parameter_deobfuscated(video_id, &url)
|
||||||
.map_err(|e| {
|
.unwrap_or(url);
|
||||||
ExtractionError::Parsing(ParsingError::Invalid(format!("nsig deobf: {e}")))
|
|
||||||
})?;
|
|
||||||
|
|
||||||
let sep_cpn = if url.contains('?') { '&' } else { '?' };
|
let sep_cpn = if url.contains('?') { '&' } else { '?' };
|
||||||
url = format!("{url}{sep_cpn}cpn={cpn}");
|
url = format!("{url}{sep_cpn}cpn={cpn}");
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue